diff --git a/.codecov.yml b/.codecov.yml index cd52e2604d4..3d9d2734e81 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -36,3 +36,8 @@ ignore: - "src/tests/" - "include/xrpl/beast/test/" - "include/xrpl/beast/unit_test/" + # Telemetry modules — conditionally compiled behind XRPL_ENABLE_TELEMETRY, + # which is not enabled in coverage builds. + - "src/xrpld/telemetry/" + - "src/libxrpl/beast/insight/OTelCollector.cpp" + - "include/xrpl/beast/insight/OTelCollector.h" diff --git a/.cspell.config.yaml b/.cspell.config.yaml index e220cd02491..d8679b0a893 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -65,6 +65,7 @@ words: - Btrfs - Buildx - canonicality + - CGNAT - canonicalised - changespq - checkme @@ -98,6 +99,8 @@ words: - dearmor - decryptor - dedented + - Dedup + - dedup - deleteme - demultiplexer - deserializaton @@ -107,14 +110,17 @@ words: - disablerepo - distro - doxyfile + - dthms - dxrpl - elgamal - enabled - enablerepo - endmacro - envrc + - EOCFG - exceptioned - EXPECT_STREQ + - exfiltration - Falco - fcontext - finalizers @@ -123,6 +129,8 @@ words: - fsanitize - funclets - Gamal + - gantt + - Gantt - gcov - gcovr - ghead @@ -130,7 +138,10 @@ words: - godexsoft - gpgcheck - gpgkey + - havetxset - hotwallet + - hicpp + - htpasswd - hwaddress - hwrap - ifndef @@ -162,6 +173,7 @@ words: - libxrpl - llection - LOCALGOOD + - logql - logwstream - lseq - lsmf @@ -169,12 +181,11 @@ words: - mathbunnyru - mcmodel - MEMORYSTATUSEX - - MPTAMM - - MPTDEX - Merkle - Metafuncton - misprediction - missingok + - MPTAMM - mptbalance - MPTDEX - mptflags @@ -203,11 +214,13 @@ words: - nixfmt - nixos - nixpkgs + - NETOP - NOLINT - NOLINTNEXTLINE - nonxrp - noreplace - noripple + - nostd - nostdinc - notifempty - nudb @@ -216,6 +229,8 @@ words: - Nyffenegger - onlatest - ostr + - otelc + - otelcol - pargs - partitioner - paychan @@ -224,8 +239,12 @@ words: - permdex - perminute - permissioned + - pgrep + - pkill + - pimpl - pointee - populator + - pratik - preauth - preauthorization - preauthorize @@ -243,7 +262,9 @@ words: - queuable - Raphson - rcflags + - reparent - replayer + - reqps - rerandomize - rerandomization - rerandomized @@ -291,6 +312,7 @@ words: - statsd - STATSDCOLLECTOR - stissue + - stledgerentry - stnum - stobj - stobject @@ -309,6 +331,7 @@ words: - TMEndpointv2 - toolchain - tparam + - traceql - trixie - tx - txid @@ -316,6 +339,7 @@ words: - txjson - txn - txns + - txqueue - txs - ubsan - UBSAN @@ -365,4 +389,7 @@ words: - xrplf - xxhash - xxhasher - - CGNAT + - xychart + - zpages + - ripplex + - mseconds diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index cf70468e32c..0641538e6d9 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -10,6 +10,12 @@ Loop: xrpld.app xrpld.rpc Loop: xrpld.app xrpld.shamap xrpld.shamap > xrpld.app +Loop: xrpld.app xrpld.telemetry + xrpld.app > xrpld.telemetry + Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay +Loop: xrpld.overlay xrpld.telemetry + xrpld.telemetry ~= xrpld.overlay + diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 3c9c5145165..fb1d20625a3 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -48,6 +48,10 @@ libxrpl.shamap > xrpl.basics libxrpl.shamap > xrpl.nodestore libxrpl.shamap > xrpl.protocol libxrpl.shamap > xrpl.shamap +libxrpl.telemetry > xrpl.basics +libxrpl.telemetry > xrpl.config +libxrpl.telemetry > xrpl.protocol +libxrpl.telemetry > xrpl.telemetry libxrpl.tx > xrpl.basics libxrpl.tx > xrpl.conditions libxrpl.tx > xrpl.core @@ -55,6 +59,7 @@ libxrpl.tx > xrpl.json libxrpl.tx > xrpl.ledger libxrpl.tx > xrpl.protocol libxrpl.tx > xrpl.server +libxrpl.tx > xrpl.telemetry libxrpl.tx > xrpl.tx test.app > test.jtx test.app > test.unit_test @@ -107,6 +112,7 @@ test.csf > xrpl.basics test.csf > xrpld.consensus test.csf > xrpl.json test.csf > xrpl.ledger +test.csf > xrpl.telemetry test.json > test.jtx test.json > xrpl.json test.jtx > test.unit_test @@ -193,6 +199,7 @@ test.unit_test > xrpl.protocol tests.libxrpl > xrpl.basics tests.libxrpl > xrpl.config tests.libxrpl > xrpl.core +tests.libxrpl > xrpld.telemetry tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger tests.libxrpl > xrpl.net @@ -202,6 +209,7 @@ tests.libxrpl > xrpl.protocol_autogen tests.libxrpl > xrpl.resource tests.libxrpl > xrpl.server tests.libxrpl > xrpl.shamap +tests.libxrpl > xrpl.telemetry tests.libxrpl > xrpl.tx xrpl.conditions > xrpl.basics xrpl.conditions > xrpl.protocol @@ -239,10 +247,12 @@ xrpl.server > xrpl.resource xrpl.shamap > xrpl.basics xrpl.shamap > xrpl.nodestore xrpl.shamap > xrpl.protocol +xrpl.telemetry > xrpl.config xrpl.tx > xrpl.basics xrpl.tx > xrpl.core xrpl.tx > xrpl.ledger xrpl.tx > xrpl.protocol +xrpl.tx > xrpl.telemetry xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpl.config @@ -258,11 +268,13 @@ xrpld.app > xrpl.rdb xrpld.app > xrpl.resource xrpld.app > xrpl.server xrpld.app > xrpl.shamap +xrpld.app > xrpl.telemetry xrpld.app > xrpl.tx xrpld.consensus > xrpl.basics xrpld.consensus > xrpl.json xrpld.consensus > xrpl.ledger xrpld.consensus > xrpl.protocol +xrpld.consensus > xrpl.telemetry xrpld.core > xrpl.basics xrpld.core > xrpl.config xrpld.core > xrpl.core @@ -281,6 +293,7 @@ xrpld.overlay > xrpl.protocol xrpld.overlay > xrpl.resource xrpld.overlay > xrpl.server xrpld.overlay > xrpl.shamap +xrpld.overlay > xrpl.telemetry xrpld.overlay > xrpl.tx xrpld.peerfinder > xrpl.basics xrpld.peerfinder > xrpl.config @@ -292,6 +305,7 @@ xrpld.perflog > xrpl.config xrpld.perflog > xrpl.core xrpld.perflog > xrpld.app xrpld.perflog > xrpld.rpc +xrpld.perflog > xrpld.telemetry xrpld.perflog > xrpl.json xrpld.perflog > xrpl.nodestore xrpld.perflog > xrpl.protocol @@ -308,9 +322,20 @@ xrpld.rpc > xrpl.rdb xrpld.rpc > xrpl.resource xrpld.rpc > xrpl.server xrpld.rpc > xrpl.shamap +xrpld.rpc > xrpl.telemetry xrpld.rpc > xrpl.tx xrpld.shamap > xrpl.basics xrpld.shamap > xrpld.core xrpld.shamap > xrpl.nodestore xrpld.shamap > xrpl.protocol xrpld.shamap > xrpl.shamap +xrpld.telemetry > xrpl.basics +xrpld.telemetry > xrpl.core +xrpld.telemetry > xrpld.consensus +xrpld.telemetry > xrpld.core +xrpld.telemetry > xrpl.json +xrpld.telemetry > xrpl.nodestore +xrpld.telemetry > xrpl.protocol +xrpld.telemetry > xrpl.rdb +xrpld.telemetry > xrpl.server +xrpld.telemetry > xrpl.telemetry diff --git a/.github/scripts/otel-naming/README.md b/.github/scripts/otel-naming/README.md new file mode 100644 index 00000000000..660b574ed06 --- /dev/null +++ b/.github/scripts/otel-naming/README.md @@ -0,0 +1,70 @@ +# OTel naming-consistency check + +`check_otel_naming.py` enforces the OpenTelemetry span-attribute naming +convention documented in +[CONTRIBUTING.md](../../../CONTRIBUTING.md#telemetry-span-attribute-naming) +across every layer of the telemetry pipeline. The `*SpanNames.h` constants are +the single source of truth (L1); every other layer must agree with them. + +## Running locally + +``` +python .github/scripts/otel-naming/check_otel_naming.py +``` + +It takes no arguments, can be run from any directory inside the repo, and uses +only the Python standard library (no `pip install`, matching the levelization +check). A non-zero exit code means a violation was found; the output lists each +violation as `RULE | location | token | expected`. + +## What it checks + +The valid key set is **derived dynamically from the OTel code** — there is no +hardcoded allowlist: + +- **L1 keys** come from the `namespace attr { ... }` blocks of every + `*SpanNames.h`, resolving the `makeStr("x")` / `join(seg::a, seg::b)` DSL + (cross-file, so `join(seg::rpc, ...)` resolves `seg::rpc` from the base + `SpanNames.h`). Each constant is resolved against **its own** header, so two + headers that define a same-named constant (e.g. a base `attr::ledgerHash` and + a domain `attr::ledgerHash`) each contribute their real wire key — a later + header cannot clobber an earlier one's value in a flat table. +- **Legitimate dotted keys** = ONLY the keys the code actually sets as resource + attributes, i.e. the entries inside `Telemetry.cpp`'s `Resource::Create({...})` + call: the `semconv::service::*` keys (`service.*`) plus any `attr::` + constants passed there (`xrpl.network.*`). A dotted key that is _declared_ in a + header but never set as a resource attr is a span attribute in resource + clothing — a Rule-A violation, even if it lives in the base `SpanNames.h`. + +### Rules (each fails the build, when its inputs are present) + +| Rule | Check | +| ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A | No stray dotted span-attribute key (only the derived resource keys may be dotted). | +| G | Attribute keys are `lower_snake_case` (`^[a-z][a-z0-9_]*$` per dot-segment) — no camelCase, UPPERCASE, or spaces. | +| F | No string literals as attribute keys or span-name arguments in `setAttribute`/`addEvent`/`span`/`rootSpan`/`childSpan` (`rootSpan` shares `span`'s `(cat, prefix, name)` signature). Attribute _values_ are exempt (runtime data); `*SpanNames.h` definitions and test files are exempt. | +| B | Every collector `spanmetrics.dimensions` name exists in the L1 key set. | +| C | Every Tempo span-filter tag exists in the L1 key set. | +| D | Every dashboard label resolves to an L1 span attribute, a native-metric label (L6, emitted by MetricsRegistry), or a Prometheus/Grafana builtin. TraceQL scope prefixes (`span.`/`resource.`/…) are stripped before the L1 lookup. | +| E | No dotted `xrpl..` attribute key in the runbook (only the L1 resource attrs `xrpl.network.*` may be dotted). Span names, filenames, OTel-standard keys, and metric labels are not flagged. | + +Rule F runs **unconditionally** (it is a purely syntactic check on the +call-sites and needs no `*SpanNames.h`), so a code path that calls +`SpanGuard::span`/`setAttribute` directly without ever defining a header is +still caught. + +### Warnings (printed, never fail the build) + +| Rule | Check | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| H | A namespace-qualified constant (e.g. `foo::bar::myKey`) used at a telemetry call-site is not defined in any `*SpanNames.h`. The constant should live in the proper header; defining it in-place bypasses rules A/G/F. Warns rather than fails — the argument may be a legitimately dynamic value, and the header may live on a later branch. Bare locals and `std::` names are not warned. | + +## Presence-gated + +Every rule runs **only when the source files it needs are present** in the tree +and is otherwise skipped (printed as `SKIP: — `), never failed. +This keeps the check correct no matter how telemetry work is split across PRs — +a stacked chain, one large PR, or independent per-stage PRs where (for example) +the collector config lands before the dashboards. The collector/Tempo/dashboard/ +runbook layers are introduced in later phases; on a branch without them, only +the L1-intrinsic rules (A, G, F) run. diff --git a/.github/scripts/otel-naming/check_otel_naming.py b/.github/scripts/otel-naming/check_otel_naming.py new file mode 100644 index 00000000000..89d02d3339f --- /dev/null +++ b/.github/scripts/otel-naming/check_otel_naming.py @@ -0,0 +1,932 @@ +#!/usr/bin/env python3 + +""" +Usage: check_otel_naming.py +This script takes no parameters and can be called from any directory inside the +repository (it locates the repo root via `git rev-parse`). + +Enforces the OpenTelemetry span-attribute naming convention documented in +CONTRIBUTING.md ("Telemetry span attribute naming") across every layer of the +telemetry pipeline. The `*SpanNames.h` constants are the single source of truth +(L1); every other layer must agree with them. + +Design principles +----------------- +1. No hardcoded allowlist. The set of valid attribute keys — including which + dotted keys are legitimate resource attributes — is derived dynamically by + parsing the repository's own OTel code: + * `*SpanNames.h` `namespace attr { ... }` blocks (the underscore/bare keys + and the `join(seg::..., ...)` dotted resource compositions), and + * the keys the code passes to `Resource::Create({ ... })` in Telemetry.cpp + (the standard `semconv::service::*` keys -> service.name/version/...). + The one narrow, explicit exception is EXTERNAL_INFRA_LABELS (Rules D & E): + identity labels stamped by infrastructure outside this repo's OTel code + (the perf-iac harness), which by definition have no source in-tree to + derive from. Kept separate from the generic Prometheus/Grafana builtins + set so the exception stays visible rather than blending into "things + every OTel setup has". perf-iac's alloy pipeline stamps each identity at + two layers -- dotted on the OTel resource attribute (xrpl.work.item/ + .branch/.node.role, checked by Rule E) and underscore on the derived + Prometheus metric-datapoint label (xrpl_work_item/_branch/_node_role, + checked by Rule D) -- so both forms are exempt from the same constant. + +2. Presence-gated enforcement. Every rule runs ONLY when the source files it + needs are present in the tree, and is otherwise skipped (never failed). This + keeps the check correct no matter how work is split across PRs: a stacked + chain, one large PR, or independent per-stage PRs where (for example) the + collector config lands in a different PR than the dashboards. The check never + assumes a file from another phase/PR exists. + +Layers +------ + L1 code : src/**/*SpanNames.h, include/**/*SpanNames.h (ground truth) + L1 resource : src/libxrpl/telemetry/Telemetry.cpp (dotted allowlist) + L1 callsites : setAttribute/addEvent/span/rootSpan/childSpan in src/**, + include/** + L2 collector : docker/telemetry/otel-collector-config.yaml (spanmetrics dims) + L3 tempo : docker/telemetry/tempo.yaml (span filter tags) + L4 dashboards: docker/telemetry/grafana/dashboards/*.json (PromQL labels) + L5 runbook : docs/telemetry-runbook.md (attr tables) + L6 metrics : MetricsRegistry.cpp instrument labels (native-metric + label keys, a valid dashboard-label source besides L1) + +Rules (each FAILS the build, when its inputs are present) +--------------------------------------------------------- + A No stray dotted span-attribute key. A dotted `.` used as a span + attribute that is not in the derived resource-key set is a violation. + G Attribute keys must be lower_snake_case (^[a-z][a-z0-9_]*$ per segment). + Flags camelCase, UPPERCASE, spaces, and other stray characters. + F No string literals as attribute keys or span-name arguments. The + setAttribute/addEvent key and the span/rootSpan/childSpan prefix/name args + must reference a *SpanNames.h constant, never a "literal". Attribute VALUES + are exempt (runtime data). Definitions inside *SpanNames.h are exempt, and + test files are exempt (they pass arbitrary literals to exercise the API). + B Every collector spanmetrics dimension exists in the L1 key set. + C Every tempo span-filter tag exists in the L1 key set. + D Every dashboard label resolves to an L1 span attribute, an L6 + native-metric label, or a builtin. TraceQL `span.`/`resource.` scope + prefixes are stripped before the L1 lookup. + E No dotted `xrpl..` attribute key in the runbook (only the + L1 resource attrs xrpl.network.* and the EXTERNAL_INFRA_LABELS dotted + form -- xrpl.work.item/.branch/.node.role -- may be dotted). Span names, + filenames, + OTel-standard keys, and metric labels are not flagged. + +Warnings (printed, but do NOT fail the build) +---------------------------------------------- + H A constant referenced at a telemetry call-site is not defined in any + *SpanNames.h. Span constants should live in the corresponding + *SpanNames.h (single source of truth); defining one in-place bypasses the + naming rules. A warning (not a failure) because the argument may instead + be a legitimately dynamic local (e.g. a computed span-name leaf). + +Exit code is non-zero if any present-and-enforced rule finds a violation. +Warnings never change the exit code. +""" + +import re +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +# --------------------------------------------------------------------------- +# Repo location +# --------------------------------------------------------------------------- + + +def repo_root() -> Path: + """Return the repository root, so the script works from any CWD. + + Exits with a readable message (not a traceback) if git is unavailable or the + CWD is outside a repository.""" + try: + out = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + print( + "error: check_otel_naming.py must be run inside the git repository.", + file=sys.stderr, + ) + sys.exit(2) + return Path(out.stdout.strip()) + + +def read_source(path: Path) -> str: + """Read a file as UTF-8, tolerating stray non-UTF-8 bytes rather than + crashing the whole check on one bad byte.""" + return path.read_text(encoding="utf-8", errors="ignore") + + +# --------------------------------------------------------------------------- +# Regexes (compiled once) +# --------------------------------------------------------------------------- + +# A segment/string constant definition: `inline constexpr auto NAME = ;` +CONST_DEF = re.compile(r"inline\s+constexpr\s+auto\s+(\w+)\s*=\s*(.+?);", re.DOTALL) +MAKESTR = re.compile(r'makeStr\(\s*"([^"]*)"\s*\)') +# A `namespace {` opener, to track which namespace a constant lives in. +NS_OPEN = re.compile(r"namespace\s+([\w:]+)\s*\{") +# A `using ::a::b::field;` re-export inside an attr block; captures the leaf. +USING_DECL = re.compile(r"using\s+(?:::)?[\w:]*::(\w+)\s*;") +# Telemetry call-sites whose string arguments must be constants, not literals. +# Require a receiver so we match real SpanGuard calls, not std::span / a math +# `span(...)` / a bare method declaration: +# - `SpanGuard::span(` / `SpanGuard::rootSpan(` / `SpanGuard::childSpan(` +# (static factories) +# - `.span(` / `->setAttribute(` etc. (member call) +# `span`/`rootSpan`/`childSpan` additionally require the `SpanGuard`/`.`/`->` +# receiver; `setAttribute`/`addEvent` only ever exist on a guard, so a `.`/`->` +# suffices. `rootSpan` shares `span`'s (cat, prefix, name) signature. +CALLSITE = re.compile( + r"(?:SpanGuard::|\.|->)\s*(setAttribute|addEvent|span|rootSpan|childSpan)\s*\(" +) +# A C++ string literal (used to flag literals inside call-site argument lists). +STRING_LITERAL = re.compile(r'"((?:[^"\\]|\\.)*)"') +# A C++ line comment (`//` ... end of line) and a block comment (`/* ... */`). +LINE_COMMENT = re.compile(r"//[^\n]*") +BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +# A TraceQL scope prefix on a label (`span.`, `resource.`, `event.`, etc.). +# Dashboards reference span attributes in TraceQL as `span.`; the bare +# attribute is what must exist in L1, so strip the scope before validating. +TRACEQL_SCOPE = re.compile(r"^(?:span|resource|event|link|instrumentation_scope)\.") +# An OTel metric label key as emitted in C++: `Add(.., {{"label", ...}})` / +# `{{"label", value}}` instrument calls in MetricsRegistry. +METRIC_LABEL = re.compile(r'\{\{\s*"([a-z_][a-z0-9_]*)"\s*,') + + +def strip_comments(text: str) -> str: + """Remove C/C++ `//` line comments and `/* ... */` block comments. + + Used only for L1 attribute-key extraction so that a commented-out or + illustrative `makeStr("...")` inside a `namespace attr` block does not leak + into the authoritative key set. Rule F deliberately does NOT strip comments + — it must still see `@code` doc-comment examples so their call-site + arguments are held to the constant-only convention. + + String literals are not specially handled; a `//` or `/*` appearing inside a + string is vanishingly rare in the *SpanNames.h headers and would at worst + drop a constant from L1 (a conservative direction). + """ + text = BLOCK_COMMENT.sub("", text) + text = LINE_COMMENT.sub("", text) + return text + + +# --------------------------------------------------------------------------- +# L1: parse *SpanNames.h into the authoritative key set +# --------------------------------------------------------------------------- + + +def find_spanname_headers(root: Path) -> List[Path]: + return sorted( + p + for p in list((root / "src").rglob("*SpanNames.h")) + + list((root / "include").rglob("*SpanNames.h")) + if p.is_file() + ) + + +def resolve_constants( + text: str, symbols: Optional[Dict[str, str]] = None +) -> Dict[str, str]: + """Resolve `inline constexpr auto NAME = ` to strings. + + Supports the small constexpr DSL used by SpanNames.h: + makeStr("x") -> "x" + join(a, b) -> resolve(a) + "." + resolve(b) + seg::xrpl / attr::foo -> looked up in the symbol table + The optional `symbols` argument seeds (and is updated in place with) the + table, so a global pass over ALL *SpanNames.h headers can resolve + cross-file references such as `join(seg::rpc, ...)` where `seg::rpc` is + defined in the base SpanNames.h. Keys are stored by their bare name + (last `::` component), so `seg::rpc` and `rpc` both resolve. + """ + if symbols is None: + symbols = {} + + def resolve_expr(expr: str) -> Optional[str]: + expr = expr.strip() + m = MAKESTR.fullmatch(expr) + if m: + return m.group(1) + if expr.startswith("join(") and expr.endswith(")"): + args = split_top_level_args(expr[len("join(") : -1]) + parts = [resolve_expr(a) for a in args] + if any(p is None for p in parts): + return None + return ".".join(p for p in parts if p is not None) + # Bare or qualified symbol reference, e.g. `seg::xrpl` or `networkId`. + key = expr.split("::")[-1] + return symbols.get(key, symbols.get(expr)) + + # Iterate definitions in source order so earlier symbols are available. + for m in CONST_DEF.finditer(text): + name, expr = m.group(1), m.group(2) + val = resolve_expr(expr) + if val is not None: + symbols[name] = val + return symbols + + +def build_global_symbols(headers: List[Path]) -> Dict[str, str]: + """Resolve constants across ALL headers so cross-file `seg::`/`join` + references (e.g. `join(seg::rpc, ...)` in RpcSpanNames.h, where `seg::rpc` + lives in the base SpanNames.h) resolve. Base SpanNames.h is processed + first so its `seg::` segments seed the table.""" + symbols: Dict[str, str] = {} + ordered = sorted(headers, key=lambda p: (p.name != "SpanNames.h", str(p))) + # Two passes: the first seeds segments, the second resolves dependents. + # Comments are stripped so a commented-out constant cannot seed the table. + for _ in range(2): + for h in ordered: + resolve_constants(strip_comments(read_source(h)), symbols) + return symbols + + +def split_top_level_args(s: str) -> List[str]: + """Split a comma-separated arg list, respecting nested parentheses and + ignoring parens/commas that appear inside a "string literal" (so a value + like `setAttribute(k, ",")` does not get mis-split).""" + args, depth, cur = [], 0, "" + in_str = False + escaped = False + for ch in s: + if in_str: + cur += ch + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + cur += ch + elif ch == "(": + depth += 1 + cur += ch + elif ch == ")": + depth -= 1 + cur += ch + elif ch == "," and depth == 0: + args.append(cur) + cur = "" + else: + cur += ch + if cur.strip(): + args.append(cur) + return args + + +def attr_namespace_spans(text: str) -> List[str]: + """Return the source text of each `namespace attr { ... }` block in `text`. + + Brace-matched over the whole (comment-stripped) text, so a definition that + wraps across several physical lines is contained in one span. Nested braces + inside the block are balanced correctly.""" + spans: List[str] = [] + for opener in NS_OPEN.finditer(text): + if opener.group(1).split("::")[-1] != "attr": + continue + # Walk from the opening brace, balancing nesting to the matching close. + i = opener.end() # one char past the namespace's `{` + depth = 1 + start = i + while i < len(text) and depth > 0: + c = text[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + i += 1 + spans.append(text[start : i - 1]) + return spans + + +def attr_keys_from_header(path: Path, symbols: Dict[str, str]) -> Set[str]: + """Return the set of attribute-key strings declared in a header's + `namespace attr { ... }` block(s). `symbols` is the global cross-file + table, used ONLY to seed `seg::`/segment references for `join(...)` + resolution — never to look up an attr constant's value. + + A constant DEFINED in this header is resolved against this header's OWN + text, so two headers that each define a same-named constant (e.g. the base + `attr::ledgerHash = xrpl.ledger.hash` and consensus + `attr::ledgerHash = ledger_hash`) each report their real wire key. The + global table is keyed by bare name and would otherwise let a later header + clobber an earlier one, erasing the real key from L1 (a Rule-A blind spot). + A `using`-re-export, by contrast, imports a constant defined elsewhere, so + it is resolved against the global table. + + Comments are stripped first (a commented constant must not enter L1), and + each attr block is brace-matched over the whole text so multi-line + `inline constexpr auto NAME = join(\\n ...);` definitions are captured.""" + text = strip_comments(read_source(path)) + # Local table: the global segments/symbols seed cross-file `join` parts, + # then this header's own definitions overwrite any same-named global entry + # so a locally-defined attr resolves to ITS value, not another header's. + local = dict(symbols) + resolve_constants(text, local) + keys: Set[str] = set() + for block in attr_namespace_spans(text): + for md in CONST_DEF.finditer(block): + # Resolve a locally-defined constant against the LOCAL table; this + # captures makeStr("x") and join(seg::y, ...) with the header's own + # value, immune to cross-header bare-name collisions. + val = local.get(md.group(1)) + if val is not None: + keys.add(val) + # `using ::ns::attr::field;` re-exports a constant defined in ANOTHER + # header (e.g. PeerSpanNames imports the base ledgerHash). Resolve the + # imported name against the global table. + for um in USING_DECL.finditer(block): + val = symbols.get(um.group(1)) + if val is not None: + keys.add(val) + return keys + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- + + +class Report: + def __init__(self) -> None: + self.violations: List[Tuple[str, str, str, str]] = [] + self.warnings: List[Tuple[str, str, str, str]] = [] + self.skips: List[str] = [] + self.checked: List[str] = [] + + def violation(self, rule: str, loc: str, token: str, expected: str) -> None: + self.violations.append((rule, loc, token, expected)) + + def warning(self, rule: str, loc: str, token: str, note: str) -> None: + """A non-fatal finding: printed, but does not fail the build. Used where + the script cannot be certain a finding is wrong (e.g. a constant used at + a call-site that is not defined in any *SpanNames.h — it might be a + misplaced constant, or a legitimately dynamic value).""" + self.warnings.append((rule, loc, token, note)) + + def skip(self, rule: str, reason: str) -> None: + self.skips.append(f"SKIP: {rule} — {reason}") + + def ok(self, msg: str) -> None: + self.checked.append(f"OK: {msg}") + + def render_and_exit(self) -> None: + for line in self.skips: + print(line) + for line in self.checked: + print(line) + if self.warnings: + print("\nNaming-convention warnings (non-fatal):\n") + print(f" {'RULE':<5} {'LOCATION':<48} {'TOKEN':<28} NOTE") + print(f" {'-' * 5} {'-' * 48} {'-' * 28} {'-' * 30}") + for rule, loc, token, note in self.warnings: + print(f" {rule:<5} {loc:<48} {token:<28} {note}") + if self.violations: + print("\nNaming-convention violations:\n") + print(f" {'RULE':<5} {'LOCATION':<48} {'TOKEN':<28} EXPECTED") + print(f" {'-' * 5} {'-' * 48} {'-' * 28} {'-' * 30}") + for rule, loc, token, expected in self.violations: + print(f" {rule:<5} {loc:<48} {token:<28} {expected}") + print( + "\nSee CONTRIBUTING.md -> 'Telemetry span attribute naming'. " + "The *SpanNames.h constants are the single source of truth." + ) + sys.exit(1) + print("\nAll present telemetry naming layers are consistent.") + sys.exit(0) + + +def main() -> None: + root = repo_root() + report = Report() + + # --- Build the L1 ground-truth key set (presence-gated) ---------------- + headers = find_spanname_headers(root) + l1_keys: Set[str] = set() + if headers: + symbols = build_global_symbols(headers) + # Map each key to the header(s) that declare it, so Rule A can tell a + # legitimate resource attr (declared in the base SpanNames.h) from a + # stray dotted key declared in a domain header. + keys_by_header: Dict[Path, Set[str]] = {} + for h in headers: + hk = attr_keys_from_header(h, symbols) + keys_by_header[h] = hk + l1_keys |= hk + report.ok( + f"L1: {len(l1_keys)} attribute keys from {len(headers)} " + f"*SpanNames.h header(s)" + ) + else: + report.skip("L1", "no *SpanNames.h present (not a naming-relevant tree)") + keys_by_header = {} + + # --- Derive the legitimate dotted (resource) keys dynamically ---------- + # ONLY the keys actually passed to Resource::Create() in Telemetry.cpp + # (semconv service.* + the attr:: constants set there, e.g. xrpl.network.*). + # A dotted key declared in a header but NOT set as a resource attr is a + # Rule-A violation, not an allowlist entry. + resource_symbols = symbols if headers else {} + dotted_allow = derive_dotted_resource_keys(root, resource_symbols, report) + + # --- Rule A: no stray dotted span-attribute keys ----------------------- + if l1_keys: + run_rule_a(keys_by_header, dotted_allow, report) + # --- Rule G: keys must be lower_snake_case ----------------------------- + if l1_keys: + run_rule_g(keys_by_header, report) + # --- Rule F (+ Rule H): scan telemetry call-sites ---------------------- + # Runs UNCONDITIONALLY: Rule F is a purely syntactic check (is this argument + # a literal?) and does not need the L1 key set, so a code path that uses + # SpanGuard::span/setAttribute directly without ever defining a *SpanNames.h + # is still caught. Rule H (warning) additionally flags constant references + # not defined in any *SpanNames.h. + header_symbols = spanname_symbol_names(headers) + run_rule_f(root, report, header_symbols) + + # --- Cross-layer rules B/C/D/E (each presence-gated) ------------------- + # L6 native-metric labels: span attributes are not the only valid dashboard + # labels — the MetricsRegistry emits OTel metrics whose label keys are an + # additional source of truth. Derive them dynamically (same principle as L1) + # so dashboards may reference them without tripping Rule D. + metric_labels = metric_label_names(root) + run_rule_b_collector(root, l1_keys, report) + run_rule_c_tempo(root, l1_keys, report) + run_rule_d_dashboards(root, l1_keys, metric_labels, report) + run_rule_e_runbook(root, l1_keys, report) + + report.render_and_exit() + + +def resource_create_block(text: str) -> str: + """Return the text inside the first `Resource::Create({ ... })` argument + list, brace-matched so nested `{key, value}` initializers are contained. + Empty string if the call is absent.""" + m = re.search(r"Resource::Create\(\s*\{", text) + if not m: + return "" + i = m.end() # one char past the opening `{` + depth, start = 1, i + while i < len(text) and depth > 0: + c = text[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + i += 1 + return text[start : i - 1] + + +def derive_dotted_resource_keys( + root: Path, symbols: Dict[str, str], report: Report +) -> Set[str]: + """Legitimate dotted keys = ONLY the keys the code actually sets as RESOURCE + attributes, i.e. the entries inside Telemetry.cpp's `Resource::Create({...})` + call: the standard semconv keys (`service.*`) plus any `attr::` + constants passed there (resolved to their wire key via the global symbol + table, e.g. `attr::networkId` -> `xrpl.network.id`). + + A dotted key DECLARED in a `*SpanNames.h` header but NOT passed to + Resource::Create() is a span attribute wearing the resource form — a Rule-A + violation, never allowlisted. Deriving the allowlist from the actual + resource call (not from "any dotted key in the base header") is what lets + Rule A catch a stray dotted span attr such as `xrpl.ledger.hash`.""" + allow: Set[str] = set() + tele = root / "src" / "libxrpl" / "telemetry" / "Telemetry.cpp" + if not tele.is_file(): + report.skip("resource-derive", "Telemetry.cpp not present") + return allow + block = resource_create_block(read_source(tele)) + # semconv::::k -> the dotted OTel-standard key. The + # CamelKey already embeds the group, e.g. service::kServiceInstanceId + # -> service.instance.id. Split the CamelCase name into dotted lowercase + # segments; if it does not lead with the group, prepend the group. + for m in re.finditer(r"semconv::(\w+)::k(\w+)", block): + group, camel = m.group(1), m.group(2) + segments = camel_to_dotsegments(camel) + if segments and segments[0] == group: + allow.add(".".join(segments)) + else: + allow.add(group + "." + ".".join(segments)) + # attr:: constants set as resource attrs (e.g. networkId/networkType); + # resolve each to its wire key and allowlist only the dotted ones. + for m in re.finditer(r"attr::(\w+)", block): + val = symbols.get(m.group(1)) + if val is not None and "." in val: + allow.add(val) + report.ok(f"resource dotted-key allowlist derived: {sorted(allow)}") + return allow + + +def camel_to_dotsegments(s: str) -> List[str]: + """Split a CamelCase identifier into lowercase dot-segment parts, e.g. + `ServiceInstanceId` -> ['service', 'instance', 'id'].""" + return [w.lower() for w in re.findall(r"[A-Z][a-z0-9]*", s)] + + +def run_rule_a( + keys_by_header: Dict[Path, Set[str]], dotted_allow: Set[str], report: Report +) -> None: + """Any dotted attribute key that is not an allowed resource key is a + violation, reported against the header that declares it.""" + found = False + for h in sorted(keys_by_header): + for key in sorted(keys_by_header[h]): + if "." in key and key not in dotted_allow: + found = True + report.violation("A", h.name, key, "underscore form, not dotted") + if not found: + report.ok("A: no stray dotted span-attribute keys") + + +# A lower_snake_case identifier segment: starts lowercase, then lowercase / +# digits / underscores. No uppercase, no spaces, no camelCase. +SNAKE_SEGMENT = re.compile(r"^[a-z][a-z0-9_]*$") + + +def run_rule_g(keys_by_header: Dict[Path, Set[str]], report: Report) -> None: + """Every attribute key must be lower_snake_case. Bare/underscore keys must + match ^[a-z][a-z0-9_]*$; dotted resource keys must be lowercase + dot-separated segments (each segment lower_snake_case). Flags camelCase, + UPPERCASE, spaces, and other stray characters.""" + found = False + for h in sorted(keys_by_header): + for key in sorted(keys_by_header[h]): + segments = key.split(".") + if all(SNAKE_SEGMENT.match(seg) for seg in segments): + continue + found = True + report.violation("G", h.name, key, "must be lower_snake_case") + if not found: + report.ok("G: all attribute keys are lower_snake_case") + + +# Which argument positions of each call must be a constant (0-based). The +# attribute VALUE position is intentionally absent: values are runtime data +# (command names, hashes, counts), not naming-convention surface. +# setAttribute(key, value) -> check arg 0 (key); value (arg 1) exempt +# addEvent(name[, attrs]) -> check arg 0 (event name) +# span(category, prefix, name) -> check args 1,2 (prefix + span-name leaf) +# rootSpan(category, prefix, name)-> check args 1,2 (same signature as span) +# childSpan(name[, parentCtx]) -> check arg 0 (span-name leaf) +CONSTANT_ARG_POSITIONS: Dict[str, Set[int]] = { + "setAttribute": {0}, + "addEvent": {0}, + "span": {1, 2}, + "rootSpan": {1, 2}, # same signature as span(cat, prefix, name) + "childSpan": {0}, +} + + +def is_test_path(path: Path) -> bool: + """True if the path is test code. Tests legitimately pass arbitrary literal + keys/names to exercise the API mechanics, so Rule F does not apply to them. + Matches a `test`/`tests` directory anywhere in the path (e.g. src/test/, + src/tests/, .../detail/tests/).""" + return any(part in ("test", "tests") for part in path.parts) + + +# A constant reference passed at a call-site, e.g. `rpc_span::attr::command` +# or a bare `myKey`. We capture the leaf identifier (after the last `::`). +IDENTIFIER_ARG = re.compile(r"^[\s&*]*([A-Za-z_][\w:]*)\s*$") + + +def spanname_symbol_names(headers: List[Path]) -> Set[str]: + """Every `inline constexpr auto NAME = ...;` symbol defined across the + *SpanNames.h headers, by bare name. Used by Rule H to tell whether a + constant referenced at a call-site actually lives in a SpanNames header.""" + names: Set[str] = set() + for h in headers: + for m in CONST_DEF.finditer(strip_comments(read_source(h))): + names.add(m.group(1)) + return names + + +def run_rule_f(root: Path, report: Report, header_symbols: Set[str]) -> None: + """Walk every telemetry call-site (non-test, non-*SpanNames.h) and check the + constant-only argument positions of + setAttribute/addEvent/span/rootSpan/childSpan: + + Rule F (FAIL): a string literal in a key / span-name position. Attribute + VALUES are exempt (runtime data). + Rule H (WARN): a constant reference whose name is not defined in any + *SpanNames.h. The constant should live in the corresponding + *SpanNames.h (single source of truth); defining it in-place bypasses + the naming rules. Warn rather than fail — the argument may instead be a + legitimately dynamic local (e.g. a computed span-name leaf).""" + found_f = False + sources = [ + p + for base in ("src", "include") + for ext in ("*.h", "*.cpp") + for p in (root / base).rglob(ext) + if p.is_file() + ] + for path in sorted(sources): + if path.name.endswith("SpanNames.h") or is_test_path(path): + continue + text = read_source(path) + rel = path.relative_to(root) + for call, arglist, lineno in iter_calls(text): + positions = CONSTANT_ARG_POSITIONS.get(call, set()) + args = split_top_level_args(arglist) + for idx in positions: + if idx >= len(args): + continue + arg = args[idx] + lit = STRING_LITERAL.search(arg) + if lit: + found_f = True + report.violation( + "F", + f"{rel}:{lineno}", + f'{call} arg{idx} "{lit.group(1)}"', + "use a *SpanNames.h constant", + ) + continue + # Not a literal: Rule H warns when a NAMESPACE-QUALIFIED constant + # reference (e.g. `consensus::span::accept`) is not defined in + # any *SpanNames.h — i.e. the constant was defined in-place + # instead of in the proper header. We only consider qualified + # refs (containing `::`): a bare lowercase identifier is almost + # always a legitimately dynamic local (a computed span-name leaf + # or attribute value), not a misplaced constant, so warning on it + # would be noise. Standard-library types (std::...) are skipped. + ident = IDENTIFIER_ARG.match(arg) + if not (ident and header_symbols): + continue + ref = ident.group(1) + if "::" not in ref or ref.startswith("std::"): + continue + leaf = ref.split("::")[-1] + if leaf not in header_symbols: + report.warning( + "H", + f"{rel}:{lineno}", + f"{call} arg{idx} {ref}", + "not defined in any *SpanNames.h", + ) + if not found_f: + report.ok("F: no string-literal keys/names at telemetry call-sites") + + +def iter_calls(text: str): + """Yield (call_name, raw_arglist, lineno) for each setAttribute/addEvent/ + span/rootSpan/childSpan invocation, spanning multiple physical lines if + needed.""" + for m in CALLSITE.finditer(text): + name = m.group(1) + # Walk from the opening paren, balancing nesting to find the close. + # Parens inside a "string literal" are ignored so a value such as + # `setAttribute(k, ")")` does not close the call early. + i = m.end() # one char past the '(' + depth = 1 + in_str = False + escaped = False + while i < len(text) and depth > 0: + c = text[i] + if in_str: + if escaped: + escaped = False + elif c == "\\": + escaped = True + elif c == '"': + in_str = False + elif c == '"': + in_str = True + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + i += 1 + arglist = text[m.end() : i - 1] + lineno = text.count("\n", 0, m.start()) + 1 + yield name, arglist, lineno + + +def run_rule_b_collector(root: Path, l1_keys: Set[str], report: Report) -> None: + path = root / "docker" / "telemetry" / "otel-collector-config.yaml" + if not path.is_file(): + report.skip("B", "collector config not present") + return + text = read_source(path) + if "spanmetrics" not in text: + report.skip("B", "no spanmetrics block in collector config") + return + dims = extract_spanmetrics_dimensions(text) + if not l1_keys: + report.skip("B", "no L1 key set to validate against") + return + miss = [d for d in dims if d not in l1_keys] + for d in miss: + report.violation("B", str(path.relative_to(root)), d, "must exist in L1") + if not miss: + report.ok(f"B: {len(dims)} collector dimension(s) all in L1") + + +def extract_spanmetrics_dimensions(text: str) -> List[str]: + dims: List[str] = [] + in_dims = False + for line in text.splitlines(): + if re.search(r"\bdimensions\s*:", line): + in_dims = True + continue + if in_dims: + m = re.search(r"-\s*name\s*:\s*([A-Za-z0-9_.]+)", line) + if m: + dims.append(m.group(1)) + elif line.strip() and not line.lstrip().startswith("-") and ":" in line: + in_dims = False + return dims + + +def run_rule_c_tempo(root: Path, l1_keys: Set[str], report: Report) -> None: + # The trace-search filter tags live in the Grafana Tempo DATASOURCE + # provisioning file (search.filters[].{tag,scope}); the Tempo server + # tempo.yaml has no such tags. Prefer the datasource file; fall back to the + # server file so the rule still does something if the layout changes. + candidates = [ + root / "docker/telemetry/grafana/provisioning/datasources/tempo.yaml", + root / "docker/telemetry/tempo.yaml", + ] + path = next((p for p in candidates if p.is_file()), None) + if path is None: + report.skip("C", "tempo datasource provisioning not present") + return + if not l1_keys: + report.skip("C", "no L1 key set to validate against") + return + # Pair each filter's `tag:` with its `scope:` (a few lines below it) and + # validate only span-scope tags — resource/intrinsic tags (service.*, name, + # status, duration) are not span attributes. Strip a TraceQL span. prefix. + lines = read_source(path).splitlines() + span_tags: List[str] = [] + for i, line in enumerate(lines): + m = re.search(r"^\s*tag:\s*(\S+)", line) + if not m: + continue + scope = next( + ( + sm.group(1) + for j in range(i, min(i + 4, len(lines))) + for sm in [re.search(r"scope:\s*(\S+)", lines[j])] + if sm + ), + "", + ) + if scope == "span": + span_tags.append(TRACEQL_SCOPE.sub("", m.group(1))) + if not span_tags: + report.skip("C", "no span-scope filter tags in tempo datasource") + return + miss = [t for t in span_tags if t not in l1_keys] + for t in sorted(set(miss)): + report.violation("C", str(path.relative_to(root)), t, "must exist in L1") + if not miss: + report.ok(f"C: {len(span_tags)} tempo span-filter tag(s) all in L1") + + +def metric_label_names(root: Path) -> Set[str]: + """L6: OTel native-metric label keys emitted by the telemetry code, e.g. + `counter->Add(1, {{"job_type", value}})` in MetricsRegistry.cpp. These are + a valid source of dashboard labels distinct from span attributes (L1).""" + labels: Set[str] = set() + for base in ("src", "include"): + for p in (root / base).rglob("*.cpp"): + if not p.is_file(): + continue + text = read_source(p) + if "MetricsRegistry" not in p.name and "metric" not in text.lower(): + continue + labels |= set(METRIC_LABEL.findall(text)) + return labels + + +# Identity labels stamped by EXTERNAL infrastructure the OTel pipeline in this +# repo does not own: the perf-iac harness attaches these to every metric it +# scrapes so dashboards can filter by which build/role produced a series. They +# have no L1 (*SpanNames.h), L2 (collector config), or L6 (MetricsRegistry.cpp) +# source to derive from, so — unlike every other dashboard label — they cannot +# be validated dynamically. This is a deliberate, narrow exception to the "no +# hardcoded allowlist" design principle, kept separate from the generic +# Prometheus/Grafana `builtins` set below so it stays visible and auditable. +# Add a label here ONLY if it is genuinely injected by infra outside this +# repo's OTel code (never as a workaround for a dashboard querying a label +# that nothing actually emits — that is a real Rule D violation). +EXTERNAL_INFRA_LABELS = { + "xrpl_work_item", # perf-iac: ticket/work-item id for the perf comparison run + "xrpl_branch", # perf-iac: git ref of the xrpld build under test + "xrpl_node_role", # perf-iac: validator/peer role in the perf cluster +} + + +def run_rule_d_dashboards( + root: Path, l1_keys: Set[str], metric_labels: Set[str], report: Report +) -> None: + dash_dir = root / "docker" / "telemetry" / "grafana" / "dashboards" + files = sorted(dash_dir.glob("*.json")) if dash_dir.is_dir() else [] + if not files: + report.skip("D", "no dashboard JSON present") + return + if not l1_keys: + report.skip("D", "no L1 key set to validate against") + return + builtins = { + "__name__", # Prometheus reserved label for the metric name itself + "le", + "exported_instance", + "span_name", + "status_code", + "service_name", + "service_version", + "service_instance_id", + "job", + "job_type", # standard Prometheus label for job-queue metrics + "instance", + } + # A dashboard label is valid if it is a span attribute (L1), a native-metric + # label (L6), a Prometheus/Grafana builtin, or an external-infra identity + # label (EXTERNAL_INFRA_LABELS). + valid = l1_keys | metric_labels | builtins | EXTERNAL_INFRA_LABELS + found = False + for f in files: + try: + text = read_source(f) + except OSError: + continue + # PromQL `sum by (a, b)` and `{label="..."}` references. + labels: Set[str] = set() + for m in re.finditer(r"by\s*\(([^)]*)\)", text): + labels |= {x.strip() for x in m.group(1).split(",") if x.strip()} + for m in re.finditer(r"\b([a-z_][a-z0-9_.]*)\s*[=!]~?\s*\"", text): + labels.add(m.group(1)) + for lbl in sorted(labels): + # Strip a TraceQL scope prefix (span./resource./...) — the bare + # attribute is what must resolve against L1. + bare = TRACEQL_SCOPE.sub("", lbl) + if bare in valid: + continue + found = True + report.violation( + "D", + str(f.relative_to(root)), + lbl, + "must exist in L1, a metric label, or be a builtin", + ) + if not found: + report.ok(f"D: dashboard PromQL labels all resolve ({len(files)} file(s))") + + +def run_rule_e_runbook(root: Path, l1_keys: Set[str], report: Report) -> None: + path = root / "docs" / "telemetry-runbook.md" + if not path.is_file(): + report.skip("E", "runbook not present") + return + if not l1_keys: + report.skip("E", "no L1 key set to validate against") + return + text = read_source(path) + found = False + # Only the dotted `xrpl..` attribute form is a violation. The + # `xrpl.`-with-trailing-dot anchor is the discriminator: it matches the old + # dotted attribute convention being migrated away from, while everything + # else legitimately dotted in the runbook does NOT match it — + # * span names (`consensus.round`, `tx.process`) no `xrpl.` prefix + # * filenames (`xrpld.cfg`, `RCLConsensus.cpp`) `xrpld.`/`.cpp`, not `xrpl.` + # * OTel-standard (`service.name`, `http.method`) no `xrpl.` prefix + # * metric labels (`xrpl_rpc_command`) underscore, no dot + # Legitimate dotted resource attrs (`xrpl.network.id`/`.type`) are in L1 and + # are skipped. A dotted `xrpl.` token absent from L1 is a genuine doc/code + # mismatch (e.g. `xrpl.tx.hash` where the code emits `tx_hash`). + # EXTERNAL_INFRA_LABELS (Rule D) holds the underscore/metric-label form of + # the perf-iac identity attrs; the resource-attribute layer stamps the same + # identities dotted (xrpl.work.item/.branch/.node.role -- see the alloy + # pipeline that owns them), so also skip a token whose dotted-to-underscore + # form is in that set. + external_infra_dotted = {lbl.replace("_", ".") for lbl in EXTERNAL_INFRA_LABELS} + for m in re.finditer(r"`(xrpl\.[a-z][a-z0-9_.]*)`", text): + token = m.group(1) + if token in l1_keys: # legitimate dotted resource attr (xrpl.network.*) + continue + if token in external_infra_dotted: # perf-iac resource-attribute layer + continue + found = True + report.violation( + "E", str(path.relative_to(root)), token, "underscore, not dotted" + ) + if not found: + report.ok("E: runbook attribute references consistent with L1") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/otel-naming/test_check_otel_naming.py b/.github/scripts/otel-naming/test_check_otel_naming.py new file mode 100644 index 00000000000..890648cefea --- /dev/null +++ b/.github/scripts/otel-naming/test_check_otel_naming.py @@ -0,0 +1,901 @@ +#!/usr/bin/env python3 + +"""Unit tests for check_otel_naming.py. + +Stdlib-only (unittest), matching the dependency-free policy of the check itself. +Run from anywhere: + + python .github/scripts/otel-naming/test_check_otel_naming.py + +Each rule is exercised in isolation against a synthetic tree / synthetic L1 key +set, covering positive (must flag), negative (must not flag), and boundary +cases. Rule E (runbook dotted-attribute detection) has the densest coverage +because its discriminator — the `xrpl..` prefix vs span names, +filenames, OTel-standard keys, and metric labels — is the subtlest. +""" + +import contextlib +import importlib.util +import io +import shutil +import tempfile +import unittest +from pathlib import Path + +# Load the check module by path (it is not an importable package). +_spec = importlib.util.spec_from_file_location( + "check_otel_naming", str(Path(__file__).with_name("check_otel_naming.py")) +) +chk = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(chk) + + +# A controlled L1 set used across tests: the two legitimate dotted resource +# attrs plus a handful of underscore span-attribute keys. +L1 = { + "xrpl.network.id", + "xrpl.network.type", + "tx_hash", + "peer_id", + "consensus_mode", + "command", + "rpc_status", + "ledger_seq", +} + + +def _run_rule_e(runbook_text: str): + """Run Rule E against a synthetic runbook; return the flagged tokens.""" + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text(runbook_text) + report = chk.Report() + chk.run_rule_e_runbook(d, set(L1), report) + return sorted(v[2] for v in report.violations) + finally: + shutil.rmtree(d) + + +class RuleERunbook(unittest.TestCase): + """Rule E: only dotted `xrpl..` attribute keys are flagged.""" + + # ----- positive: genuine dotted attribute-key violations ----- + def test_single_dotted_attr(self): + self.assertEqual(_run_rule_e("`xrpl.tx.hash`"), ["xrpl.tx.hash"]) + + def test_multiple_dotted_attrs(self): + self.assertEqual( + _run_rule_e("`xrpl.tx.hash` and `xrpl.consensus.mode`"), + ["xrpl.consensus.mode", "xrpl.tx.hash"], + ) + + def test_deep_dotted_three_segments(self): + self.assertEqual( + _run_rule_e("`xrpl.consensus.ledger.seq`"), ["xrpl.consensus.ledger.seq"] + ) + + def test_dotted_attr_with_underscore_field(self): + self.assertEqual( + _run_rule_e("`xrpl.consensus.round_id`"), ["xrpl.consensus.round_id"] + ) + + def test_repeated_token_reported_each_occurrence(self): + self.assertEqual( + _run_rule_e("`xrpl.tx.hash` ... `xrpl.tx.hash`"), + ["xrpl.tx.hash", "xrpl.tx.hash"], + ) + + def test_resource_attr_not_in_l1_is_flagged(self): + self.assertEqual( + _run_rule_e("`xrpl.network.unknown`"), ["xrpl.network.unknown"] + ) + + # ----- negative: legitimately-dotted tokens that must NOT be flagged ----- + def test_span_name_single(self): + self.assertEqual(_run_rule_e("`consensus.round`"), []) + + def test_span_name_multi_segment(self): + self.assertEqual( + _run_rule_e("`consensus.phase.open` `rpc.command.server_info`"), [] + ) + + def test_filename_cfg(self): + self.assertEqual(_run_rule_e("`xrpld.cfg`"), []) + + def test_filename_cpp(self): + self.assertEqual(_run_rule_e("`RCLConsensus.cpp`"), []) + + def test_otel_standard_service_name(self): + self.assertEqual(_run_rule_e("`service.name`"), []) + + def test_otel_standard_http_method(self): + self.assertEqual(_run_rule_e("`http.method`"), []) + + def test_metric_label_underscore(self): + self.assertEqual(_run_rule_e("`xrpl_rpc_command`"), []) + + def test_bare_underscore_attrs(self): + self.assertEqual(_run_rule_e("`tx_hash` `consensus_mode`"), []) + + def test_legit_dotted_resource_attrs_in_l1(self): + self.assertEqual(_run_rule_e("`xrpl.network.id` `xrpl.network.type`"), []) + + def test_external_infra_dotted_resource_attrs_not_flagged(self): + # perf-iac stamps these as dotted resource attrs (alloy pipeline); + # EXTERNAL_INFRA_LABELS (Rule D) holds their underscore metric-label + # form -- Rule E must also exempt the dotted resource-attr form. + self.assertEqual( + _run_rule_e("`xrpl.work.item` `xrpl.branch` `xrpl.node.role`"), [] + ) + + def test_prose_word(self): + self.assertEqual(_run_rule_e("the `command` attribute"), []) + + def test_plain_prose_no_backticks(self): + self.assertEqual(_run_rule_e("xrpl.tx.hash without backticks is prose"), []) + + # ----- boundary ----- + def test_empty_runbook(self): + self.assertEqual(_run_rule_e(""), []) + + def test_lookalike_prefix_xrpld(self): + # `xrpld.` is NOT `xrpl.` — must not match. + self.assertEqual(_run_rule_e("`xrpld.foo`"), []) + + def test_lookalike_prefix_underscore(self): + # `xrpl_rpc.command` starts with `xrpl_`, not `xrpl.`. + self.assertEqual(_run_rule_e("`xrpl_rpc.command`"), []) + + def test_uppercase_segment_not_matched(self): + # The pattern requires a lowercase char after `xrpl.`; uppercase keys are + # caught by Rule G at the L1 layer, not by the runbook text scan. + self.assertEqual(_run_rule_e("`xrpl.TX.hash`"), []) + + def test_token_touching_table_pipes(self): + self.assertEqual(_run_rule_e("| `xrpl.tx.hash` | desc |"), ["xrpl.tx.hash"]) + + def test_mixed_line_only_xrpl_dotted_flagged(self): + self.assertEqual( + _run_rule_e("`consensus.round` uses `xrpl.tx.hash` and `service.name`"), + ["xrpl.tx.hash"], + ) + + def test_skips_when_runbook_absent(self): + d = Path(tempfile.mkdtemp()) + try: + report = chk.Report() + chk.run_rule_e_runbook(d, set(L1), report) + self.assertEqual(report.violations, []) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + def test_skips_when_l1_empty(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text("`xrpl.tx.hash`") + report = chk.Report() + chk.run_rule_e_runbook(d, set(), report) + self.assertEqual(report.violations, []) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class DslParser(unittest.TestCase): + """The makeStr/join/seg:: constexpr DSL resolver — the foundation of the + L1 key set. Covers flat, nested, cross-file, alias, and multi-line forms.""" + + def test_flat_join(self): + syms = chk.resolve_constants( + 'inline constexpr auto a = makeStr("xrpl");\n' + 'inline constexpr auto b = makeStr("network");\n' + "inline constexpr auto c = join(a, b);\n" + ) + self.assertEqual(syms["c"], "xrpl.network") + + def test_nested_join_three_segments(self): + syms = chk.resolve_constants( + 'inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto network = makeStr("network");\n' + "inline constexpr auto networkId = " + 'join(join(xrpl, network), makeStr("id"));\n' + ) + self.assertEqual(syms["networkId"], "xrpl.network.id") + + def test_qualified_seg_reference(self): + # `seg::rpc` resolves by its bare leaf `rpc`. + syms = chk.resolve_constants('inline constexpr auto rpc = makeStr("rpc");\n') + syms2 = chk.resolve_constants( + 'inline constexpr auto command = join(seg::rpc, makeStr("command"));\n', + syms, + ) + self.assertEqual(syms2["command"], "rpc.command") + + def test_alias_reference(self): + syms = chk.resolve_constants('inline constexpr auto rpc = makeStr("rpc");\n') + chk.resolve_constants("inline constexpr auto alias = seg::rpc;\n", syms) + self.assertEqual(syms["alias"], "rpc") + + def test_unresolvable_expr_omitted(self): + syms = chk.resolve_constants("inline constexpr auto x = join(unknown, y);\n") + self.assertNotIn("x", syms) + + def test_split_top_level_args_respects_nesting(self): + self.assertEqual( + chk.split_top_level_args("join(seg::a, b), c"), + ["join(seg::a, b)", " c"], + ) + + def test_split_top_level_args_ignores_comma_in_string(self): + self.assertEqual( + chk.split_top_level_args('key, ","'), + ["key", ' ","'], + ) + + def test_strip_comments_removes_line_and_block(self): + self.assertEqual( + chk.strip_comments("a // line\nb /* blk */ c").split(), + ["a", "b", "c"], + ) + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +def _header(ns_attr_body: str, prefix_seg: str = "") -> str: + """A minimal *SpanNames.h body: optional seg defs + a namespace attr block.""" + return ( + "#pragma once\n" + + prefix_seg + + "namespace xrpl::telemetry::demo::span {\n" + + "namespace attr {\n" + + ns_attr_body + + "} // namespace attr\n" + + "}\n" + ) + + +class AttrKeyExtraction(unittest.TestCase): + """attr_keys_from_header: comment-stripping + multi-line + using re-export.""" + + def _l1(self, header_text): + d = Path(tempfile.mkdtemp()) + try: + h = d / "src" / "DemoSpanNames.h" + _write(h, header_text) + syms = chk.build_global_symbols([h]) + return chk.attr_keys_from_header(h, syms) + finally: + shutil.rmtree(d) + + def test_single_line_makestr(self): + keys = self._l1(_header('inline constexpr auto k = makeStr("tx_hash");\n')) + self.assertIn("tx_hash", keys) + + def test_multiline_constexpr_captured(self): + keys = self._l1( + _header("inline constexpr auto k =\n" ' makeStr("round_time_ms");\n') + ) + self.assertIn("round_time_ms", keys) + + def test_commented_makestr_not_leaked(self): + keys = self._l1( + _header( + 'inline constexpr auto k = makeStr("good");\n' + '// inline constexpr auto bad = makeStr("old.dotted");\n' + ) + ) + self.assertIn("good", keys) + self.assertNotIn("old.dotted", keys) + + def test_block_commented_makestr_not_leaked(self): + keys = self._l1( + _header( + 'inline constexpr auto k = makeStr("good");\n' + '/* makeStr("blockbad") */\n' + ) + ) + self.assertNotIn("blockbad", keys) + + +class CamelToDotSegments(unittest.TestCase): + """semconv CamelCase -> dotted OTel-standard key derivation.""" + + def test_service_instance_id(self): + self.assertEqual( + chk.camel_to_dotsegments("ServiceInstanceId"), + ["service", "instance", "id"], + ) + + def test_service_name(self): + self.assertEqual(chk.camel_to_dotsegments("ServiceName"), ["service", "name"]) + + def test_derive_keys_from_telemetry_cpp(self): + d = Path(tempfile.mkdtemp()) + try: + tele = d / "src" / "libxrpl" / "telemetry" / "Telemetry.cpp" + _write( + tele, + "resource::Resource::Create({\n" + " {semconv::service::kServiceName, x},\n" + " {semconv::service::kServiceInstanceId, y},\n" + "});\n", + ) + report = chk.Report() + allow = chk.derive_dotted_resource_keys(d, {}, report) + self.assertIn("service.name", allow) + self.assertIn("service.instance.id", allow) + finally: + shutil.rmtree(d) + + +class SymbolCollision(unittest.TestCase): + """attr_keys_from_header must resolve a constant against ITS OWN header, so + two headers defining a same-named constant each report their real wire key. + Regression for the flat-symbol-table collision that let a later header + clobber an earlier one and erased a dotted key from L1 (a Rule-A blind + spot).""" + + def _build(self, files): + d = Path(tempfile.mkdtemp()) + paths = {} + for rel, text in files.items(): + p = d / rel + _write(p, text) + paths[rel] = p + return d, paths + + def test_same_named_const_not_clobbered_across_headers(self): + base = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + 'namespace seg { inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto ledger = makeStr("ledger"); }\n' + "namespace attr {\n" + "inline constexpr auto ledgerHash = " + 'join(join(seg::xrpl, seg::ledger), makeStr("hash"));\n' + "}\n}\n" + ) + cons = ( + "#pragma once\n" + "namespace xrpl::telemetry::consensus::span {\n" + "namespace attr { inline constexpr auto ledgerHash = " + 'makeStr("ledger_hash"); }\n}\n' + ) + d, paths = self._build( + { + "include/xrpl/telemetry/SpanNames.h": base, + "src/xrpld/consensus/ConsensusSpanNames.h": cons, + } + ) + try: + headers = chk.find_spanname_headers(d) + syms = chk.build_global_symbols(headers) + by_name = {p.name: chk.attr_keys_from_header(p, syms) for p in headers} + # The base header keeps its dotted key; consensus keeps the bare one. + self.assertIn("xrpl.ledger.hash", by_name["SpanNames.h"]) + self.assertEqual(by_name["ConsensusSpanNames.h"], {"ledger_hash"}) + finally: + shutil.rmtree(d) + + def test_using_reexport_still_resolves_globally(self): + # A `using`-re-export imports a constant defined elsewhere; it must + # resolve against the global table, not the local header. + base = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + "namespace attr { inline constexpr auto txHash = " + 'makeStr("tx_hash"); }\n}\n' + ) + dom = ( + "#pragma once\n" + "namespace xrpl::telemetry::tx::span {\n" + "namespace attr { using ::xrpl::telemetry::attr::txHash; }\n}\n" + ) + d, paths = self._build( + { + "include/xrpl/telemetry/SpanNames.h": base, + "src/xrpld/app/misc/TxSpanNames.h": dom, + } + ) + try: + headers = chk.find_spanname_headers(d) + syms = chk.build_global_symbols(headers) + keys = chk.attr_keys_from_header( + paths["src/xrpld/app/misc/TxSpanNames.h"], syms + ) + self.assertEqual(keys, {"tx_hash"}) + finally: + shutil.rmtree(d) + + +class ResourceAllowlistScope(unittest.TestCase): + """derive_dotted_resource_keys must allowlist ONLY the dotted keys actually + passed to Resource::Create() — not every dotted key in the base header. A + dotted attr declared in a header but not set as a resource attr is a Rule-A + violation.""" + + def _derive(self, tele_text, span_text): + d = Path(tempfile.mkdtemp()) + try: + _write(d / "src" / "libxrpl" / "telemetry" / "Telemetry.cpp", tele_text) + _write(d / "include" / "xrpl" / "telemetry" / "SpanNames.h", span_text) + headers = chk.find_spanname_headers(d) + syms = chk.build_global_symbols(headers) + allow = chk.derive_dotted_resource_keys(d, syms, chk.Report()) + return allow, syms, headers, d + except Exception: + shutil.rmtree(d) + raise + + def test_dotted_span_attr_not_allowlisted_and_flagged(self): + span = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + 'namespace seg { inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto ledger = makeStr("ledger");\n' + 'inline constexpr auto network = makeStr("network"); }\n' + "namespace attr {\n" + "inline constexpr auto networkId = " + 'join(join(seg::xrpl, seg::network), makeStr("id"));\n' + "inline constexpr auto ledgerHash = " + 'join(join(seg::xrpl, seg::ledger), makeStr("hash"));\n' + "}\n}\n" + ) + tele = ( + "auto r = resource::Resource::Create({\n" + " {semconv::service::kServiceName, x},\n" + " {std::string(attr::networkId), n},\n" + "});\n" + ) + allow, syms, headers, d = self._derive(tele, span) + try: + # networkId IS a resource attr; ledgerHash is NOT, despite living in + # the base header. + self.assertIn("xrpl.network.id", allow) + self.assertNotIn("xrpl.ledger.hash", allow) + kbh = {h: chk.attr_keys_from_header(h, syms) for h in headers} + report = chk.Report() + chk.run_rule_a(kbh, allow, report) + self.assertEqual([v[2] for v in report.violations], ["xrpl.ledger.hash"]) + finally: + shutil.rmtree(d) + + def test_resource_block_brace_matched(self): + # A nested {key,value} initializer must not truncate the block scan. + tele = ( + "auto r = resource::Resource::Create({\n" + " {semconv::service::kServiceName, x},\n" + " {std::string(attr::networkType), t},\n" + "});\n" + ) + span = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + 'namespace seg { inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto network = makeStr("network"); }\n' + "namespace attr { inline constexpr auto networkType = " + 'join(join(seg::xrpl, seg::network), makeStr("type")); }\n}\n' + ) + allow, _syms, _headers, d = self._derive(tele, span) + try: + self.assertIn("xrpl.network.type", allow) + self.assertIn("service.name", allow) + finally: + shutil.rmtree(d) + + +def _run_rule_a(keys_by_header, allow): + report = chk.Report() + chk.run_rule_a(keys_by_header, allow, report) + return sorted(v[2] for v in report.violations) + + +class RuleADotted(unittest.TestCase): + def test_dotted_attr_not_in_allow_flagged(self): + kbh = {Path("src/RpcSpanNames.h"): {"xrpl.tx.hash", "command"}} + self.assertEqual(_run_rule_a(kbh, {"xrpl.network.id"}), ["xrpl.tx.hash"]) + + def test_resource_attr_in_allow_passes(self): + kbh = {Path("src/SpanNames.h"): {"xrpl.network.id"}} + self.assertEqual(_run_rule_a(kbh, {"xrpl.network.id"}), []) + + def test_bare_key_never_flagged(self): + kbh = {Path("src/TxSpanNames.h"): {"tx_hash", "command"}} + self.assertEqual(_run_rule_a(kbh, set()), []) + + +def _run_rule_g(keys_by_header): + report = chk.Report() + chk.run_rule_g(keys_by_header, report) + return sorted(v[2] for v in report.violations) + + +class RuleGSnakeCase(unittest.TestCase): + def test_camelcase_flagged(self): + self.assertEqual(_run_rule_g({Path("h"): {"txHash"}}), ["txHash"]) + + def test_uppercase_flagged(self): + self.assertEqual(_run_rule_g({Path("h"): {"TX_HASH"}}), ["TX_HASH"]) + + def test_space_flagged(self): + self.assertEqual(_run_rule_g({Path("h"): {"bad key"}}), ["bad key"]) + + def test_snake_case_passes(self): + self.assertEqual(_run_rule_g({Path("h"): {"tx_hash", "rpc_status"}}), []) + + def test_dotted_resource_segments_pass(self): + self.assertEqual(_run_rule_g({Path("h"): {"xrpl.network.id"}}), []) + + def test_dotted_with_bad_segment_flagged(self): + self.assertEqual( + _run_rule_g({Path("h"): {"xrpl.Network.id"}}), ["xrpl.Network.id"] + ) + + +class RuleFAndH(unittest.TestCase): + """run_rule_f: literal keys/span-names flagged; values & tests exempt. + Rule H: qualified constant not in any header warns (non-fatal).""" + + def _run(self, rel_path, source, header_symbols=frozenset()): + d = Path(tempfile.mkdtemp()) + try: + _write(d / rel_path, source) + report = chk.Report() + chk.run_rule_f(d, report, set(header_symbols)) + return ( + sorted(v[2] for v in report.violations), + sorted(w[2] for w in report.warnings), + ) + finally: + shutil.rmtree(d) + + def test_literal_key_flagged(self): + v, _ = self._run("src/Foo.cpp", 'g.setAttribute("lit_key", v);\n') + self.assertEqual(v, ['setAttribute arg0 "lit_key"']) + + def test_literal_value_exempt(self): + v, _ = self._run("src/Foo.cpp", 'g.setAttribute(attr::command, "submit");\n') + self.assertEqual(v, []) + + def test_span_name_args_flagged(self): + v, _ = self._run("src/Foo.cpp", 'SpanGuard::span(cat, "rpc", "command");\n') + self.assertEqual(v, ['span arg1 "rpc"', 'span arg2 "command"']) + + def test_rootspan_literal_flagged_by_rule_f(self): + # rootSpan(cat, prefix, name) shares span()'s signature, so a string + # literal in the prefix/name position must FAIL rule F exactly as it + # does for span() — otherwise a call switched to rootSpan silently + # escapes span-name validation. + v, _ = self._run( + "src/Foo.cpp", + 'SpanGuard::rootSpan(cat, "peer", "validation.receive");\n', + ) + self.assertEqual( + v, ['rootSpan arg1 "peer"', 'rootSpan arg2 "validation.receive"'] + ) + + def test_rootspan_constant_args_accepted(self): + # Constant references in the prefix/name position are accepted (no + # rule F), mirroring span()'s constant-arg handling. + v, _ = self._run( + "src/Foo.cpp", + "SpanGuard::rootSpan(TraceCategory::Peer, seg::peer, " + "peer_span::op::validationReceive);\n", + ) + self.assertEqual(v, []) + + def test_test_path_exempt(self): + v, _ = self._run("src/test/Foo.cpp", 'g.setAttribute("lit_key", v);\n') + self.assertEqual(v, []) + + def test_spannames_header_exempt(self): + v, _ = self._run("src/DemoSpanNames.h", 'g.setAttribute("lit_key", v);\n') + self.assertEqual(v, []) + + def test_bare_span_call_not_matched(self): + # No SpanGuard/./-> receiver -> not a telemetry call-site. + v, _ = self._run("src/Foo.cpp", 'auto s = span("not", "telemetry");\n') + self.assertEqual(v, []) + + def test_multiline_call_reports_first_line(self): + v, _ = self._run("src/Foo.cpp", 'g.setAttribute(\n "k",\n v);\n') + self.assertEqual(v, ['setAttribute arg0 "k"']) + + def test_paren_in_string_value_does_not_break_parsing(self): + # The ")" inside the value must not end the call early; key still seen. + v, _ = self._run("src/Foo.cpp", 'g.setAttribute("k", ")");\n') + self.assertEqual(v, ['setAttribute arg0 "k"']) + + def test_rule_h_qualified_constant_warns(self): + v, w = self._run( + "src/Foo.cpp", + "g.setAttribute(consensus::span::accept, v);\n", + header_symbols={"command"}, + ) + self.assertEqual(v, []) + self.assertEqual(w, ["setAttribute arg0 consensus::span::accept"]) + + def test_rule_h_known_constant_no_warning(self): + _, w = self._run( + "src/Foo.cpp", + "g.setAttribute(rpc_span::attr::command, v);\n", + header_symbols={"command"}, + ) + self.assertEqual(w, []) + + def test_rule_h_bare_local_no_warning(self): + _, w = self._run( + "src/Foo.cpp", "g.setAttribute(myLeaf, v);\n", header_symbols={"command"} + ) + self.assertEqual(w, []) + + +class RuleBCollector(unittest.TestCase): + def _run(self, yaml_text, l1): + d = Path(tempfile.mkdtemp()) + try: + _write(d / "docker" / "telemetry" / "otel-collector-config.yaml", yaml_text) + report = chk.Report() + chk.run_rule_b_collector(d, set(l1), report) + return sorted(v[2] for v in report.violations), report.skips + finally: + shutil.rmtree(d) + + def test_dimension_not_in_l1_flagged(self): + y = "spanmetrics:\n dimensions:\n - name: bogus_dim\n - name: command\n" + v, _ = self._run(y, {"command"}) + self.assertEqual(v, ["bogus_dim"]) + + def test_all_dimensions_in_l1_pass(self): + y = "spanmetrics:\n dimensions:\n - name: command\n - name: rpc_status\n" + v, _ = self._run(y, {"command", "rpc_status"}) + self.assertEqual(v, []) + + def test_skip_when_no_spanmetrics_block(self): + v, skips = self._run("receivers:\n otlp:\n", {"command"}) + self.assertEqual(v, []) + self.assertTrue(any("SKIP: B" in s for s in skips)) + + +class RuleCTempo(unittest.TestCase): + """Rule C reads the Grafana Tempo DATASOURCE file's search.filters and + validates only span-scope tags against L1.""" + + DS = "docker/telemetry/grafana/provisioning/datasources/tempo.yaml" + + def _run(self, yaml_text, l1): + d = Path(tempfile.mkdtemp()) + try: + _write(d / self.DS, yaml_text) + report = chk.Report() + chk.run_rule_c_tempo(d, set(l1), report) + return sorted(v[2] for v in report.violations), report.skips + finally: + shutil.rmtree(d) + + def _filter(self, fid, tag, scope): + return ( + f" - id: {fid}\n" + f" tag: {tag}\n" + f' operator: "="\n' + f" scope: {scope}\n" + f" type: static\n" + ) + + def test_span_tag_not_in_l1_flagged(self): + y = "search:\n filters:\n" + self._filter("f1", "bogus_tag", "span") + v, _ = self._run(y, {"command"}) + self.assertEqual(v, ["bogus_tag"]) + + def test_span_tags_in_l1_pass(self): + y = ( + "search:\n filters:\n" + + self._filter("f1", "command", "span") + + self._filter("f2", "tx_hash", "span") + ) + v, _ = self._run(y, {"command", "tx_hash"}) + self.assertEqual(v, []) + + def test_resource_and_intrinsic_tags_ignored(self): + # service.* (resource) and name/status/duration (intrinsic) are not + # span attributes — they must not be validated against L1. + y = ( + "search:\n filters:\n" + + self._filter("f1", "service.instance.id", "resource") + + self._filter("f2", "name", "intrinsic") + + self._filter("f3", "duration", "intrinsic") + ) + v, skips = self._run(y, {"command"}) + self.assertEqual(v, []) + self.assertTrue(any("SKIP: C" in s for s in skips)) + + def test_skip_when_datasource_absent(self): + d = Path(tempfile.mkdtemp()) + try: + report = chk.Report() + chk.run_rule_c_tempo(d, {"command"}, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("SKIP: C" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class RuleDDashboards(unittest.TestCase): + def _run(self, json_text, l1, metric_labels=frozenset()): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "docker" / "telemetry" / "grafana" / "dashboards" / "x.json", + json_text, + ) + report = chk.Report() + chk.run_rule_d_dashboards(d, set(l1), set(metric_labels), report) + return sorted(v[2] for v in report.violations) + finally: + shutil.rmtree(d) + + def test_unknown_promql_label_flagged(self): + self.assertEqual( + self._run('"expr": "sum by (bogus_label) (x)"', {"command"}), + ["bogus_label"], + ) + + def test_builtin_labels_not_flagged(self): + self.assertEqual( + self._run('"expr": "sum by (le, span_name, exported_instance) (x)"', set()), + [], + ) + + def test_external_infra_labels_not_flagged(self): + # EXTERNAL_INFRA_LABELS (perf-iac identity labels with no in-tree + # source) must be recognized as valid, distinct from `builtins`. + expr = "sum by (" + ", ".join(sorted(chk.EXTERNAL_INFRA_LABELS)) + ") (x)" + self.assertEqual(self._run(f'"expr": "{expr}"', set()), []) + + def test_prometheus_name_label_not_flagged(self): + # `__name__` is the Prometheus reserved metric-name label; the renamed + # system-*.json dashboards use `sum by (le, __name__)`. + self.assertEqual( + self._run('"expr": "sum by (le, __name__) (rate(x[5m]))"', set()), + [], + ) + + def test_l1_label_passes(self): + self.assertEqual(self._run('"q": "{command=\\"x\\"}"', {"command"}), []) + + def test_traceql_span_prefix_stripped(self): + # `span.establish_count` must validate against the bare L1 key. + self.assertEqual( + self._run( + '"expr": "count_over_time(x) by (span.establish_count)"', + {"establish_count"}, + ), + [], + ) + + def test_traceql_resource_prefix_stripped(self): + self.assertEqual(self._run('"q": "{resource.service_name=\\"x\\"}"', set()), []) + + def test_native_metric_label_passes(self): + # `job_type` / `reason` are emitted by MetricsRegistry, not span attrs. + self.assertEqual( + self._run( + '"expr": "sum by (job_type, reason) (x)"', + {"command"}, + metric_labels={"job_type", "reason"}, + ), + [], + ) + + def test_unknown_label_still_flagged_with_metric_labels(self): + # A label that is neither L1, metric label, nor builtin still fails. + self.assertEqual( + self._run( + '"expr": "sum by (bogus) (x)"', + {"command"}, + metric_labels={"job_type"}, + ), + ["bogus"], + ) + + def test_span_prefixed_unknown_still_flagged(self): + # `span.not_a_key` whose bare form is unknown is still a violation. + self.assertEqual( + self._run('"expr": "x by (span.not_a_key)"', {"command"}), + ["span.not_a_key"], + ) + + +class MetricLabelExtraction(unittest.TestCase): + """L6: native-metric label keys parsed from C++ instrument calls.""" + + def test_extracts_add_label(self): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "telemetry" / "MetricsRegistry.cpp", + 'counter->Add(1, {{"job_type", std::string(jobType)}});\n' + 'c2->Add(1, {{"reason", std::string(r)}});\n', + ) + self.assertEqual(chk.metric_label_names(d), {"job_type", "reason"}) + finally: + shutil.rmtree(d) + + def test_no_metrics_file_empty(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "src").mkdir() + self.assertEqual(chk.metric_label_names(d), set()) + finally: + shutil.rmtree(d) + + +class ReportExitContract(unittest.TestCase): + @staticmethod + def _exit_code(report): + """Call render_and_exit (which prints + raises SystemExit), swallowing + its stdout, and return the exit code.""" + with contextlib.redirect_stdout(io.StringIO()): + try: + report.render_and_exit() + except SystemExit as e: + return e.code + return None # pragma: no cover - render_and_exit always exits + + def test_violation_exits_nonzero(self): + r = chk.Report() + r.violation("A", "f", "tok", "exp") + self.assertEqual(self._exit_code(r), 1) + + def test_clean_exits_zero(self): + r = chk.Report() + r.ok("all good") + self.assertEqual(self._exit_code(r), 0) + + def test_warning_only_exits_zero(self): + r = chk.Report() + r.warning("H", "f", "tok", "note") + self.assertEqual(self._exit_code(r), 0) + + +class RuleEReportTuple(unittest.TestCase): + """Assert Rule E records the full (rule, expected) tuple, not just token.""" + + def test_violation_tuple_fields(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text("`xrpl.tx.hash`") + report = chk.Report() + chk.run_rule_e_runbook(d, {"xrpl.network.id"}, report) + self.assertEqual(len(report.violations), 1) + rule, _loc, token, expected = report.violations[0] + self.assertEqual(rule, "E") + self.assertEqual(token, "xrpl.tx.hash") + self.assertEqual(expected, "underscore, not dotted") + finally: + shutil.rmtree(d) + + def test_clean_runbook_records_ok(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text( + "`tx_hash` `consensus.round`" + ) + report = chk.Report() + chk.run_rule_e_runbook(d, {"tx_hash"}, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("E:" in c for c in report.checked)) + finally: + shutil.rmtree(d) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 442a202a444..28718bfa91d 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -70,8 +70,10 @@ jobs: files: | # These paths are unique to `on-pr.yml`. .github/scripts/levelization/** + .github/scripts/otel-naming/** .github/scripts/rename/** .github/workflows/reusable-check-levelization.yml + .github/workflows/reusable-check-otel-naming.yml .github/workflows/reusable-check-rename.yml .github/workflows/on-pr.yml @@ -130,6 +132,11 @@ jobs: if: ${{ needs.should-run.outputs.go == 'true' }} uses: ./.github/workflows/reusable-check-levelization.yml + check-otel-naming: + needs: should-run + if: ${{ needs.should-run.outputs.go == 'true' }} + uses: ./.github/workflows/reusable-check-otel-naming.yml + check-rename: needs: should-run if: ${{ needs.should-run.outputs.go == 'true' }} @@ -200,6 +207,7 @@ jobs: if: failure() || cancelled() needs: - check-levelization + - check-otel-naming - check-rename - clang-tidy - build-test diff --git a/.github/workflows/reusable-check-otel-naming.yml b/.github/workflows/reusable-check-otel-naming.yml new file mode 100644 index 00000000000..a7af2da8cd6 --- /dev/null +++ b/.github/workflows/reusable-check-otel-naming.yml @@ -0,0 +1,28 @@ +# This workflow checks that OpenTelemetry span-attribute names stay consistent +# across the code (*SpanNames.h), collector, Tempo, dashboards, and docs. +# See .github/scripts/otel-naming/check_otel_naming.py and the +# "Telemetry span attribute naming" section in CONTRIBUTING.md. +name: Check OTel naming + +# This workflow can only be triggered by other workflows. +on: workflow_call + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-otel-naming + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + otel-naming: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Check OTel naming + # The script is stdlib-only and reads only files already in the tree; + # it enforces each rule only when the layer it needs is present, so it + # works whether telemetry changes land in one PR or several. + run: python .github/scripts/otel-naming/check_otel_naming.py diff --git a/.github/workflows/telemetry-validation.yml b/.github/workflows/telemetry-validation.yml new file mode 100644 index 00000000000..e65505bf881 --- /dev/null +++ b/.github/workflows/telemetry-validation.yml @@ -0,0 +1,310 @@ +# Telemetry Validation CI Workflow +# +# Builds rippled with telemetry enabled, runs the multi-node workload +# harness, validates all telemetry data, and runs performance benchmarks. +# +# This is a separate workflow from the main CI. It runs: +# - On manual dispatch (workflow_dispatch) +# - On pushes to telemetry-related branches +# +# The workflow is intentionally heavyweight (builds rippled, starts Docker +# services, runs a multi-node cluster) — it validates the full telemetry +# stack end-to-end rather than individual unit tests. +# +# Architecture: two jobs to leverage cached dependencies: +# 1. build-xrpld — runs on a self-hosted runner inside the same container +# image the main CI uses (debian-bookworm-gcc-13). This ensures Conan +# packages are fetched from the XRPLF remote instead of built from +# source, and ccache hits the remote cache. +# 2. validate-telemetry — runs on ubuntu-latest (which has Docker) to +# launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.) +# and validate the full pipeline end-to-end. + +name: Telemetry Validation + +on: + workflow_dispatch: + inputs: + rpc_rate: + description: "RPC load rate (requests per second)" + required: false + default: "50" + rpc_duration: + description: "RPC load duration (seconds)" + required: false + default: "120" + tx_tps: + description: "Transaction submit rate (TPS)" + required: false + default: "5" + tx_duration: + description: "Transaction submit duration (seconds)" + required: false + default: "120" + run_benchmark: + description: "Run performance benchmarks" + required: false + type: boolean + default: false + + push: + branches: + - "pratik/otel-phase*" + - "feature/otel-*" + - "feature/telemetry-*" + paths: + - ".github/workflows/telemetry-validation.yml" + - "docker/telemetry/**" + - "include/xrpl/basics/Telemetry*.h" + - "src/xrpld/app/misc/Telemetry*" + +concurrency: + group: telemetry-validation-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + BUILD_DIR: build + +jobs: + # ── Job 1: Build xrpld in the same container the main CI uses ────── + # This ensures Conan binary packages are fetched from the XRPLF remote + # (matching package IDs) and ccache hits the remote compilation cache. + build-xrpld: + name: Build xrpld + runs-on: [self-hosted, Linux, X64, heavy] + container: ghcr.io/xrplf/ci/debian-bookworm:gcc-13-sha-ab4d1f0 + timeout-minutes: 60 + env: + CCACHE_NAMESPACE: telemetry-validation + CCACHE_REMOTE_ONLY: true + CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel + CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Prepare runner + uses: XRPLF/actions/prepare-runner@90f11ee655d1687824fb8793db770477d52afbab + with: + enable_ccache: ${{ github.repository_owner == 'XRPLF' }} + + - name: Print build environment + uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574 + + - name: Get number of processors + uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf + id: nproc + with: + subtract: 2 + + - name: Setup Conan + uses: ./.github/actions/setup-conan + + - name: Build dependencies + uses: ./.github/actions/build-deps + with: + build_nproc: ${{ steps.nproc.outputs.nproc }} + build_type: Release + log_verbosity: verbose + + - name: Configure CMake + working-directory: ${{ env.BUILD_DIR }} + run: | + cmake \ + -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_BUILD_TYPE=Release \ + .. + + - name: Build xrpld + working-directory: ${{ env.BUILD_DIR }} + env: + BUILD_NPROC: ${{ steps.nproc.outputs.nproc }} + run: | + cmake \ + --build . \ + --config Release \ + --parallel "${BUILD_NPROC}" \ + --target xrpld + + - name: Show ccache statistics + if: ${{ github.repository_owner == 'XRPLF' }} + run: ccache --show-stats -vv + + - name: Upload xrpld binary + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xrpld-telemetry + path: ${{ env.BUILD_DIR }}/xrpld + retention-days: 1 + if-no-files-found: error + + # ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ── + validate-telemetry: + name: Telemetry Stack Validation + needs: build-xrpld + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Python dependencies + run: pip3 install -r docker/telemetry/workload/requirements.txt + + - name: Download xrpld binary + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + name: xrpld-telemetry + path: ${{ env.BUILD_DIR }} + + - name: Make binaries and scripts executable + run: | + chmod +x ${{ env.BUILD_DIR }}/xrpld + chmod +x docker/telemetry/workload/*.sh + + - name: Run full telemetry validation + id: validation + env: + RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }} + RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }} + TX_TPS: ${{ github.event.inputs.tx_tps || '5' }} + TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }} + RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }} + run: | + ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld --skip-loki" + ARGS="$ARGS --rpc-rate $RPC_RATE" + ARGS="$ARGS --rpc-duration $RPC_DURATION" + ARGS="$ARGS --tx-tps $TX_TPS" + ARGS="$ARGS --tx-duration $TX_DURATION" + if [ "$RUN_BENCHMARK" = "true" ]; then + ARGS="$ARGS --with-benchmark" + fi + docker/telemetry/workload/run-full-validation.sh $ARGS + # continue-on-error allows subsequent steps (artifact upload, + # summary printing) to run even if validation fails. The final + # "Check validation result" step re-checks steps.validation.outcome + # (the pre-continue-on-error result) and fails the job properly. + continue-on-error: true + + - name: Upload validation reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: telemetry-validation-reports + path: /tmp/xrpld-validation/reports/ + retention-days: 30 + + - name: Upload node logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xrpld-node-logs + path: /tmp/xrpld-validation/node*/debug.log + retention-days: 7 + + - name: Print validation summary + if: always() + run: | + REPORT="/tmp/xrpld-validation/reports/validation-report.json" + if [ -f "$REPORT" ]; then + echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + TOTAL=$(jq '.summary.total' "$REPORT") + PASSED=$(jq '.summary.passed' "$REPORT") + FAILED=$(jq '.summary.failed' "$REPORT") + echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY" + echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY" + echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY" + echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY" + echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + if [ "$FAILED" -gt 0 ]; then + echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY" + fi + fi + + # Publishes captured OTel timings + regression report to the Step Summary. + # When the committed baseline is a placeholder, emits a fenced JSON block + # that can be copy-pasted directly into baselines/baseline-timings.json. + # When the baseline is populated, summarises the top regressions so the + # PR author sees the failure reason without downloading artifacts. + - name: Print regression summary + if: always() + run: | + set -euo pipefail + TIMINGS="/tmp/xrpld-validation/reports/timings.json" + REGRESSION="/tmp/xrpld-validation/reports/regression-report.json" + BASELINE="docker/telemetry/workload/baselines/baseline-timings.json" + + if [ ! -f "$TIMINGS" ]; then + echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY" + echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated." + exit 0 + fi + + if [ ! -f "$BASELINE" ]; then + echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY" + echo "::error::baselines/baseline-timings.json not found in checkout" + exit 1 + fi + + # NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the + # filter's result is boolean false — which is the normal case for a + # populated (non-placeholder) baseline — and that would be + # misreported as a parse failure. Plain `jq -r` exits 0 on any valid + # JSON, so a real non-zero exit genuinely means malformed JSON. + IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || { + echo "::error::Failed to parse baseline JSON" + exit 1 + } + + echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + + if [ "$IS_PLACEHOLDER" = "true" ]; then + echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo "The committed baseline is a placeholder. Open a PR replacing" \ + "its contents with the JSON block below to activate the" \ + "regression gate." >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo '```json' >>"$GITHUB_STEP_SUMMARY" + cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY" + echo '```' >>"$GITHUB_STEP_SUMMARY" + elif [ -f "$REGRESSION" ]; then + REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0 + IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0 + TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0 + echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY" + echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY" + echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY" + echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + if [ "$REGR_COUNT" -gt 0 ]; then + echo "### Regressions" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY" + echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY" + jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \ + "$REGRESSION" >>"$GITHUB_STEP_SUMMARY" + fi + fi + + - name: Cleanup + if: always() + run: | + docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true + + - name: Check validation result + if: steps.validation.outcome == 'failure' + run: | + echo "Telemetry validation failed. Check the uploaded reports for details." + exit 1 diff --git a/.gitignore b/.gitignore index 6bd34ece04c..2554435bf17 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,5 @@ __pycache__ # clangd cache /.cache +docker/telemetry/workload/__pycache__/ +.claude/ diff --git a/CMakeLists.txt b/CMakeLists.txt index f2e8fb3ae52..228bea4adc0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,6 +121,18 @@ if(rocksdb) target_link_libraries(xrpl_libs INTERFACE RocksDB::rocksdb) endif() +# OpenTelemetry distributed tracing (optional). +# When ON, links against opentelemetry-cpp and defines XRPL_ENABLE_TELEMETRY +# so that SpanGuard factory methods produce real OTel spans. +# When OFF (default), all tracing code compiles to no-ops with zero overhead. +# Enable via: conan install -o telemetry=True, or cmake -Dtelemetry=ON. +option(telemetry "Enable OpenTelemetry tracing" ON) +if(telemetry) + find_package(opentelemetry-cpp CONFIG REQUIRED) + add_compile_definitions(XRPL_ENABLE_TELEMETRY) + message(STATUS "OpenTelemetry tracing enabled") +endif() + # Work around changes to Conan recipe for now. if(TARGET nudb::core) set(nudb nudb::core) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7632741e358..31702e7c3c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -354,6 +354,66 @@ If you wish to automatically fix whatever clang-tidy finds _and_ is capable of f run-clang-tidy -p build -quiet -fix -allow-no-checks src tests ``` +## Telemetry span attribute naming + +OpenTelemetry span attribute keys follow these rules so they stay consistent +across the code, the OTel collector, Tempo, Grafana dashboards, and docs. The +constants in the `*SpanNames.h` headers are the single source of truth; every +other layer must match them. A CI check enforces this end to end. + +1. Per-span unique attribute: bare field name — allowed when the field is + recorded by a single span/workflow, so the span name already supplies the + domain (e.g. `command`, `local`, `version` on `rpc.command` / `tx.process`). +2. Shared attribute (same concept on more than one span): ONE key, reused + verbatim on every span that records it — the span name tells the occurrences + apart, so no per-emitter prefix is added. Pick the name by the field's + meaning: a property of a domain object keeps that object's bare field name + (`ledger_hash`, `ledger_seq`, `tx_hash`, `peer_id`, `full_validation`); a + field already qualified by a sub-kind keeps that qualifier on every emitter + (`proposal_trusted` on both `consensus.proposal.receive` and + `peer.proposal.receive`; `validation_trusted` likewise). Define it once in + the base `SpanNames.h` `namespace attr` block and re-export (`using`) it from + each domain header, so all emitters share the exact string. +3. Collision qualifier: `_` — only when a bare name would collide + with a DIFFERENT concept in the shared spanmetrics label space, or with the + OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, + `consensus_phase`, `consensus_round`). This disambiguates distinct concepts + that share a word; it is NOT used to tag the same concept with the workflow + that emitted it — that is rule 2 (one shared name). +4. Resource attribute: dotted `xrpl..` — reserved ONLY for + process/network identity set once at startup (`xrpl.network.id`, + `xrpl.network.type`). Never use the dotted `xrpl.` form for span attributes. +5. Span names use `[.]` (dotted). Only attribute _keys_ + follow rules 1–4. + +All attribute keys are `lower_snake_case` (lowercase letters, digits, and +underscores; each dot-separated segment of a resource key likewise). No +camelCase, uppercase, or spaces. + +Standard OpenTelemetry semantic-convention keys keep their canonical dotted +form (e.g. `service.*` resource attributes, `http.*` span attributes); the +"no dotted form" rule above applies to xrpl-custom keys, not to OTel-standard +conventions. + +Always reference the `*SpanNames.h` constants for attribute keys and span +names — never pass a string literal as a key or as a `span`/`childSpan` name +argument. (Attribute _values_ may be runtime data.) + +These rules are enforced by `.github/scripts/otel-naming/check_otel_naming.py`, +run in CI on every pull request. The check derives the set of valid keys +directly from the `*SpanNames.h` constants and the resource attributes the code +registers, so there is no separate list to keep in sync. It cross-validates the +collector, Tempo, dashboards, and docs against those keys, and each rule runs +only when the file it needs is present — so it works whether telemetry changes +land in one pull request or several. Run it locally with: + +``` +python .github/scripts/otel-naming/check_otel_naming.py +``` + +See [.github/scripts/otel-naming/README.md](.github/scripts/otel-naming/README.md) +for the full rule list. + ## Contracts and instrumentation We are using [Antithesis](https://antithesis.com/) for continuous fuzzing, diff --git a/OpenTelemetryPlan/00-tracing-fundamentals.md b/OpenTelemetryPlan/00-tracing-fundamentals.md new file mode 100644 index 00000000000..9c6f96d7afd --- /dev/null +++ b/OpenTelemetryPlan/00-tracing-fundamentals.md @@ -0,0 +1,565 @@ +# Distributed Tracing Fundamentals + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Next**: [Architecture Analysis](./01-architecture-analysis.md) + +--- + +## What is Distributed Tracing? + +Distributed tracing is a method for tracking data objects as they flow through distributed systems. In a network like XRP Ledger, a single transaction touches multiple independent nodes—each with no shared memory or logging. Distributed tracing connects these dots. + +**Without tracing:** You see isolated logs on each node with no way to correlate them. + +**With tracing:** You see the complete journey of a transaction or an event across all nodes it touched. + +--- + +## Actors and Actions at a Glance + +### Actors + +| Who (Plain English) | Technical Term | +| ---------------------------------------------- | --------------- | +| A single unit of work being tracked | Span | +| The complete journey of a request | Trace | +| Data that links spans across services | Trace Context | +| Code that creates spans and propagates context | Instrumentation | +| Service that receives and processes traces | Collector | +| Storage and visualization system | Backend (Tempo) | +| Decision logic for which traces to keep | Sampler | + +### Actions + +| What Happens (Plain English) | Technical Term | +| --------------------------------------- | ----------------------- | +| Start tracking a new operation | Create a Span | +| Connect a child operation to its parent | Set `parent_span_id` | +| Group all related operations together | Share a `trace_id` | +| Pass tracking data between services | Context Propagation | +| Decide whether to record a trace | Sampling (Head or Tail) | +| Send completed traces to storage | Export (OTLP) | + +--- + +## Core Concepts + +### 1. Trace + +A **trace** represents the entire journey of a request through the system. It has a unique `trace_id` that stays constant across all nodes. + +``` +Trace ID: abc123 +├── Node A: received transaction +├── Node B: relayed transaction +├── Node C: included in consensus +└── Node D: applied to ledger +``` + +### 2. Span + +A **span** represents a single unit of work within a trace. Each span has: + +| Attribute | Description | Example | +| ---------------- | -------------------------------- | -------------------------- | +| `trace_id` | Identifies the trace | `event123` | +| `span_id` | Unique identifier | `span456` | +| `parent_span_id` | Parent span (if any) | `p_span123` | +| `name` | Operation name | `rpc.submit` | +| `start_time` | When work began (local time) | `2024-01-15T10:30:00Z` | +| `end_time` | When work completed (local time) | `2024-01-15T10:30:00.050Z` | +| `attributes` | Key-value metadata | `tx_hash=ABC...` | +| `status` | OK, ERROR MSG | `OK` | + +### 3. Trace Context + +**Trace context** is the data that propagates between services to link spans together. It contains: + +- `trace_id` - The trace this span belongs to +- `span_id` - The current span (becomes parent for child spans) +- `trace_flags` - Sampling decisions + +--- + +## How Spans Form a Trace + +Spans have parent-child relationships forming a tree structure: + +```mermaid +flowchart TB + subgraph trace["Trace: abc123"] + A["tx.submit
span_id: 001
50ms"] --> B["tx.validate
span_id: 002
5ms"] + A --> C["tx.relay
span_id: 003
10ms"] + A --> D["tx.apply
span_id: 004
30ms"] + D --> E["ledger.update
span_id: 005
20ms"] + end + + style A fill:#0d47a1,stroke:#082f6a,color:#ffffff + style B fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style C fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style D fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style E fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **tx.submit (blue, root)**: The top-level span representing the entire transaction submission; all other spans are its descendants. +- **tx.validate, tx.relay, tx.apply (green)**: Direct children of tx.submit, representing the three main stages -- validation, relay to peers, and application to the ledger. +- **ledger.update (red)**: A grandchild span nested under tx.apply, representing the actual ledger state mutation triggered by applying the transaction. +- **Arrows (parent to child)**: Each arrow indicates a parent-child span relationship where the parent's completion depends on the child finishing. + +The same trace visualized as a **timeline (Gantt chart)**: + +``` +Time → 0ms 10ms 20ms 30ms 40ms 50ms + ├───────────────────────────────────────────┤ +tx.submit│▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│ + ├─────┤ +tx.valid │▓▓▓▓▓│ + │ ├──────────┤ +tx.relay │ │▓▓▓▓▓▓▓▓▓▓│ + │ ├────────────────────────────┤ +tx.apply │ │▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│ + │ ├──────────────────┤ +ledger │ │▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│ +``` + +--- + +## Span Relationships + +Spans don't always form simple parent-child trees. Distributed tracing defines several relationship types to capture different causal patterns: + +### 1. Parent-Child (ChildOf) + +The default relationship. The parent span **depends on** or **contains** the child span. The child runs within the scope of the parent. + +``` +tx.submit (parent) +├── tx.validate (child) ← parent waits for this +├── tx.relay (child) ← parent waits for this +└── tx.apply (child) ← parent waits for this +``` + +**When to use:** Synchronous calls, nested operations, any case where the parent's completion depends on the child. + +### 2. Follows-From + +A causal relationship where the first span **triggers** the second, but does **not wait** for it. The originator fires and moves on. + +``` +Time → + +tx.receive [=======] + ↓ triggers (follows-from) + tx.relay [===========] ← runs independently +``` + +**When to use:** Asynchronous jobs, queued work, fire-and-forget patterns. For example, a node receives a transaction and queues it for relay — the relay span _follows from_ the receive span but the receiver doesn't wait for relaying to complete. + +> **OpenTracing** defined `FollowsFrom` as a first-class reference type alongside `ChildOf`. +> **OpenTelemetry** represents this using **Span Links** with descriptive attributes instead (see below). + +### 3. Span Links (Cross-Trace and Non-Hierarchical) + +Links connect spans that are **causally related but not in a parent-child hierarchy**. Unlike parent-child, links can cross trace boundaries. + +``` +Trace A Trace B +────── ────── +batch.schedule batch.execute +├─ item.enqueue (span X) ┌──► process.item +├─ item.enqueue (span Y) ───┤ (links to X, Y, Z) +├─ item.enqueue (span Z) └──► +``` + +**Use cases:** + +| Pattern | Description | +| -------------------- | --------------------------------------------------------------------------- | +| **Batch processing** | A batch span links back to all individual spans that contributed to it | +| **Fan-in** | An aggregation span links to the multiple producer spans it merges | +| **Fan-out** | Multiple downstream spans link back to the single span that triggered them | +| **Async handoff** | A deferred job links back to the request that queued it (follows-from) | +| **Cross-trace** | Correlating spans across independent traces (e.g., retries, related events) | + +**Link structure:** Each link carries the target span's context plus optional attributes: + +``` +Link { + trace_id: + span_id: + attributes: { "link.description": "triggered by batch scheduler" } +} +``` + +### Relationship Summary + +```mermaid +flowchart LR + subgraph parent_child["Parent-Child"] + direction TB + P["Parent"] --> C["Child"] + end + + subgraph follows_from["Follows-From"] + direction TB + A["Span A"] -.->|triggers| B["Span B"] + end + + subgraph links["Span Links"] + direction TB + X["Span X\n(Trace 1)"] -.-|link| Y["Span Y\n(Trace 2)"] + end + + parent_child ~~~ follows_from ~~~ links + + style P fill:#0d47a1,stroke:#082f6a,color:#ffffff + style C fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style A fill:#0d47a1,stroke:#082f6a,color:#ffffff + style B fill:#bf360c,stroke:#8c2809,color:#ffffff + style X fill:#4a148c,stroke:#38006b,color:#ffffff + style Y fill:#4a148c,stroke:#38006b,color:#ffffff +``` + +| Relationship | Same Trace? | Dependency? | OTel Mechanism | +| ---------------- | ----------- | -------------------------- | ----------------- | +| **Parent-Child** | Yes | Parent depends on child | `parent_span_id` | +| **Follows-From** | Usually | Causal but no dependency | Link + attributes | +| **Span Link** | Either | Correlation, no dependency | Link + attributes | + +--- + +## Trace ID Generation + +A `trace_id` is a 128-bit (16-byte) identifier that groups all spans belonging to one logical operation. How it's generated determines how easily you can find and correlate traces later. + +### General Approaches + +#### 1. Random (W3C Default) + +Generate a random 128-bit ID when a trace starts. Standard approach for most services. + +``` +trace_id = random_128_bits() +``` + +| Pros | Cons | +| --------------------------- | --------------------------------------------- | +| Simple, standard | No natural correlation to domain events | +| Guaranteed unique per trace | If propagation is lost, trace is broken | +| Works with all OTel tooling | "Find trace for TX abc" requires index lookup | + +#### 2. Deterministic (Derived from Domain Data) + +Compute the trace_id from a hash of a natural identifier. Every node independently derives the **same** trace_id for the same event. + +``` +trace_id = SHA-256(domain_identifier)[0:16] // truncate to 128 bits +``` + +| Pros | Cons | +| --------------------------------------------------- | ---------------------------------------------------------- | +| Propagation-resilient — same ID computed everywhere | Same event processed twice (retry) shares trace_id | +| Natural search — domain ID maps directly to trace | Non-standard (tooling assumes random) | +| No coordination needed between nodes | 256→128 bit truncation (collision risk negligible at ~2⁶⁴) | + +#### 3. Hybrid (Deterministic Prefix + Random Suffix) + +First 8 bytes derived from domain data, last 8 bytes random. + +``` +trace_id = SHA-256(domain_identifier)[0:8] || random_64_bits() +``` + +| Pros | Cons | +| ------------------------------------------- | ---------------------------------------- | +| Prefix search: "find all traces for TX abc" | Must propagate to maintain full trace_id | +| Unique per processing instance | More complex generation logic | +| Retries get distinct trace_ids | Partial correlation only (prefix match) | + +### XRPL Workflow Analysis + +XRPL has a unique advantage: its core workflows produce **globally unique 256-bit hashes** that are known on every node. This makes deterministic trace_id generation practical in ways most systems can't achieve. + +#### Natural Identifiers by Workflow + +| Workflow | Natural Identifier | Size | Known at Start? | Same on All Nodes? | +| ------------------- | --------------------------------- | ---------- | ----------------------------- | -------------------------------- | +| **Transaction** | Transaction hash (`tid_`) | 256-bit | Yes — computed before signing | Yes — hash of canonical tx data | +| **Consensus round** | Previous ledger hash + ledger seq | 256+32 bit | Yes — known when round opens | Yes — all validators agree | +| **Validation** | Ledger hash being validated | 256-bit | Yes — from consensus result | Yes — same closed ledger | +| **Ledger catch-up** | Target ledger hash | 256-bit | Yes — we know what to fetch | Yes — identifies ledger globally | + +#### Where These Identifiers Live in Code + +``` +Transaction: STTx::getTransactionID() → uint256 tid_ + TMTransaction::rawTransaction → recompute hash from bytes + +Consensus: ConsensusProposal::prevLedger_ → uint256 (previous ledger hash) + ConsensusProposal::position_ → uint256 (TxSet hash) + LedgerHeader::seq → uint32_t (ledger sequence) + +Validation: STValidation::getLedgerHash() → uint256 + STValidation::getNodeID() → NodeID (160-bit) + +Ledger fetch: InboundLedger constructor → uint256 hash, uint32_t seq + TMGetLedger::ledgerHash → bytes (uint256) +``` + +### Recommended Strategy: Workflow-Scoped Deterministic + +Each workflow type derives its trace_id from its natural domain identifier: + +``` +Transaction trace: trace_id = SHA-256("tx" || tx_hash)[0:16] +Consensus trace: trace_id = SHA-256("cons" || prev_ledger_hash || ledger_seq)[0:16] +Ledger catch-up: trace_id = SHA-256("fetch" || target_ledger_hash)[0:16] +``` + +The string prefix (`"tx"`, `"cons"`, `"fetch"`) prevents collisions between workflows that might share underlying hashes. + +**Why this works for XRPL:** + +1. **Propagation-resilient** — Even if a P2P message drops trace context, every node independently computes the same trace_id from the same tx_hash or ledger_hash. Spans still correlate. + +2. **Zero-cost search** — "Show me the trace for transaction ABC" becomes a direct lookup: compute `SHA-256("tx" || ABC)[0:16]` and query. No secondary index needed. + +3. **Cross-workflow linking via Span Links** — A consensus trace links to individual transaction traces. A validation span links to the consensus trace. This connects the full picture without forcing everything into one giant trace. + +### Cross-Workflow Correlation + +Each workflow gets its own trace. Span Links tie them together: + +```mermaid +flowchart TB + subgraph tx_trace["Transaction Trace"] + direction LR + Tn["trace_id = f(tx_hash)"]:::note --> T1["tx.receive"] --> T2["tx.validate"] --> T3["tx.relay"] + end + + subgraph cons_trace["Consensus Trace"] + direction LR + Cn["trace_id = f(prev_ledger, seq)"]:::note --> C1["cons.open"] --> C2["cons.propose"] --> C3["cons.accept"] + end + + subgraph val_trace["Validation"] + direction LR + Vn["spans within consensus trace"]:::note --> V1["val.create"] --> V2["val.broadcast"] + end + + subgraph fetch_trace["Catch-Up Trace"] + direction LR + Fn["trace_id = f(ledger_hash)"]:::note --> F1["fetch.request"] --> F2["fetch.receive"] --> F3["fetch.apply"] + end + + C1 -.-|"span link\n(tx traces)"| T3 + C3 --> V1 + F1 -.-|"span link\n(target ledger)"| C3 + + classDef note fill:none,stroke:#888,stroke-dasharray:5 5,color:#333,font-style:italic + style T1 fill:#0d47a1,stroke:#082f6a,color:#ffffff + style T2 fill:#0d47a1,stroke:#082f6a,color:#ffffff + style T3 fill:#0d47a1,stroke:#082f6a,color:#ffffff + style C1 fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style C2 fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style C3 fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style V1 fill:#bf360c,stroke:#8c2809,color:#ffffff + style V2 fill:#bf360c,stroke:#8c2809,color:#ffffff + style F1 fill:#4a148c,stroke:#38006b,color:#ffffff + style F2 fill:#4a148c,stroke:#38006b,color:#ffffff + style F3 fill:#4a148c,stroke:#38006b,color:#ffffff +``` + +**Reading the diagram:** + +- **Transaction Trace (blue)**: An independent trace whose `trace_id` is deterministically derived from the transaction hash. Contains receive, validate, and relay spans. +- **Consensus Trace (green)**: An independent trace whose `trace_id` is derived from the previous ledger hash and sequence number. Covers the open, propose, and accept phases. +- **Validation (red)**: Validation spans live within the consensus trace (not a separate trace). They are created after the accept phase completes. +- **Catch-Up Trace (purple)**: An independent trace for ledger acquisition, derived from the target ledger hash. Used when a node is behind and fetching missing ledgers. +- **Dotted arrows (span links)**: Cross-trace correlations. Consensus links to transaction traces it included; catch-up links to the consensus trace that produced the target ledger. +- **Solid arrow (C3 to V1)**: A parent-child relationship -- validation spans are direct children of the consensus accept span within the same trace. + +**How a query flows:** + +``` +"Why was TX abc slow?" + 1. Compute trace_id = SHA-256("tx" || abc)[0:16] + 2. Find transaction trace → see it was included in consensus round N + 3. Follow span link → consensus trace for round N + 4. See which phase was slow (propose? accept?) + 5. If a node was catching up, follow link → catch-up trace +``` + +### Trade-offs to Consider + +| Concern | Mitigation | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| **Retries get same trace_id** | Add `attempt` attribute to root span; spans have unique span_ids and timestamps | +| **256→128 bit truncation** | Birthday-bound collision at ~2⁶⁴ operations — negligible for XRPL's throughput | +| **Non-standard generation** | OTel spec allows any 16-byte non-zero value; tooling works on the hex string | +| **Hash computation cost** | SHA-256 is ~0.3μs per call; XRPL already computes these hashes for other purposes | +| **Late-binding identifiers** | Ledger hash isn't known until after consensus — validation spans use ledger_seq as fallback, then link to the consensus trace | + +--- + +## Distributed Traces Across Nodes + +In distributed systems like xrpld, traces span **multiple independent nodes**. The trace context must be propagated in network messages: + +```mermaid +sequenceDiagram + participant Client + participant NodeA as Node A + participant NodeB as Node B + participant NodeC as Node C + + Client->>NodeA: Submit TX
(no trace context) + + Note over NodeA: Creates new trace
trace_id: abc123
span: tx.receive + + NodeA->>NodeB: Relay TX
(trace_id: abc123, parent: 001) + + Note over NodeB: Creates child span
span: tx.relay
parent_span_id: 001 + + NodeA->>NodeC: Relay TX
(trace_id: abc123, parent: 001) + + Note over NodeC: Creates child span
span: tx.relay
parent_span_id: 001 + + Note over NodeA,NodeC: All spans share trace_id: abc123
enabling correlation across nodes +``` + +**Reading the diagram:** + +- **Client**: The external entity that submits a transaction. It does not carry trace context -- the trace originates at the first node. +- **Node A**: The entry point that creates a new trace (trace_id: abc123) and the root span `tx.receive`. It relays the transaction to peers with trace context attached. +- **Node B and Node C**: Peer nodes that receive the relayed transaction along with the propagated trace context. Each creates a child span under Node A's span, preserving the same `trace_id`. +- **Arrows with trace context**: The relay messages carry `trace_id` and `parent_span_id`, allowing each downstream node to link its spans back to the originating span on Node A. + +--- + +## Context Propagation + +For traces to work across nodes, **trace context must be propagated** in messages. + +### What's in the Context (~26 bytes) + +| Field | Size | Description | +| ------------- | -------- | ------------------------------------------------------- | +| `trace_id` | 16 bytes | Identifies the entire trace (constant across all nodes) | +| `span_id` | 8 bytes | The sender's current span (becomes parent on receiver) | +| `trace_flags` | 1 byte | Sampling decision (bit 0 = sampled; bits 1-7 reserved) | +| `trace_state` | variable | Optional vendor-specific data (typically omitted) | + +### How span_id Changes at Each Hop + +Only **one** `span_id` travels in the context - the sender's current span. Each node: + +1. Extracts the received `span_id` and uses it as the `parent_span_id` +2. Creates a **new** `span_id` for its own span +3. Sends its own `span_id` as the parent when forwarding + +``` +Node A Node B Node C +────── ────── ────── + +Span AAA Span BBB Span CCC + │ │ │ + ▼ ▼ ▼ +Context out: Context out: Context out: +├─ trace_id: abc123 ├─ trace_id: abc123 ├─ trace_id: abc123 +├─ span_id: AAA ──────────► ├─ span_id: BBB ──────────► ├─ span_id: CCC ──────► +└─ flags: 01 └─ flags: 01 └─ flags: 01 + │ │ + parent = AAA parent = BBB +``` + +The `trace_id` stays constant, but `span_id` **changes at every hop** to maintain the parent-child chain. + +### Propagation Formats + +There are two patterns: + +### HTTP/RPC Headers (W3C Trace Context) + +``` +traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01 + │ │ │ │ + │ │ │ └── Flags (sampled) + │ │ └── Parent span ID (16 hex) + │ └── Trace ID (32 hex) + └── Version +``` + +### Protocol Buffers (xrpld P2P messages) + +xrpld P2P messages such as `TMTransaction` carry the trace context in two added byte fields alongside the existing payload: `trace_parent` holds the W3C traceparent (`trace_id`, `span_id`, and `trace_flags`), and `trace_state` holds the optional W3C tracestate. Together they propagate the trace across the P2P boundary so a receiving node can attach its spans to the sender's span. + +--- + +## Sampling + +Not every trace needs to be recorded. **Sampling** reduces overhead: + +### Head Sampling (at trace start) + +``` +Request arrives → Random N% chance → Record or skip entire trace +``` + +- ✅ Low overhead +- ❌ May miss interesting traces + +> **xrpld note**: xrpld intentionally fixes head sampling at 100% (sample +> everything) and does not expose a configurable ratio. A per-node ratio +> would let different nodes make divergent keep/drop decisions for the same +> distributed trace, producing broken/partial traces. xrpld uses a +> `ParentBased` sampler so spans with a remote parent honor the upstream +> decision. Volume reduction is delegated to collector-side tail sampling. + +### Tail Sampling (after trace completes) + +``` +Trace completes → Collector evaluates: + - Error? → KEEP + - Slow? → KEEP + - Normal? → Sample 10% +``` + +- ✅ Never loses important traces +- ❌ Higher memory usage at collector + +--- + +## Key Benefits for xrpld + +| Challenge | How Tracing Helps | +| ---------------------------------- | ---------------------------------------- | +| "Where is my transaction?" | Follow trace across all nodes it touched | +| "Why was consensus slow?" | See timing breakdown of each phase | +| "Which node is the bottleneck?" | Compare span durations across nodes | +| "What happened during the outage?" | Correlate errors across the network | + +--- + +## Glossary + +| Term | Definition | +| -------------------- | ------------------------------------------------------------------- | +| **Trace** | Complete journey of a request, identified by `trace_id` | +| **Span** | Single operation within a trace | +| **Parent-Child** | Span relationship where the parent depends on the child | +| **Follows-From** | Causal relationship where originator doesn't wait for the result | +| **Span Link** | Non-hierarchical connection between spans, possibly across traces | +| **Deterministic ID** | Trace ID derived from domain data (e.g., tx_hash) instead of random | +| **Context** | Data propagated between services (`trace_id`, `span_id`, flags) | +| **Instrumentation** | Code that creates spans and propagates context | +| **Collector** | Service that receives, processes, and exports traces | +| **Backend** | Storage/visualization system (Tempo) | +| **Head Sampling** | Sampling decision at trace start | +| **Tail Sampling** | Sampling decision after trace completes | + +--- + +_Next: [Architecture Analysis](./01-architecture-analysis.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/01-architecture-analysis.md b/OpenTelemetryPlan/01-architecture-analysis.md new file mode 100644 index 00000000000..1161a1645b0 --- /dev/null +++ b/OpenTelemetryPlan/01-architecture-analysis.md @@ -0,0 +1,467 @@ +# Architecture Analysis + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Design Decisions](./02-design-decisions.md) | [Implementation Strategy](./03-implementation-strategy.md) + +--- + +## 1.1 Current xrpld Architecture Overview + +> **WS** = WebSocket | **UNL** = Unique Node List | **TxQ** = Transaction Queue | **StatsD** = Statistics Daemon + +The xrpld node software consists of several interconnected components that need instrumentation for distributed tracing: + +```mermaid +flowchart TB + subgraph xrpld["xrpld Node"] + subgraph services["Core Services"] + RPC["RPC Server
(HTTP/WS/gRPC)"] + Overlay["Overlay
(P2P Network)"] + Consensus["Consensus
(RCLConsensus)"] + ValidatorList["ValidatorList
(UNL Mgmt)"] + end + + JobQueue["JobQueue
(Thread Pool)"] + + subgraph processing["Processing Layer"] + NetworkOPs["NetworkOPs
(Tx Processing)"] + LedgerMaster["LedgerMaster
(Ledger Mgmt)"] + NodeStore["NodeStore
(Database)"] + InboundLedgers["InboundLedgers
(Ledger Sync)"] + end + + subgraph appservices["Application Services"] + PathFind["PathFinding
(Payment Paths)"] + TxQ["TxQ
(Fee Escalation)"] + LoadMgr["LoadManager
(Fee/Load)"] + end + + subgraph observability["Existing Observability"] + PerfLog["PerfLog
(JSON)"] + Insight["Insight
(StatsD)"] + Logging["Logging
(Journal)"] + end + + services --> JobQueue + JobQueue --> processing + JobQueue --> appservices + end + + style xrpld fill:#424242,stroke:#212121,color:#ffffff + style services fill:#1565c0,stroke:#0d47a1,color:#ffffff + style processing fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style appservices fill:#6a1b9a,stroke:#4a148c,color:#ffffff + style observability fill:#e65100,stroke:#bf360c,color:#ffffff +``` + +**Reading the diagram:** + +- **Core Services (blue)**: The entry points into xrpld -- RPC Server handles client requests, Overlay manages peer-to-peer networking, Consensus drives agreement, and ValidatorList manages trusted validators. +- **JobQueue (center)**: The asynchronous thread pool that decouples Core Services from the Processing and Application layers. All work flows through it. +- **Processing Layer (green)**: Core business logic -- NetworkOPs processes transactions, LedgerMaster manages ledger state, NodeStore handles persistence, and InboundLedgers synchronizes missing data. +- **Application Services (purple)**: Higher-level features -- PathFinding computes payment routes, TxQ manages fee-based queuing, and LoadManager tracks server load. +- **Existing Observability (orange)**: The current monitoring stack (PerfLog, Insight, Journal logging) that OpenTelemetry will complement, not replace. +- **Arrows (Services to JobQueue to layers)**: Work originates at Core Services, is enqueued onto the JobQueue, and dispatched to Processing or Application layers for execution. + +--- + +## 1.1.1 Actors and Actions + +### Actors + +| Who (Plain English) | Technical Term | +| ----------------------------------------- | -------------------------- | +| Network node running XRPL software | xrpld node | +| External client submitting requests | RPC Client | +| Network neighbor sharing data | Peer (PeerImp) | +| Request handler for client queries | RPC Server (ServerHandler) | +| Command executor for specific RPC methods | RPCHandler | +| Agreement process between nodes | Consensus (RCLConsensus) | +| Transaction processing coordinator | NetworkOPs | +| Background task scheduler | JobQueue | +| Ledger state manager | LedgerMaster | +| Payment route calculator | PathFinding (Pathfinder) | +| Transaction waiting room | TxQ (Transaction Queue) | +| Fee adjustment system | LoadManager | +| Trusted validator list manager | ValidatorList | +| Protocol upgrade tracker | AmendmentTable | +| Ledger state hash tree | SHAMap | +| Persistent key-value storage | NodeStore | + +### Actions + +| What Happens (Plain English) | Technical Term | +| ---------------------------------------------- | ---------------------- | +| Client sends a request to a node | `rpc.request` | +| Node executes a specific RPC command | `rpc.command.*` | +| Node receives a transaction from a peer | `tx.receive` | +| Node checks if a transaction is valid | `tx.validate` | +| Node forwards a transaction to neighbors | `tx.relay` | +| Nodes agree on which transactions to include | `consensus.round` | +| Consensus progresses through phases | `consensus.phase.*` | +| Node builds a new confirmed ledger | `ledger.build` | +| Node fetches missing ledger data from peers | `ledger.acquire` | +| Node computes payment routes | `pathfind.compute` | +| Node queues a transaction for later processing | `txq.enqueue` | +| Node increases fees due to high load | `fee.escalate` | +| Node fetches the latest trusted validator list | `validator.list.fetch` | +| Node votes on a protocol amendment | `amendment.vote` | +| Node synchronizes state tree data | `shamap.sync` | + +--- + +## 1.2 Key Components for Instrumentation + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List + +| Component | Location | Purpose | Trace Value | +| ------------------ | ------------------------------------------ | ------------------------ | -------------------------------- | +| **Overlay** | `src/xrpld/overlay/` | P2P communication | Message propagation timing | +| **PeerImp** | `src/xrpld/overlay/detail/PeerImp.cpp` | Individual peer handling | Per-peer latency | +| **RCLConsensus** | `src/xrpld/app/consensus/RCLConsensus.cpp` | Consensus algorithm | Round timing, phase analysis | +| **NetworkOPs** | `src/xrpld/app/misc/NetworkOPs.cpp` | Transaction processing | Tx lifecycle tracking | +| **ServerHandler** | `src/xrpld/rpc/detail/ServerHandler.cpp` | RPC entry point | Request latency | +| **RPCHandler** | `src/xrpld/rpc/detail/RPCHandler.cpp` | Command execution | Per-command timing | +| **JobQueue** | `src/xrpl/core/JobQueue.h` | Async task execution | Queue wait times | +| **PathFinding** | `src/xrpld/app/paths/` | Payment path computation | Path latency, cache hits | +| **TxQ** | `src/xrpld/app/misc/TxQ.cpp` | Transaction queue/fees | Queue depth, eviction rates | +| **LoadManager** | `src/xrpld/app/main/LoadManager.cpp` | Fee escalation/load | Fee levels, load factors | +| **InboundLedgers** | `src/xrpld/app/ledger/InboundLedgers.cpp` | Ledger acquisition | Sync time, peer reliability | +| **ValidatorList** | `src/xrpld/app/misc/ValidatorList.cpp` | UNL management | List freshness, fetch failures | +| **AmendmentTable** | `src/xrpld/app/misc/AmendmentTable.cpp` | Protocol amendments | Voting status, activation events | +| **SHAMap** | `src/xrpld/shamap/` | State hash tree | Sync speed, missing nodes | + +--- + +## 1.3 Transaction Flow Diagram + +Transaction flow spans multiple nodes in the network. Each node creates linked spans to form a distributed trace: + +```mermaid +sequenceDiagram + participant Client + participant PeerA as Peer A (Receive) + participant PeerB as Peer B (Relay) + participant PeerC as Peer C (Validate) + + Client->>PeerA: 1. Submit TX + + rect rgb(230, 245, 255) + Note over PeerA: tx.receive SPAN START + PeerA->>PeerA: HashRouter Deduplication + PeerA->>PeerA: tx.validate (child span) + end + + PeerA->>PeerB: 2. Relay TX (with trace ctx) + + rect rgb(230, 245, 255) + Note over PeerB: tx.receive (linked span) + end + + PeerB->>PeerC: 3. Relay TX + + rect rgb(230, 245, 255) + Note over PeerC: tx.receive (linked span) + PeerC->>PeerC: tx.process + end + + Note over Client,PeerC: DISTRIBUTED TRACE (same trace_id: abc123) +``` + +**Reading the diagram:** + +- **Client**: The external entity that submits a transaction to Peer A. It has no trace context -- the trace starts at the first node. +- **Peer A (Receive)**: The entry node that creates the root span `tx.receive`, runs HashRouter deduplication to avoid processing duplicates, and creates a child `tx.validate` span. +- **Peer A to Peer B arrow**: The relay message carries trace context (trace_id + parent span_id), enabling Peer B to create a linked span under the same trace. +- **Peer B (Relay)**: Receives the transaction and trace context, creates a `tx.receive` span linked to Peer A's trace, then relays onward. +- **Peer C (Validate)**: Final hop in this example. Creates a linked `tx.receive` span and runs `tx.process` to fully process the transaction. +- **Blue rectangles**: Highlight the span boundaries on each node, showing where instrumentation creates and closes spans. + +### Trace Structure + +``` +trace_id: abc123 +├── span: tx.receive (Peer A) +│ ├── span: tx.validate +│ └── span: tx.relay +├── span: tx.receive (Peer B) [parent: Peer A] +│ └── span: tx.relay +└── span: tx.receive (Peer C) [parent: Peer B] + └── span: tx.process +``` + +--- + +## 1.4 Consensus Round Flow + +Consensus rounds are multi-phase operations that benefit significantly from tracing: + +```mermaid +flowchart TB + subgraph round["consensus.round (root span)"] + attrs["Attributes:
ledger_seq = 12345678
consensus_mode = proposing
proposers = 35"] + + subgraph open["consensus.phase.open"] + open_desc["Duration: ~3s
Waiting for transactions"] + end + + subgraph establish["consensus.phase.establish"] + est_attrs["proposals_received = 28
disputes_resolved = 3"] + est_children["├── consensus.proposal.receive (×28)
├── consensus.proposal.send (×1)
└── consensus.dispute.resolve (×3)"] + end + + subgraph accept["consensus.phase.accept"] + acc_attrs["transactions_applied = 150
ledger_hash = DEF456..."] + acc_children["├── ledger.build
└── ledger.validate"] + end + + attrs --> open + open --> establish + establish --> accept + end + + style round fill:#f57f17,stroke:#e65100,color:#ffffff + style open fill:#1565c0,stroke:#0d47a1,color:#ffffff + style establish fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style accept fill:#c2185b,stroke:#880e4f,color:#ffffff +``` + +**Reading the diagram:** + +- **consensus.round (orange, root span)**: The top-level span encompassing the entire consensus round, with attributes like ledger sequence, mode, and proposer count. +- **consensus.phase.open (blue)**: The first phase where the node waits (~3s) to collect incoming transactions before proposing. +- **consensus.phase.establish (green)**: The negotiation phase where validators exchange proposals, resolve disputes, and converge on a transaction set. Child spans track each proposal received/sent and each dispute resolved. +- **consensus.phase.accept (pink)**: The final phase where the agreed transaction set is applied, a new ledger is built, and the ledger is validated. Child spans cover `ledger.build` and `ledger.validate`. +- **Arrows (open to establish to accept)**: The sequential flow through the three consensus phases. Each phase must complete before the next begins. + +--- + +## 1.5 RPC Request Flow + +> **WS** = WebSocket + +RPC requests support W3C Trace Context headers for distributed tracing across services: + +```mermaid +flowchart TB + subgraph request["rpc.request (root span)"] + http["HTTP Request — POST /
traceparent:
00-abc123...-def456...-01"] + + attrs["Attributes:
http.method = POST
net.peer.ip = 192.168.1.100
command = submit"] + + subgraph enqueue["jobqueue.enqueue"] + job_attr["job_type = jtCLIENT_RPC"] + end + + subgraph command["rpc.command.submit"] + cmd_attrs["version = 2
rpc_role = user"] + cmd_children["├── tx.deserialize
├── tx.validate_local
└── tx.submit_to_network"] + end + + response["Response: 200 OK
Duration: 45ms"] + + http --> attrs + attrs --> enqueue + enqueue --> command + command --> response + end + + style request fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style enqueue fill:#1565c0,stroke:#0d47a1,color:#ffffff + style command fill:#e65100,stroke:#bf360c,color:#ffffff +``` + +**Reading the diagram:** + +- **rpc.request (green, root span)**: The outermost span representing the full RPC request lifecycle, from HTTP receipt to response. Carries the W3C `traceparent` header for distributed tracing. +- **HTTP Request node**: Shows the incoming POST request with its `traceparent` header and extracted attributes (method, peer IP, command name). +- **jobqueue.enqueue (blue)**: The span covering the asynchronous handoff from the RPC thread to the JobQueue worker thread. The trace context is preserved across this async boundary. +- **rpc.command.submit (orange)**: The span for the actual command execution, with child spans for deserialization, local validation, and network submission. +- **Response node**: The final output with HTTP status and total duration, marking the end of the root span. +- **Arrows (top to bottom)**: The sequential processing pipeline -- receive request, extract attributes, enqueue job, execute command, return response. + +--- + +## 1.6 Key Trace Points + +> **TxQ** = Transaction Queue + +The following table identifies priority instrumentation points across the codebase: + +| Category | Span Name | File | Method | Priority | +| --------------- | ---------------------- | ---------------------- | ----------------------- | -------- | +| **Transaction** | `tx.receive` | `PeerImp.cpp` | `handleTransaction()` | High | +| **Transaction** | `tx.validate` | `NetworkOPs.cpp` | `processTransaction()` | High | +| **Transaction** | `tx.process` | `NetworkOPs.cpp` | `doTransactionSync()` | High | +| **Transaction** | `tx.relay` | `OverlayImpl.cpp` | `relay()` | Medium | +| **Consensus** | `consensus.round` | `RCLConsensus.cpp` | `startRound()` | High | +| **Consensus** | `consensus.phase.*` | `Consensus.h` | `timerEntry()` | High | +| **Consensus** | `consensus.proposal.*` | `RCLConsensus.cpp` | `peerProposal()` | Medium | +| **RPC** | `rpc.request` | `ServerHandler.cpp` | `onRequest()` | High | +| **RPC** | `rpc.command.*` | `RPCHandler.cpp` | `doCommand()` | High | +| **Peer** | `peer.connect` | `OverlayImpl.cpp` | `onHandoff()` | Low | +| **Peer** | `peer.message.*` | `PeerImp.cpp` | `onMessage()` | Low | +| **Ledger** | `ledger.acquire` | `InboundLedgers.cpp` | `acquire()` | Medium | +| **Ledger** | `ledger.build` | `RCLConsensus.cpp` | `buildLCL()` | High | +| **PathFinding** | `pathfind.request` | `PathRequest.cpp` | `doUpdate()` | High | +| **PathFinding** | `pathfind.compute` | `Pathfinder.cpp` | `findPaths()` | High | +| **TxQ** | `txq.enqueue` | `TxQ.cpp` | `apply()` | High | +| **TxQ** | `txq.apply` | `TxQ.cpp` | `processClosedLedger()` | High | +| **Fee** | `fee.escalate` | `LoadManager.cpp` | `raiseLocalFee()` | Medium | +| **Ledger** | `ledger.replay` | `LedgerReplayer.h` | `replay()` | Medium | +| **Ledger** | `ledger.delta` | `LedgerDeltaAcquire.h` | `processData()` | Medium | +| **Validator** | `validator.list.fetch` | `ValidatorList.cpp` | `verify()` | Medium | +| **Validator** | `validator.manifest` | `Manifest.cpp` | `applyManifest()` | Low | +| **Amendment** | `amendment.vote` | `AmendmentTable.cpp` | `doVoting()` | Low | +| **SHAMap** | `shamap.sync` | `SHAMap.cpp` | `fetchRoot()` | Medium | + +--- + +## 1.7 Instrumentation Priority + +> **TxQ** = Transaction Queue + +```mermaid +quadrantChart + title Instrumentation Priority Matrix + x-axis Low Complexity --> High Complexity + y-axis Low Value --> High Value + quadrant-1 Implement First + quadrant-2 Plan Carefully + quadrant-3 Quick Wins + quadrant-4 Consider Later + + RPC Tracing: [0.2, 0.92] + Transaction Tracing: [0.55, 0.88] + Consensus Tracing: [0.78, 0.82] + PathFinding: [0.38, 0.75] + TxQ and Fees: [0.25, 0.65] + Ledger Sync: [0.62, 0.58] + Peer Message Tracing: [0.35, 0.25] + JobQueue Tracing: [0.2, 0.48] + Validator Mgmt: [0.48, 0.42] + Amendment Tracking: [0.15, 0.32] + SHAMap Operations: [0.72, 0.45] +``` + +--- + +## 1.8 Observable Outcomes + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List + +After implementing OpenTelemetry, operators and developers will gain visibility into the following: + +### 1.8.1 What You Will See: Traces + +| Trace Type | Description | Example Query in Grafana/Tempo | +| -------------------------- | ------------------------------------------------------------------------------------------- | ----------------------------------------------- | +| **Transaction Lifecycle** | Full journey from RPC submission through validation, relay, consensus, and ledger inclusion | `{service.name="xrpld" && tx_hash="ABC123..."}` | +| **Cross-Node Propagation** | Transaction path across multiple xrpld nodes with timing | `{relay_count > 0}` | +| **Consensus Rounds** | Complete round with all phases (open, establish, accept) | `{span.name=~"consensus.round.*"}` | +| **RPC Request Processing** | Individual command execution with timing breakdown | `{command="account_info"}` | +| **Ledger Acquisition** | Peer-to-peer ledger data requests and responses | `{span.name="ledger.acquire"}` | +| **PathFinding Latency** | Path computation time and cache effectiveness for payment RPCs | `{span.name="pathfind.compute"}` | +| **TxQ Behavior** | Queue depth, eviction patterns, fee escalation during congestion | `{span.name=~"txq.*"}` | +| **Ledger Sync** | Full acquisition timeline including delta and transaction fetches | `{span.name=~"ledger.acquire.*"}` | +| **Validator Health** | UNL fetch success, manifest updates, stale list detection | `{span.name=~"validator.*"}` | + +### 1.8.2 What You Will See: Metrics (Derived from Traces) + +| Metric | Description | Dashboard Panel | +| ----------------------------- | --------------------------------------- | --------------------------- | +| **RPC Latency (p50/p95/p99)** | Response time distribution per command | Heatmap by command | +| **Transaction Throughput** | Transactions processed per second | Time series graph | +| **Consensus Round Duration** | Time to complete consensus phases | Histogram | +| **Cross-Node Latency** | Time for transaction to reach N nodes | Line chart with percentiles | +| **Error Rate** | Failed transactions/RPC calls by type | Stacked bar chart | +| **PathFinding Latency** | Path computation time per currency pair | Heatmap by currency | +| **TxQ Depth** | Queued transactions over time | Time series with thresholds | +| **Fee Escalation Level** | Current fee multiplier | Gauge with alert thresholds | +| **Ledger Sync Duration** | Time to acquire missing ledgers | Histogram | + +### 1.8.3 Concrete Dashboard Examples + +**Transaction Trace View (Tempo):** + +``` +┌────────────────────────────────────────────────────────────────────────────────┐ +│ Trace: abc123... (Transaction Submission) Duration: 847ms │ +├────────────────────────────────────────────────────────────────────────────────┤ +│ ├── rpc.request [ServerHandler] ████░░░░░░ 45ms │ +│ │ └── rpc.command.submit [RPCHandler] ████░░░░░░ 42ms │ +│ │ └── tx.receive [NetworkOPs] ███░░░░░░░ 35ms │ +│ │ ├── tx.validate [TxQ] █░░░░░░░░░ 8ms │ +│ │ └── tx.relay [Overlay] ██░░░░░░░░ 15ms │ +│ │ ├── tx.receive [Node-B] █████░░░░░ 52ms │ +│ │ │ └── tx.relay [Node-B] ██░░░░░░░░ 18ms │ +│ │ └── tx.receive [Node-C] ██████░░░░ 65ms │ +│ └── consensus.round [RCLConsensus] ████████░░ 720ms │ +│ ├── consensus.phase.open ██░░░░░░░░ 180ms │ +│ ├── consensus.phase.establish █████░░░░░ 480ms │ +│ └── consensus.phase.accept █░░░░░░░░░ 60ms │ +└────────────────────────────────────────────────────────────────────────────────┘ +``` + +**RPC Performance Dashboard Panel:** + +``` +┌─────────────────────────────────────────────────────────────┐ +│ RPC Command Latency (Last 1 Hour) │ +├─────────────────────────────────────────────────────────────┤ +│ Command │ p50 │ p95 │ p99 │ Errors │ Rate │ +│──────────────────┼────────┼────────┼────────┼────────┼──────│ +│ account_info │ 12ms │ 45ms │ 89ms │ 0.1% │ 150/s│ +│ submit │ 35ms │ 120ms │ 250ms │ 2.3% │ 45/s│ +│ ledger │ 8ms │ 25ms │ 55ms │ 0.0% │ 80/s│ +│ tx │ 15ms │ 50ms │ 100ms │ 0.5% │ 60/s│ +│ server_info │ 5ms │ 12ms │ 20ms │ 0.0% │ 200/s│ +└─────────────────────────────────────────────────────────────┘ +``` + +**Consensus Health Dashboard Panel:** + +```mermaid +--- +config: + xyChart: + width: 1200 + height: 400 + plotReservedSpacePercent: 50 + chartOrientation: vertical + themeVariables: + xyChart: + plotColorPalette: "#3498db" +--- +xychart-beta + title "Consensus Round Duration (Last 24 Hours)" + x-axis "Time of Day (Hours)" [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24] + y-axis "Duration (seconds)" 1 --> 5 + line [2.1, 2.4, 2.8, 3.2, 3.8, 4.3, 4.5, 5.0, 4.7, 4.0, 3.2, 2.6, 2.0] +``` + +### 1.8.4 Operator Actionable Insights + +| Scenario | What You'll See | Action | +| ------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------ | +| **Slow RPC** | Span showing which phase is slow (parsing, execution, serialization) | Optimize specific code path | +| **Transaction Stuck** | Trace stops at validation; error attribute shows reason | Fix transaction parameters | +| **Consensus Delay** | Phase.establish taking too long; proposer attribute shows missing validators | Investigate network connectivity | +| **Memory Spike** | Large batch of spans correlating with memory increase | Tune batch_size or sampling | +| **Network Partition** | Traces missing cross-node links for specific peer | Check peer connectivity | +| **Path Computation Slow** | pathfind.compute span shows high latency; cache miss rate in attributes | Warm the RippleLineCache, check order book depth | +| **TxQ Full** | txq.enqueue spans show evictions; fee.escalate spans increasing | Monitor fee levels, alert operators | +| **Ledger Sync Stalled** | ledger.acquire spans timing out; peer reliability attributes show issues | Check peer connectivity, add trusted peers | +| **UNL Stale** | validator.list.fetch spans failing; last_update attribute aging | Verify validator site URLs, check DNS | + +### 1.8.5 Developer Debugging Workflow + +1. **Find Transaction**: Query by `tx_hash` to get full trace +2. **Identify Bottleneck**: Look at span durations to find slowest component +3. **Check Attributes**: Review `validity`, `rpc_status` for errors +4. **Correlate Logs**: Use `trace_id` to find related PerfLog entries +5. **Compare Nodes**: Filter by `service.instance.id` to compare behavior across nodes + +--- + +_Next: [Design Decisions](./02-design-decisions.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md new file mode 100644 index 00000000000..9bfee122c68 --- /dev/null +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -0,0 +1,686 @@ +# Design Decisions + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Architecture Analysis](./01-architecture-analysis.md) + +--- + +## 2.1 OpenTelemetry Components + +> **OTLP** = OpenTelemetry Protocol + +### 2.1.1 SDK Selection + +**Primary Choice**: OpenTelemetry C++ SDK (`opentelemetry-cpp`) + +| Component | Purpose | Required | +| --------------------------------------- | ---------------------- | ------------------------- | +| `opentelemetry-cpp::api` | Tracing API headers | Yes | +| `opentelemetry-cpp::sdk` | SDK implementation | Yes | +| `opentelemetry-cpp::ext` | Extensions (exporters) | Yes | +| `opentelemetry-cpp::otlp_http_exporter` | OTLP/HTTP export | Yes (shipped in Phase 1b) | +| `opentelemetry-cpp::otlp_grpc_exporter` | OTLP/gRPC export | Future (not yet wired up) | + +### 2.1.2 Instrumentation Strategy + +**Manual Instrumentation** (recommended): + +| Approach | Pros | Cons | +| ---------- | --------------------------------------------------------------- | ------------------------------------------------------- | +| **Manual** | Precise control, optimized placement, xrpld-specific attributes | More development effort | +| **Auto** | Less code, automatic coverage | Less control, potential overhead, limited customization | + +--- + +## 2.2 Exporter Configuration + +> **OTLP** = OpenTelemetry Protocol + +```mermaid +flowchart TB + subgraph nodes["xrpld Nodes"] + node1["xrpld
Node 1"] + node2["xrpld
Node 2"] + node3["xrpld
Node 3"] + end + + collector["OpenTelemetry
Collector
(sidecar or standalone)"] + + subgraph backends["Observability Backends"] + tempo["Tempo"] + elastic["Elastic
APM"] + end + + node1 -->|"OTLP/HTTP
:4318"| collector + node2 -->|"OTLP/HTTP
:4318"| collector + node3 -->|"OTLP/HTTP
:4318"| collector + + collector --> tempo + collector --> elastic + + style nodes fill:#0d47a1,stroke:#082f6a,color:#ffffff + style backends fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style collector fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **xrpld Nodes (blue)**: The source of telemetry data. Each xrpld node exports spans via OTLP/HTTP on port 4318 (the only exporter shipped in Phase 1b). +- **OpenTelemetry Collector (red)**: The central aggregation point that receives spans from all nodes. Can run as a sidecar (per-node) or standalone (shared). Handles batching, filtering, and routing. +- **Observability Backends (green)**: The storage and visualization destinations. Tempo is the recommended backend for both development and production, and Elastic APM is an alternative. The Collector routes to one or more backends. +- **Arrows (nodes to collector to backends)**: The data pipeline -- spans flow from nodes to the Collector over HTTP, then the Collector fans out to the configured backends. + +### 2.2.1 OTLP/HTTP (Shipped in Phase 1b) + +OTLP/HTTP is the only exporter wired up in Phase 1b. It is configured via +`OtlpHttpExporterOptions` with the collector traces endpoint +(`http://localhost:4318/v1/traces` by default) and a JSON content type +(binary protobuf is also available). + +### 2.2.2 OTLP/gRPC (Future Work — Planned Upgrade) + +OTLP/gRPC is planned as a future upgrade from the HTTP exporter. The gRPC +transport offers lower per-span overhead and tighter back-pressure semantics +than HTTP/JSON, making it attractive for production deployments once the HTTP +path is validated in earlier phases. + +Required to land this upgrade: + +1. Add `opentelemetry-cpp::otlp_grpc_exporter` to the Conan recipe (the + dependency already exists but is not linked in Phase 1b builds). +2. Extend `TelemetryConfig.cpp` to parse an `exporter` key (`otlp_http` + default, `otlp_grpc` opt-in) and a gRPC endpoint override. +3. In `Telemetry::start()` branch on the parsed exporter type and construct + either `OtlpHttpExporterFactory::Create(httpOpts)` or + `OtlpGrpcExporterFactory::Create(grpcOpts)` accordingly. +4. Update the runbook and dashboards to document the alternate port and TLS + settings. + +When wired up, the gRPC path will use `OtlpGrpcExporterOptions` configured with +the collector endpoint (host on port 4317), TLS credentials enabled, and a CA +certificate path. + +Until that work lands, `OtlpGrpcExporterOptions` is **not** used by any code +path in Phase 1b through Phase 5. + +--- + +## 2.3 Span Naming Conventions + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List | **WS** = WebSocket + +### 2.3.1 Naming Schema + +``` +.[.] +``` + +**Examples**: + +- `tx.receive` - Transaction received from peer +- `consensus.phase.establish` - Consensus establish phase +- `rpc.command.server_info` - server_info RPC command + +### 2.3.2 Complete Span Catalog + +| Span name | Description | +| ------------------------------ | --------------------------------------- | +| `tx.receive` | Transaction received from network | +| `tx.validate` | Transaction signature/format validation | +| `tx.process` | Full transaction processing | +| `tx.relay` | Transaction relay to peers | +| `tx.apply` | Apply transaction to ledger | +| `consensus.round` | Complete consensus round | +| `consensus.phase.open` | Open phase - collecting transactions | +| `consensus.phase.establish` | Establish phase - reaching agreement | +| `consensus.phase.accept` | Accept phase - applying consensus | +| `consensus.proposal.receive` | Receive peer proposal | +| `consensus.proposal.send` | Send our proposal | +| `consensus.validation.receive` | Receive peer validation | +| `consensus.validation.send` | Send our validation | +| `rpc.request` | HTTP/WebSocket request handling | +| `rpc.command.*` | Specific RPC command (dynamic) | +| `peer.connect` | Peer connection establishment | +| `peer.disconnect` | Peer disconnection | +| `peer.message.send` | Send protocol message | +| `peer.message.receive` | Receive protocol message | +| `ledger.acquire` | Ledger acquisition from network | +| `ledger.build` | Build new ledger | +| `ledger.validate` | Ledger validation | +| `ledger.close` | Close ledger | +| `ledger.replay` | Ledger replay executed | +| `ledger.delta` | Delta-based ledger acquired | +| `pathfind.request` | Path request initiated | +| `pathfind.compute` | Path computation executed | +| `txq.enqueue` | Transaction queued | +| `txq.apply` | Queued transaction applied | +| `fee.escalate` | Fee escalation triggered | +| `validator.list.fetch` | UNL list fetched | +| `validator.manifest` | Manifest update processed | +| `amendment.vote` | Amendment voting executed | +| `shamap.sync` | State tree synchronization | +| `job.enqueue` | Job added to queue | +| `job.execute` | Job execution | + +### 2.3.3 Attribute Naming Conventions + +Span **names** follow §2.3.1 (dotted `.`). Span +**attribute keys** follow the rules below. The constants in the `*SpanNames.h` +headers are the single source of truth; the collector, Tempo, the Grafana +dashboards, and the runbook all consume these exact keys, so every layer must +agree with the code. A CI check enforces this end to end. + +1. **Per-span unique attribute** → bare field name, allowed when the field is + recorded by a single span/workflow so the span name already supplies the + domain (e.g. `command`, `version`, `local` on `rpc.command`). +2. **Shared attribute (same concept on more than one span)** → ONE key, reused + verbatim on every span that records it; the span name tells the occurrences + apart, so no per-emitter prefix is added. Name it by the field's meaning: a + property of a domain object keeps that object's bare field name (`ledger_hash`, + `ledger_seq`, `tx_hash`, `peer_id`, `full_validation`); a field already + qualified by a sub-kind keeps that qualifier on every emitter (`proposal_trusted` + on both `consensus.proposal.receive` and `peer.proposal.receive`; + `validation_trusted` likewise). Defined once in the base `SpanNames.h` + `namespace attr` block and re-exported (`using`) by each domain header. +3. **Collision qualifier** → `_`, only when a bare name would + collide with a DIFFERENT concept in the shared spanmetrics label space or with + the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, + `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates + distinct concepts that share a word; it is NOT used to tag the same concept + with its emitting workflow — that is rule 2 (one shared name). +4. **Resource attribute** → dotted `xrpl..`, reserved ONLY + for process/network identity set once at startup (`xrpl.network.id`, + `xrpl.network.type`). Span attributes are never dotted in the `xrpl.` form — + it blurs the resource/span scope boundary and parses awkwardly in TraceQL. +5. **Span names** use `[.]` (dotted, per §2.3.1). Only + attribute _keys_ follow rules 1–4. + +Standard OpenTelemetry semantic-convention keys keep their canonical dotted +form (e.g. `service.*` resource attributes, `http.*` span attributes); the +"no dotted form" rule applies to xrpl-custom keys only. + +The same rules are recorded in `CONTRIBUTING.md` (the permanent home, since +`OpenTelemetryPlan/` is removed once the rollout completes). The attribute +examples in §2.4 below follow these rules. + +--- + +## 2.4 Attribute Schema + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List | **OTLP** = OpenTelemetry Protocol + +### 2.4.1 Resource Attributes (Set Once at Startup) + +Resource attributes identify the process and are set once at startup. They use +the standard OpenTelemetry semantic conventions plus custom dotted `xrpl.*` +keys (the dotted form is reserved for resource scope per §2.3.3). + +| Key | Type / value | Description | +| --------------------- | ------------------------------------------------------- | ------------------------------ | +| `service.name` | `"xrpld"` | Standard `SERVICE_NAME` | +| `service.version` | `BuildInfo::getVersionString()` | Standard `SERVICE_VERSION` | +| `service.instance.id` | node public key (base58) | Standard `SERVICE_INSTANCE_ID` | +| `xrpl.network.id` | network id (e.g. 0 for mainnet) | Network identifier | +| `xrpl.network.type` | `"mainnet"` \| `"testnet"` \| `"devnet"` \| `"unknown"` | Network kind | +| `xrpl.node.type` | `"validator"` \| `"stock"` \| `"reporting"` | Node role | +| `xrpl.node.cluster` | cluster name | Cluster name, if clustered | + +### 2.4.2 Span Attributes by Category + +> Span attribute keys use the underscore form from §2.3.3 (shared/qualified +> keys are `_`; per-span unique keys are bare). The dotted form +> is reserved for the resource attributes in §2.4.1 above. This catalog lists +> the planned attribute set by category; the exact emitted key for each +> implemented span is defined by the `*SpanNames.h` constants, which are the +> single source of truth where the two differ. + +#### Transaction Attributes + +| Key | Type | Description | +| -------------- | ------ | ------------------------------------- | +| `tx_hash` | string | Transaction hash (hex) | +| `tx_type` | string | `"Payment"`, `"OfferCreate"`, etc. | +| `tx_account` | string | Source account (redacted in prod) | +| `tx_sequence` | int64 | Account sequence number | +| `tx_fee` | int64 | Fee in drops | +| `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. | +| `ledger_index` | int64 | Ledger containing transaction | +| `relay_count` | int64 | Peers the transaction was relayed to | +| `suppressed` | bool | `true` when HashRouter dropped a dup | + +#### Consensus Attributes + +| Key | Type | Description | +| -------------------- | ------- | ----------------------------------- | +| `consensus_round` | int64 | Round number | +| `consensus_phase` | string | `"open"`, `"establish"`, `"accept"` | +| `consensus_mode` | string | `"proposing"`, `"observing"`, etc. | +| `proposers` | int64 | Number of proposers | +| `prev_ledger_prefix` | string | Previous ledger hash prefix | +| `ledger_seq` | int64 | Ledger sequence | +| `tx_count` | int64 | Transactions in consensus set | +| `round_time_ms` | float64 | Round duration | + +Establish-phase gap fill and cross-node correlation attributes (Phase 4a): + +| Key | Type | Description | +| --------------------- | ------ | --------------------------------------------------------- | +| `consensus_round_id` | int64 | Consensus round number | +| `consensus_ledger_id` | string | `previousLedger.id()` — shared across nodes | +| `trace_strategy` | string | `"deterministic"` or `"attribute"` | +| `converge_percent` | int64 | Convergence % (0-100+) | +| `establish_count` | int64 | Number of establish iterations | +| `disputes_count` | int64 | Active disputed transactions | +| `agree_count` | int64 | Peers that agree (haveConsensus) | +| `disagree_count` | int64 | Peers that disagree | +| `threshold_percent` | int64 | Close-time consensus threshold (`avCT_CONSENSUS_PCT`=75%) | +| `consensus_result` | string | `"yes"`, `"no"`, `"moved_on"`, `"expired"` | +| `mode_old` | string | Previous consensus mode | +| `mode_new` | string | New consensus mode | + +#### RPC Attributes + +| Key | Type | Description | +| ------------- | ------- | ----------------------------------------------------------------------------- | +| `command` | string | Command name (per-span unique on `rpc.command`) | +| `version` | int64 | API version | +| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | +| `params` | string | Sanitized parameters (optional) | +| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) | +| `duration_ms` | float64 | Request duration in milliseconds | + +#### Peer & Message Attributes + +| Key | Type | Description | +| -------------------- | ------- | -------------------------- | +| `peer_id` | string | Peer public key (base58) | +| `peer_address` | string | IP:port | +| `peer_latency_ms` | float64 | Measured latency | +| `peer_cluster` | string | Cluster name if clustered | +| `message_type` | string | Protocol message type name | +| `message_size_bytes` | int64 | Message size | +| `message_compressed` | bool | Whether compressed | + +#### Ledger & Job Attributes + +| Key | Type | Description | +| ----------------- | ------- | --------------------- | +| `ledger_hash` | string | Ledger hash | +| `ledger_index` | int64 | Ledger sequence/index | +| `close_time` | int64 | Close time (epoch) | +| `ledger_tx_count` | int64 | Transaction count | +| `job_type` | string | Job type name | +| `job_queue_ms` | float64 | Time spent in queue | +| `job_worker` | int64 | Worker thread ID | + +#### PathFinding Attributes + +| Key | Type | Description | +| -------------------------- | ------ | ------------------------- | +| `pathfind_source_currency` | string | Source currency code | +| `pathfind_dest_currency` | string | Destination currency code | +| `pathfind_path_count` | int64 | Number of paths found | +| `pathfind_cache_hit` | bool | RippleLineCache hit | + +#### TxQ Attributes + +| Key | Type | Description | +| --------------------- | ------ | --------------------------- | +| `txq_queue_depth` | int64 | Current queue depth | +| `txq_fee_level` | int64 | Fee level of transaction | +| `txq_eviction_reason` | string | Why transaction was evicted | + +#### Fee Attributes + +| Key | Type | Description | +| ---------------------- | ----- | ------------------------- | +| `fee_load_factor` | int64 | Current load factor | +| `fee_escalation_level` | int64 | Fee escalation multiplier | + +#### Validator Attributes + +| Key | Type | Description | +| ------------------------ | ----- | ------------------------- | +| `validator_list_size` | int64 | UNL size | +| `validator_list_age_sec` | int64 | Seconds since last update | + +#### Amendment Attributes + +| Key | Type | Description | +| ------------------ | ------ | -------------------------------------- | +| `amendment_name` | string | Amendment name | +| `amendment_status` | string | `"enabled"`, `"vetoed"`, `"supported"` | + +#### SHAMap Attributes + +| Key | Type | Description | +| ---------------------- | ------- | --------------------------------------------- | +| `shamap_type` | string | `"transaction"`, `"state"`, `"account_state"` | +| `shamap_missing_nodes` | int64 | Number of missing nodes during sync | +| `shamap_duration_ms` | float64 | Sync duration | + +### 2.4.3 Data Collection Summary + +The following table summarizes what data is collected by category: + +| Category | Attributes Collected | Purpose | +| --------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle | +| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers`, `round_time_ms` | Analyze consensus timing | +| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance | +| **Peer** | `peer_id` (public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis | +| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking | +| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance | +| **PathFinding** | `pathfind_fast`, `pathfind_search_level`, `pathfind_num_paths`, `pathfind_ledger_index`, `pathfind_num_requests` | Payment path analysis | +| **TxQ** | `txq_queue_depth`, `txq_fee_level`, `txq_eviction_reason` | Queue depth and fee tracking | +| **Fee** | `fee_load_factor`, `fee_escalation_level` | Fee escalation monitoring | +| **Validator** | `validator_list_size`, `validator_list_age_sec` | UNL health monitoring | +| **Amendment** | `amendment_name`, `amendment_status` | Protocol upgrade tracking | +| **SHAMap** | `shamap_type`, `shamap_missing_nodes`, `shamap_duration_ms` | State tree sync performance | + +### 2.4.4 Privacy & Sensitive Data Policy + +> **PII** = Personally Identifiable Information + +OpenTelemetry instrumentation is designed to collect **operational metadata only**, never sensitive content. + +#### Data NOT Collected + +The following data is explicitly **excluded** from telemetry collection: + +| Excluded Data | Reason | +| ----------------------- | ----------------------------------------- | +| **Private Keys** | Never exposed; not relevant to tracing | +| **Account Balances** | Financial data; privacy sensitive | +| **Transaction Amounts** | Financial data; privacy sensitive | +| **Raw TX Payloads** | May contain sensitive memo/data fields | +| **Personal Data** | No PII collected | +| **IP Addresses** | Configurable; excluded by default in prod | + +#### Privacy Protection Mechanisms + +| Mechanism | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | +| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | +| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | +| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | + +#### Account Address Hashing + +Account addresses are **always** hashed before they reach the telemetry +backend — there is no opt-out flag and therefore no insecure-by-default +failure mode. Protection is applied in two independent layers: + +1. **SDK-side** (this node): the path-finding RPC handlers call + `redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the + `pathfind_source_account` / `pathfind_dest_account` span attributes. The + helper emits the first 16 characters of `sha512Half(address)` as + lowercase hex — deterministic (spans for one account still correlate) + but non-reversible. +2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in + the OpenTelemetry Collector re-hashes those same attributes, so any node + that emitted a raw value is still redacted before storage. + +#### Collector-Level Data Protection + +The OpenTelemetry Collector can be configured (via an `attributes` processor) +to hash or redact sensitive attributes before export — for example, hashing +`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address` +to drop IP addresses, and deleting `params` to redact request parameters. + +#### Configuration Options for Privacy + +In `xrpld.cfg`, operators control data collection granularity through the +`[telemetry]` section. Besides `enabled`, per-component toggles +(`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last +often disabled due to high volume) select which spans are emitted. Account +address hashing is not configurable: addresses are hashed unconditionally by +the SDK helper described above, with collector-level hashing as a second +layer. + +> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts, raw payloads). + +> **See also**: [Securing the OTel Pipeline](./secure-OTel.md) covers transport-level protection for telemetry leaving the node — mTLS to the collector and validation of incoming peer trace context. Privacy controls in this section keep sensitive data out of spans; the security doc keeps the spans themselves out of untrusted hands. + +--- + +## 2.5 Context Propagation Design + +> **WS** = WebSocket + +### 2.5.0 Deterministic Trace ID Strategy + +Both transaction and consensus tracing use **deterministic trace IDs** derived from +a globally known hash, so all nodes handling the same workflow independently produce +spans under the same `trace_id`. This is combined with protobuf `span_id` propagation +for parent-child relay ordering when available. + +#### Transactions — `trace_id = txHash[0:16]` + +Every node that handles a transaction knows its `txID` (the `uint256` transaction +hash). The first 16 bytes of this hash are used as the OTel `trace_id`: + +``` +uint256 txHash: A1B2C3D4 E5F6A7B8 C9D0E1F2 A3B4C5D6 E7F8A9B0 C1D2E3F4 A5B6C7D8 E9F0A1B2 + |---------- trace_id (16 bytes) ---------| (remaining 16 bytes unused) +``` + +Each node generates a **random 8-byte `span_id`** so its span is unique within the +shared trace. When protobuf `TraceContext` is present in the incoming `TMTransaction`, +the sender's `span_id` is extracted and used as the parent — preserving the relay +chain as a parent-child tree. When absent (older peers, first hop from client), the +span appears as a root in the same trace — correlation is preserved, only the tree +structure degrades. + +``` +Node A (submitter) Node B (relay) Node C (relay) +trace_id: A1B2... trace_id: A1B2... trace_id: A1B2... +span_id: 1234 (random) span_id: 5678 (random) span_id: 9ABC (random) +parent: (none) parent: 1234 (proto) parent: 5678 (proto) + ↑ ↑ + protobuf propagation protobuf propagation +``` + +If protobuf propagation fails at Node B (old peer): + +``` +Node A Node B (old peer) Node C +trace_id: A1B2... trace_id: A1B2... trace_id: A1B2... +span_id: 1234 span_id: 5678 span_id: 9ABC +parent: (none) parent: (none) parent: 5678 (proto) + ↑ no parent, but same trace_id — still grouped +``` + +#### Consensus — `trace_id = prevLedgerHash[0:16]` + +All validators in the same consensus round share the same `previousLedger.id()`. +The first 16 bytes are used as trace_id. See [Phase 4a implementation status](./06-implementation-phases.md) +and `createDeterministicContext()` in `RCLConsensus.cpp` for the implementation. + +Switchable via `consensus_trace_strategy` config: +`"deterministic"` (default) or `"attribute"` (random trace_id, correlation via attribute queries). + +#### Why Not Random IDs with Propagation Only? + +Random trace IDs require **unbroken context propagation** across every hop. In a +mixed-version network (common during upgrades), older peers silently drop the +`trace_context` protobuf field. The trace splits and downstream spans become +impossible to find. Deterministic IDs make correlation **propagation-resilient** — the trace +backend groups all spans for the same transaction/round regardless of whether +propagation succeeded. + +#### Why Keep Protobuf Propagation? + +Deterministic trace IDs alone provide correlation (all spans grouped) but not +**causality** (which node relayed to which). Protobuf `span_id` propagation adds +parent-child ordering that shows the exact relay path. The two mechanisms complement +each other: + +| Mechanism | Provides | Fails when | +| ---------------------------- | --------------------------- | -------------------------------------- | +| Deterministic trace_id | Cross-node correlation | Never (hash is always known) | +| Protobuf span_id propagation | Parent-child relay ordering | Older peer drops `trace_context` field | + +#### Implementation Reference + +The utility function `createDeterministicTxContext(uint256 const& txHash)` follows +the same pattern as `createDeterministicContext(uint256 const& ledgerId)` in +`RCLConsensus.cpp`. See [Phase 3 Task 3.9](./Phase3_taskList.md) for the full spec. + +### 2.5.1 Propagation Boundaries + +```mermaid +flowchart TB + subgraph http["HTTP/WebSocket (RPC)"] + w3c["W3C Trace Context Headers:
traceparent:
00-trace_id-span_id-flags
tracestate: xrpld=..."] + end + + subgraph protobuf["Protocol Buffers (P2P)"] + proto["message TraceContext {
bytes trace_id = 1; // 16 bytes
bytes span_id = 2; // 8 bytes
uint32 trace_flags = 3;
string trace_state = 4;
}"] + end + + subgraph jobqueue["JobQueue (Internal Async)"] + job["Context captured at job creation,
restored at execution

class Job {
otel::context::Context
traceContext_;
};"] + end + + style http fill:#0d47a1,stroke:#082f6a,color:#ffffff + style protobuf fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style jobqueue fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **HTTP/WebSocket - RPC (blue)**: For client-facing RPC requests, trace context is propagated using the W3C `traceparent` header. This is the standard approach and works with any OTel-compatible client. +- **Protocol Buffers - P2P (green)**: For peer-to-peer messages between xrpld nodes, trace context is embedded as a protobuf `TraceContext` message carrying trace_id, span_id, flags, and optional trace_state. +- **JobQueue - Internal Async (red)**: For asynchronous work within a single node, the OTel context is captured when a job is created and restored when the job executes on a worker thread. This bridges the async gap so spans remain linked. + +--- + +## 2.6 Integration with Existing Observability + +> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket + +### 2.6.1 Existing Frameworks Comparison + +xrpld already has two observability mechanisms. OpenTelemetry complements (not replaces) them: + +| Aspect | PerfLog | Beast Insight (StatsD) | OpenTelemetry | +| --------------------- | ----------------------------- | ---------------------------- | ------------------------- | +| **Type** | Logging | Metrics | Distributed Tracing | +| **Data** | JSON log entries | Counters, gauges, histograms | Spans with context | +| **Scope** | Single node | Single node | **Cross-node** | +| **Output** | `perf.log` file | StatsD server | OTLP Collector | +| **Question answered** | "What happened on this node?" | "How many? How fast?" | "What was the journey?" | +| **Correlation** | By timestamp | By metric name | By `trace_id` | +| **Overhead** | Low (file I/O) | Low (UDP packets) | Low-Medium (configurable) | + +### 2.6.2 What Each Framework Does Best + +#### PerfLog + +- **Purpose**: Detailed local event logging for RPC and job execution +- **Strengths**: + - Rich JSON output with timing data + - Already integrated in RPC handlers + - File-based, no external dependencies +- **Limitations**: + - Single-node only (no cross-node correlation) + - No parent-child relationships between events + - Manual log parsing required + +A PerfLog entry is a JSON object with fields such as `time`, `method`, +`duration_us`, and `result`. + +#### Beast Insight (StatsD) + +- **Purpose**: Real-time metrics for monitoring dashboards +- **Strengths**: + - Aggregated metrics (counters, gauges, histograms) + - Low overhead (UDP, fire-and-forget) + - Good for alerting thresholds +- **Limitations**: + - No request-level detail + - No causal relationships + - Single-node perspective + +In xrpld, Beast Insight is used through `increment` (counters), `gauge` +(point-in-time values), and `timing` (durations) calls. + +#### OpenTelemetry (NEW) + +- **Purpose**: Distributed request tracing across nodes +- **Strengths**: + - **Cross-node correlation** via `trace_id` + - Parent-child span relationships + - Rich attributes per span + - Industry standard (CNCF) +- **Limitations**: + - Requires collector infrastructure + - Higher complexity than logging + +A span is created via `startSpan` (e.g. `"tx.relay"`), annotated with +attributes such as `tx_hash` and `peer_id`, and is automatically linked to its +parent through the active context. + +### 2.6.3 When to Use Each + +| Scenario | PerfLog | StatsD | OpenTelemetry | +| --------------------------------------- | ---------- | ------ | ------------- | +| "How many TXs per second?" | ❌ | ✅ | ✅ | +| "What's the p99 RPC latency?" | ❌ | ✅ | ✅ | +| "Why was this specific TX slow?" | ⚠️ partial | ❌ | ✅ | +| "Which node delayed consensus?" | ❌ | ❌ | ✅ | +| "What happened on node X at time T?" | ✅ | ❌ | ✅ | +| "Show me the TX journey across 5 nodes" | ❌ | ❌ | ✅ | + +### 2.6.4 Coexistence Strategy + +> **Note**: Phase 7 replaces the StatsD bridge with native OTel Metrics SDK export. The diagram below shows the Phase 6 intermediate state. See [Phase7_taskList.md](./Phase7_taskList.md) for the migration design where Beast Insight emits via OTLP instead of StatsD. + +```mermaid +flowchart TB + subgraph xrpld["xrpld Process"] + perflog["PerfLog
(JSON to file)"] + insight["Beast Insight
(StatsD)"] + otel["OpenTelemetry
(Tracing)"] + end + + perflog --> perffile["perf.log"] + insight --> statsd["StatsD Server"] + otel --> collector["OTLP Collector"] + + perffile --> grafana["Grafana
(Unified UI)"] + statsd --> grafana + collector --> grafana + + style xrpld fill:#212121,stroke:#0a0a0a,color:#ffffff + style grafana fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference. +- **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource. +- **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter). +- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) +- **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. + +**Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details. + +### 2.6.5 Correlation with PerfLog + +Trace IDs can be correlated with existing PerfLog entries for comprehensive +debugging. The design is for `RPCHandler.cpp` to start an `rpc.command.` +span alongside the existing PerfLog `rpcStart`/`rpcFinish`/`rpcError` calls, +extract the span's `trace_id` (when valid), and eventually stamp it onto the +PerfLog entry (a planned `setTraceId` hook) so logs and traces share a key. The +span status is set to OK on success or to error (recording the exception) on +failure. + +--- + +_Previous: [Architecture Analysis](./01-architecture-analysis.md)_ | _Next: [Implementation Strategy](./03-implementation-strategy.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md new file mode 100644 index 00000000000..922d3d4999a --- /dev/null +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -0,0 +1,483 @@ +# Implementation Strategy + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Configuration Reference](./05-configuration-reference.md) + +--- + +## 3.1 Directory Structure + +The telemetry implementation follows xrpld's existing code organization pattern: + +``` +include/xrpl/ +├── telemetry/ +│ ├── Telemetry.h # Main telemetry interface (global singleton) +│ ├── TelemetryConfig.h # Configuration structures +│ ├── TraceContext.h # Context propagation utilities +│ ├── SpanGuard.h # RAII span management with factory methods + discard() +│ ├── DiscardFlag.h # Thread-local discard flag +│ └── SpanAttributes.h # Attribute helper functions + +src/libxrpl/ +├── telemetry/ +│ ├── Telemetry.cpp # Implementation + FilteringSpanProcessor +│ ├── TelemetryConfig.cpp # Config parsing +│ ├── TraceContext.cpp # Context serialization +│ └── NullTelemetry.cpp # No-op implementation +``` + +--- + +## 3.2 Implementation Approach + +
+ +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 20, 'rankSpacing': 30}}}%% +flowchart TB + subgraph phase1["Phase 1: Core"] + direction LR + sdk["SDK Integration"] ~~~ interface["Telemetry Interface"] ~~~ config["Configuration"] + end + + subgraph phase2["Phase 2: RPC"] + direction LR + http["HTTP Context"] ~~~ rpc["RPC Handlers"] + end + + subgraph phase3["Phase 3: P2P"] + direction LR + proto["Protobuf Context"] ~~~ tx["Transaction Relay"] + end + + subgraph phase4["Phase 4: Consensus"] + direction LR + consensus["Consensus Rounds"] ~~~ proposals["Proposals"] + end + + phase1 --> phase2 --> phase3 --> phase4 + + style phase1 fill:#1565c0,stroke:#0d47a1,color:#ffffff + style phase2 fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style phase3 fill:#e65100,stroke:#bf360c,color:#ffffff + style phase4 fill:#c2185b,stroke:#880e4f,color:#ffffff +``` + +
+ +### Key Principles + +1. **Minimal Intrusion**: Instrumentation should not alter existing control flow +2. **Zero-Cost When Disabled**: Use compile-time flags and no-op implementations +3. **Backward Compatibility**: Protocol Buffer extensions use high field numbers +4. **Graceful Degradation**: Tracing failures must not affect node operation + +--- + +## 3.3 Performance Overhead Summary + +> **OTLP** = OpenTelemetry Protocol + +| Metric | Overhead | Notes | +| ------------- | ---------- | ------------------------------------------------ | +| CPU | 1-3% | Of per-transaction CPU cost (~200μs baseline) | +| Memory | ~10 MB | SDK statics + batch buffer + worker thread stack | +| Network | 10-50 KB/s | Compressed OTLP export to collector | +| Latency (p99) | <2% | With proper sampling configuration | + +--- + +## 3.4 Detailed CPU Overhead Analysis + +### 3.4.1 Per-Operation Costs + +> **Note on hardware assumptions**: The costs below are based on the official OTel C++ SDK CI benchmarks +> (969 runs on GitHub Actions 2-core shared runners). On production server hardware (3+ GHz Xeon), +> expect costs at the **lower end** of each range (~30-50% improvement over CI hardware). + +| Operation | Time (ns) | Frequency | Impact | +| --------------------- | --------- | ---------------------- | ---------- | +| Span creation | 500-1000 | Every traced operation | Low | +| Span end | 100-200 | Every traced operation | Low | +| SetAttribute (string) | 80-120 | 3-5 per span | Low | +| SetAttribute (int) | 40-60 | 2-3 per span | Negligible | +| AddEvent | 100-200 | 0-2 per span | Low | +| Context injection | 150-250 | Per outgoing message | Low | +| Context extraction | 100-180 | Per incoming message | Low | +| GetCurrent context | 10-20 | Thread-local access | Negligible | + +**Source**: Span creation based on OTel C++ SDK `BM_SpanCreation` benchmark (AlwaysOnSampler + +SimpleSpanProcessor + InMemoryExporter), median ~1,000 ns on CI hardware. AddEvent includes +timestamp read + string copy + vector push + mutex acquisition. Context injection/extraction +confirmed by `BM_SpanCreationWithScope` benchmark delta (~160 ns). + +### 3.4.2 Transaction Processing Overhead + +
+ +```mermaid +%%{init: {'pie': {'textPosition': 0.75}}}%% +pie showData + "tx.receive (1400ns)" : 1400 + "tx.validate (1200ns)" : 1200 + "tx.relay (1200ns)" : 1200 + "Context inject (200ns)" : 200 +``` + +**Transaction Tracing Overhead (~4.0μs total)** + +
+ +**Overhead percentage**: 4.0 μs / 200 μs (avg tx processing) = **~2.0%** + +> **Breakdown**: Each span (tx.receive, tx.validate, tx.relay) costs ~1,000 ns for creation plus +> ~200-400 ns for 3-5 attribute sets. Context injection is ~200 ns (confirmed by benchmarks). +> On production hardware, expect ~2.6 μs total (~1.3% overhead) due to faster span creation (~500-600 ns). + +### 3.4.3 Consensus Round Overhead + +| Operation | Count | Cost (ns) | Total | +| ---------------------- | ----- | --------- | ---------- | +| consensus.round span | 1 | ~1200 | ~1.2 μs | +| consensus.phase spans | 3 | ~1100 | ~3.3 μs | +| proposal.receive spans | ~20 | ~1100 | ~22 μs | +| proposal.send spans | ~3 | ~1100 | ~3.3 μs | +| Context operations | ~30 | ~200 | ~6 μs | +| **TOTAL** | | | **~36 μs** | + +> **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for 1-2 attributes, totaling ~1,100-1,200 ns. +> Context operations remain ~200 ns (confirmed by benchmarks). On production hardware, expect ~24 μs total. + +**Overhead percentage**: 36 μs / 3s (typical round) = **~0.001%** (negligible) + +### 3.4.4 RPC Request Overhead + +| Operation | Cost (ns) | +| ---------------- | ------------ | +| rpc.request span | ~1200 | +| rpc.command span | ~1100 | +| Context extract | ~250 | +| Context inject | ~200 | +| **TOTAL** | **~2.75 μs** | + +> **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for attributes (command name, +> version, role). Context extract/inject costs are confirmed by OTel C++ benchmarks. + +- Fast RPC (1ms): 2.75 μs / 1ms = **~0.275%** +- Slow RPC (100ms): 2.75 μs / 100ms = **~0.003%** + +--- + +## 3.5 Memory Overhead Analysis + +> **OTLP** = OpenTelemetry Protocol + +### 3.5.1 Static Memory + +| Component | Size | Allocated | +| ------------------------------------ | ----------- | ---------- | +| TracerProvider singleton | ~64 KB | At startup | +| BatchSpanProcessor (circular buffer) | ~16 KB | At startup | +| BatchSpanProcessor (worker thread) | ~8 MB | At startup | +| OTLP/HTTP exporter (client init) | ~64 KB | At startup | +| Propagator registry | ~8 KB | At startup | +| **Total static** | **~8.1 MB** | | + +> **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB +> (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default +> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS +> initialization buffer. The worker thread stack dominates the static footprint. + +### 3.5.2 Dynamic Memory + +| Component | Size per unit | Max units | Peak | +| -------------------- | -------------- | ---------- | --------------- | +| Active span | ~500-800 bytes | 1000 | ~500-800 KB | +| Queued span (export) | ~500 bytes | 2048 | ~1 MB | +| Attribute storage | ~80 bytes | 5 per span | Included | +| Context storage | ~64 bytes | Per thread | ~6.4 KB | +| **Total dynamic** | | | **~1.5-1.8 MB** | + +> **Why active spans are larger**: An active `Span` object includes the wrapper (~88 bytes: shared_ptr, +> mutex, unique_ptr to Recordable) plus `SpanData` (~250 bytes: SpanContext, timestamps, name, status, +> empty containers) plus attribute storage (~200-500 bytes for 3-5 string attributes in a `std::map`). +> Source: `sdk/src/trace/span.h` and `sdk/include/opentelemetry/sdk/trace/span_data.h`. +> Queued spans release the wrapper, keeping only `SpanData` + attributes (~500 bytes). + +### 3.5.3 Memory Growth Characteristics + +```mermaid +--- +config: + xyChart: + width: 700 + height: 400 +--- +xychart-beta + title "Memory Usage vs Span Rate (bounded by queue limit)" + x-axis "Spans/second" [0, 200, 400, 600, 800, 1000] + y-axis "Memory (MB)" 0 --> 12 + line [8.5, 9.2, 9.6, 9.9, 10.0, 10.0] +``` + +**Notes**: + +- Memory increases with span rate but **plateaus at queue capacity** (default 2048 spans) +- Batch export prevents unbounded growth +- At queue limit, oldest spans are dropped (not blocked) +- Maximum memory is bounded: ~8.3 MB static (dominated by worker thread stack) + 2048 queued spans x ~500 bytes (~1 MB) + active spans (~0.8 MB) ≈ **~10 MB ceiling** +- The worker thread stack (~8 MB) is virtual memory; actual RSS depends on stack usage (typically much less) + +> **Measured outcome**: A perf-iac comparison (telemetry compiled-in + enabled vs compiled-out, +> 9 nodes — validators and client-handlers — under sustained payment load) recorded **no measurable +> RSS increase over the telemetry-off baseline** (~15 GiB mean / ~18–19 GiB peak on both sides), +> with no OOM, no swap, and no leak across the run. The ~10 MB ceiling above is therefore a +> provisioning safety margin (dominated by virtual thread-stack address space), not an expected +> resident-memory increase. Steady-state cost shows up as throughput (~3–4% at head sampling 1.0), +> not memory. + +### 3.5.4 Performance Data Sources + +The overhead estimates in Sections 3.3-3.5 are derived from the following sources: + +| Source | What it covers | URL | +| ------------------------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| OTel C++ SDK CI benchmarks (969 runs) | Span creation, context activation, sampler overhead | [Benchmark Dashboard](https://open-telemetry.github.io/opentelemetry-cpp/benchmarks/) | +| `api/test/trace/span_benchmark.cc` | API-level span creation (~22 ns no-op) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/api/test/trace/span_benchmark.cc) | +| `sdk/test/trace/sampler_benchmark.cc` | SDK span creation with samplers (~1,000 ns AlwaysOn) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/test/trace/sampler_benchmark.cc) | +| `sdk/include/.../span_data.h` | SpanData memory layout (~250 bytes base) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/trace/span_data.h) | +| `sdk/src/trace/span.h` | Span wrapper memory layout (~88 bytes) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/trace/span.h) | +| `sdk/include/.../batch_span_processor_options.h` | Default queue size (2048), batch size (512) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/trace/batch_span_processor_options.h) | +| `sdk/include/.../circular_buffer.h` | CircularBuffer implementation (AtomicUniquePtr array) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/common/circular_buffer.h) | +| OTLP proto definition | Serialized span size estimation | [Proto](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto) | + +--- + +## 3.6 Network Overhead Analysis + +### 3.6.1 Export Bandwidth + +> **Bytes per span**: Estimates use ~500 bytes/span (conservative upper bound). OTLP protobuf analysis +> shows a typical span with 3-5 string attributes serializes to ~200-300 bytes raw; with gzip +> compression (~60-70% of raw) and batching (amortized headers), ~350 bytes/span is more realistic. +> The table uses the conservative estimate for capacity planning. + +| Sampling Rate | Spans/sec | Bandwidth | Notes | +| ------------- | --------- | --------- | ---------------- | +| 100% | ~500 | ~250 KB/s | Development only | +| 10% | ~50 | ~25 KB/s | Staging | +| 1% | ~5 | ~2.5 KB/s | Production | +| Error-only | ~1 | ~0.5 KB/s | Minimal overhead | + +### 3.6.2 Trace Context Propagation + +| Message Type | Context Size | Messages/sec | Overhead | +| ---------------------- | ------------ | ------------ | ----------- | +| TMTransaction | 25 bytes | ~100 | ~2.5 KB/s | +| TMProposeSet | 25 bytes | ~10 | ~250 B/s | +| TMValidation | 25 bytes | ~50 | ~1.25 KB/s | +| **Total P2P overhead** | | | **~4 KB/s** | + +--- + +## 3.7 Optimization Strategies + +### 3.7.1 Sampling Strategies + +#### Tail Sampling + +```mermaid +flowchart TD + trace["New Trace"] + + trace --> errors{"Is Error?"} + errors -->|Yes| sample["SAMPLE"] + errors -->|No| consensus{"Is Consensus?"} + + consensus -->|Yes| sample + consensus -->|No| slow{"Is Slow?"} + + slow -->|Yes| sample + slow -->|No| prob{"Random < 10%?"} + + prob -->|Yes| sample + prob -->|No| drop["DROP"] + + style sample fill:#4caf50,stroke:#388e3c,color:#fff + style drop fill:#f44336,stroke:#c62828,color:#fff +``` + +### 3.7.2 Batch Tuning Recommendations + +| Environment | Batch Size | Batch Delay | Max Queue | +| ------------------ | ---------- | ----------- | --------- | +| Low-latency | 128 | 1000ms | 512 | +| High-throughput | 1024 | 10000ms | 8192 | +| Memory-constrained | 256 | 2000ms | 512 | + +### 3.7.3 Conditional Instrumentation + +Instrumentation is gated on two levels. A compile-time feature flag (`XRPL_ENABLE_TELEMETRY`) reduces the trace macros to no-ops when telemetry is built out, so disabled builds carry zero cost. At runtime, per-component guards (e.g. `shouldTracePeer()`) skip span creation for components whose tracing is turned off, incurring no overhead beyond a single boolean check. + +--- + +## 3.8 Links to Detailed Documentation + +- **[Configuration Reference](./05-configuration-reference.md)**: Configuration options and collector setup +- **[Implementation Phases](./06-implementation-phases.md)**: Detailed timeline and milestones + +--- + +## 3.9 Code Intrusiveness Assessment + +> **TxQ** = Transaction Queue + +This section provides a detailed assessment of how intrusive the OpenTelemetry integration is to the existing xrpld codebase. + +### 3.9.1 Files Modified Summary + +| Component | Files Modified | Architectural Impact | +| --------------------- | -------------- | -------------------- | +| **Core Telemetry** | 10 new files | None (new module) | +| **Application Init** | 2 files | Minimal | +| **RPC Layer** | 3 files | Minimal | +| **Transaction Relay** | 4 files | Low | +| **Consensus** | 3 files | Low-Medium | +| **Protocol Buffers** | 1 file | Low | +| **CMake/Build** | 3 files | Minimal | +| **PathFinding** | 2 | Minimal | +| **TxQ/Fee** | 2 | Minimal | +| **Validator/Amend** | 3 | Minimal | +| **Total** | **~33 files** | **Low** | + +### 3.9.2 Detailed File Impact + +```mermaid +pie title Code Changes by Component + "New Telemetry Module" : 800 + "Transaction Relay" : 160 + "Consensus" : 130 + "RPC Layer" : 100 + "PathFinding" : 80 + "TxQ/Fee" : 60 + "Validator/Amendment" : 40 + "Application Init" : 35 + "Protocol Buffers" : 25 + "Build System" : 60 +``` + +#### New Files (No Impact on Existing Code) + +| File | Purpose | +| ------------------------------------------- | ------------------------- | +| `include/xrpl/telemetry/Telemetry.h` | Main interface | +| `include/xrpl/telemetry/TelemetryConfig.h` | Configuration structures | +| `include/xrpl/telemetry/TraceContext.h` | Context propagation | +| `include/xrpl/telemetry/SpanGuard.h` | RAII wrapper | +| `include/xrpl/telemetry/DiscardFlag.h` | Thread-local discard flag | +| `include/xrpl/telemetry/SpanAttributes.h` | Attribute helpers | +| `src/libxrpl/telemetry/Telemetry.cpp` | Implementation | +| `src/libxrpl/telemetry/TelemetryConfig.cpp` | Config parsing | +| `src/libxrpl/telemetry/TraceContext.cpp` | Context serialization | +| `src/libxrpl/telemetry/NullTelemetry.cpp` | No-op implementation | + +#### Modified Files (Existing Xrpld Code) + +| File | Risk Level | +| ------------------------------------------------- | ---------- | +| `src/xrpld/app/main/Application.cpp` | Low | +| `include/xrpl/core/ServiceRegistry.h` | Low | +| `src/xrpld/rpc/detail/ServerHandler.cpp` | Low | +| `src/xrpld/rpc/handlers/*.cpp` | Low | +| `src/xrpld/overlay/detail/PeerImp.cpp` | Medium | +| `src/xrpld/overlay/detail/OverlayImpl.cpp` | Medium | +| `src/xrpld/app/consensus/RCLConsensus.cpp` | Medium | +| `src/xrpld/app/consensus/RCLConsensusAdaptor.cpp` | Medium | +| `src/xrpld/core/JobQueue.cpp` | Low | +| `src/xrpld/app/paths/PathRequest.cpp` | Low | +| `src/xrpld/app/paths/Pathfinder.cpp` | Low | +| `src/xrpld/app/misc/TxQ.cpp` | Low | +| `src/xrpld/app/main/LoadManager.cpp` | Low | +| `src/xrpld/app/misc/ValidatorList.cpp` | Low | +| `src/xrpld/app/misc/AmendmentTable.cpp` | Low | +| `src/xrpld/app/misc/Manifest.cpp` | Low | +| `src/xrpld/shamap/SHAMap.cpp` | Low | +| `src/xrpld/overlay/detail/ripple.proto` | Low | +| `CMakeLists.txt` | Low | +| `cmake/FindOpenTelemetry.cmake` | None (new) | + +### 3.9.3 Risk Assessment by Component + +
+ +**Do First** ↖ ↗ **Plan Carefully** + +```mermaid +quadrantChart + title Code Intrusiveness Risk Matrix + x-axis Low Risk --> High Risk + y-axis Low Value --> High Value + + RPC Tracing: [0.2, 0.55] + Transaction Relay: [0.55, 0.85] + Consensus Tracing: [0.75, 0.92] + Peer Message Tracing: [0.85, 0.35] + JobQueue Context: [0.3, 0.42] + Ledger Acquisition: [0.48, 0.65] + PathFinding: [0.38, 0.72] + TxQ and Fees: [0.25, 0.62] + Validator Mgmt: [0.15, 0.35] +``` + +**Optional** ↙ ↘ **Avoid** + +
+ +#### Risk Level Definitions + +| Risk Level | Definition | Mitigation | +| ---------- | ---------------------------------------------------------------- | ---------------------------------- | +| **Low** | Additive changes only; no modification to existing logic | Standard code review | +| **Medium** | Minor modifications to existing functions; clear boundaries | Comprehensive unit tests | +| **High** | Changes to core logic or data structures; potential side effects | Integration tests + staged rollout | + +### 3.9.4 Architectural Impact Assessment + +| Aspect | Impact | Justification | +| -------------------- | ------- | -------------------------------------------------------------------------------- | +| **Data Flow** | Minimal | Read-only instrumentation; no modification to consensus or transaction data flow | +| **Threading Model** | Minimal | Context propagation uses thread-local storage (standard OTel pattern) | +| **Memory Model** | Low | Bounded queues prevent unbounded growth; RAII ensures cleanup | +| **Network Protocol** | Low | Optional fields in protobuf (high field numbers); backward compatible | +| **Configuration** | None | New config section; existing configs unaffected | +| **Build System** | Low | Optional CMake flag; builds work without OpenTelemetry | +| **Dependencies** | Low | OpenTelemetry SDK is optional; null implementation when disabled | + +### 3.9.5 Backward Compatibility + +| Compatibility | Status | Notes | +| --------------- | ------- | ----------------------------------------------------- | +| **Config File** | ✅ Full | New `[telemetry]` section is optional | +| **Protocol** | ✅ Full | Optional protobuf fields with high field numbers | +| **Build** | ✅ Full | `XRPL_ENABLE_TELEMETRY=OFF` produces identical binary | +| **Runtime** | ✅ Full | `enabled=0` produces zero overhead | +| **API** | ✅ Full | No changes to public RPC or P2P APIs | + +### 3.9.6 Rollback Strategy + +If issues are discovered after deployment: + +1. **Immediate**: Set `enabled=0` in config and restart (zero code change) +2. **Quick**: Rebuild with `XRPL_ENABLE_TELEMETRY=OFF` +3. **Complete**: Revert telemetry commits (clean separation makes this easy) + +### 3.9.7 Code Change Examples + +**Minimal RPC Instrumentation (Low Intrusiveness):** Instrumenting an RPC handler adds roughly 3-4 lines: one macro to start the span and one or two `setAttribute` calls (command name, status). The span ends automatically via RAII, so the existing control flow — process the request, send the result — is untouched. + +**Consensus Instrumentation (Medium Intrusiveness):** Consensus is slightly more intrusive because child spans in later phase transitions need the round's context. Beyond the span-start and attribute macros, this requires storing the active context in a new member variable (`currentRoundContext_`) at round start. The existing round logic itself remains unchanged. + +--- + +_Previous: [Design Decisions](./02-design-decisions.md)_ | _Next: [Configuration Reference](./05-configuration-reference.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md new file mode 100644 index 00000000000..7e47023f516 --- /dev/null +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -0,0 +1,275 @@ +# Configuration Reference + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Implementation Phases](./06-implementation-phases.md) + +--- + +## 5.1 xrpld Configuration + +> **OTLP** = OpenTelemetry Protocol | **TxQ** = Transaction Queue + +### 5.1.1 Configuration File Section + +The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Telemetry is disabled by default (`enabled=0`); enabling it turns on distributed tracing for transaction flow, consensus, and RPC calls, with traces exported to an OpenTelemetry Collector over OTLP. Head sampling is intentionally fixed at 1.0 (sample everything) and is not configurable — per-node head-sampling would produce broken/partial distributed traces, so volume reduction is delegated to the collector's tail sampling (see Section 7.4.2). The full option reference follows. + +### 5.1.2 Configuration Options Summary + +| Option | Type | Default | Description | +| -------------------------- | ------ | --------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `enabled` | bool | `false` | Enable/disable telemetry | +| `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint | +| `use_tls` | bool | `false` | Enable TLS for exporter connection | +| `tls_ca_cert` | string | `""` | Path to CA certificate file | +| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | +| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | +| `batch_size` | uint | `512` | Spans per export batch | +| `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | +| `max_queue_size` | uint | `2048` | Maximum queued spans | +| `trace_transactions` | bool | `true` | Enable transaction tracing | +| `trace_consensus` | bool | `true` | Enable consensus tracing | +| `trace_rpc` | bool | `true` | Enable RPC tracing | +| `trace_peer` | bool | `true` | Enable peer message tracing (high volume) | +| `trace_ledger` | bool | `true` | Enable ledger tracing | +| `tx_trace_strategy` | string | `"deterministic"` | TX trace ID strategy: `"deterministic"` (trace_id = txHash[0:16]) or `"attribute"` (random) | +| `consensus_trace_strategy` | string | `"deterministic"` | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"attribute"` (random) | +| `service_name` | string | `"xrpld"` | Service name (`service.name`) for traces and metrics | +| `service_instance_id` | string | `` | Instance identifier | + +**Planned (not yet implemented)**: the following options appear in the design +documents but are not parsed by `TelemetryConfig.cpp` in Phase 1b and later +phases. They will be added as the corresponding subsystems are instrumented: + +| Option | Planned Phase | Purpose | +| -------------------------- | ------------- | ----------------------------------------------------------------------- | +| `exporter` | Future | Select between OTLP/HTTP and OTLP/gRPC | +| `trace_pathfind` | Phase 2 | Path computation tracing toggle | +| `trace_txq` | Phase 3 | Transaction queue tracing toggle | +| `trace_validator` | Future | Validator list / manifest update tracing | +| `trace_amendment` | Future | Amendment voting tracing | +| `consensus_trace_strategy` | Phase 4 | Trace ID strategy for consensus rounds (`deterministic` \| `attribute`) | + +--- + +## 5.2 Configuration Parser + +> **TxQ** = Transaction Queue + +The parser `setupTelemetry()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` reads the `[telemetry]` `Section` and populates a `Telemetry::Setup` struct, applying the defaults listed in Section 5.1.2 via `section.value_or(...)`. It derives `serviceInstanceId` from the node public key when not overridden, selects the exporter endpoint default by exporter type, and leaves the sampling ratio at its fixed 1.0 default (not read from config — see Section 7.4.2). + +--- + +## 5.3 Application Integration + +### 5.3.1 ApplicationImp Changes + +> **Deferred identity**: The node public key (`nodeIdentity_`) is not +> available during `ApplicationImp`'s member initializer list — it is +> resolved later in `setup()`. The `Telemetry` object is therefore +> constructed with an empty `serviceInstanceId` and patched via +> `setServiceInstanceId()` once `setup()` has called `getNodeIdentity()`. + +`ApplicationImp` (in `src/xrpld/app/main/Application.cpp`) owns a `std::unique_ptr telemetry_`. It is built in the member initializer list via `makeTelemetry(setupTelemetry(...))` with an empty `serviceInstanceId`, then patched in `setup()` by calling `setServiceInstanceId()` with the Base58 node public key (unless the user supplied a custom `service_instance_id`). `start()` and `run()` forward to `telemetry_->start()` / `telemetry_->stop()`, and `getTelemetry()` returns the owned instance. + +### 5.3.2 ServiceRegistry Interface Addition + +`include/xrpl/core/ServiceRegistry.h` gains a pure-virtual `telemetry::Telemetry& getTelemetry()` (with a forward declaration of `telemetry::Telemetry`), giving every component a uniform accessor for the tracing subsystem. + +> **Note:** `Application` extends `ServiceRegistry`, so `getTelemetry()` is +> available on both. Components that hold a `ServiceRegistry&` (e.g. +> `NetworkOPsImp`) call `registry_.get().getTelemetry()`. Components that +> still hold an `Application&` (e.g. `ServerHandler`, `PeerImp`, +> `RCLConsensusAdaptor`) call `app_.getTelemetry()` directly. + +--- + +## 5.4 CMake Integration + +> **OTLP** = OpenTelemetry Protocol + +### 5.4.1 Find OpenTelemetry Module + +A `cmake/FindOpenTelemetry.cmake` module locates the OpenTelemetry C++ SDK. It first tries `find_package(opentelemetry-cpp CONFIG)`, aliasing the imported targets `OpenTelemetry::api`, `OpenTelemetry::sdk`, and `OpenTelemetry::otlp_grpc_exporter`, and falls back to `pkg-config` when no CMake config package is present. + +### 5.4.2 CMakeLists.txt Changes + +The top-level `CMakeLists.txt` adds an `XRPL_ENABLE_TELEMETRY` option (default `OFF`). When enabled, it runs `find_package(OpenTelemetry REQUIRED)`, defines the `XRPL_ENABLE_TELEMETRY` compile flag, and builds the `xrpl_telemetry` library from the real telemetry sources linked against the OpenTelemetry targets; when disabled, it builds the same target from a no-op `NullTelemetry.cpp` so call sites compile unchanged. + +--- + +## 5.5 OpenTelemetry Collector Configuration + +> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring + +> **Production hardening**: The configurations in this section are starting points. For production deployments where xrpld ships telemetry across a network to a centrally-hosted collector, see [Securing the OTel Pipeline](./secure-OTel.md) for the required mTLS receiver config, NetworkPolicy, and peer trace-context validation. + +The authoritative collector config lives in the repo at `docker/telemetry/otel-collector-config.yaml` (with Tempo backend config in `docker/telemetry/tempo.yaml`). The sections below summarize the development and production shapes of that pipeline. + +### 5.5.1 Development Configuration + +The development collector enables an OTLP receiver on both gRPC (`0.0.0.0:4317`) and HTTP (`0.0.0.0:4318`), a single `batch` processor (1s timeout, batch size 100), and two exporters: a `logging` exporter for console debugging and `otlp/tempo` (insecure) for trace visualization. The single `traces` pipeline wires receiver → batch → both exporters. + +### 5.5.2 Production Configuration + +The production collector adds TLS on the OTLP gRPC receiver and a richer processor chain: a `memory_limiter` (OOM guard), `batch` (5s timeout, size 512), `tail_sampling`, and an `attributes` processor that hashes sensitive fields (e.g. `tx_account`) and stamps `deployment.environment`. Tail sampling keeps all `ERROR` traces, slow consensus rounds (>5s) and slow RPC requests (>1s), and probabilistically samples the remainder at 10%. Exporters target Grafana Tempo (TLS) and Elastic APM; `health_check` and `zpages` extensions are enabled for operability. + +--- + +## 5.6 Docker Compose Development Environment + +> **OTLP** = OpenTelemetry Protocol + +The authoritative development stack lives in the repo at `docker/telemetry/docker-compose.yml`. It brings up four services on a shared `xrpld-telemetry` network: an `otel-collector` (otel/opentelemetry-collector-contrib) exposing OTLP gRPC `4317`, OTLP HTTP `4318`, and health check `13133`; `tempo` for trace storage/visualization; `grafana` with provisioned datasources and dashboards (anonymous admin enabled); and an optional `prometheus` for metric correlation. + +--- + +## 5.7 Configuration Architecture + +> **OTLP** = OpenTelemetry Protocol + +```mermaid +flowchart TB + subgraph config["Configuration Sources"] + cfgFile["xrpld.cfg
[telemetry] section"] + cmake["CMake
XRPL_ENABLE_TELEMETRY"] + end + + subgraph init["Initialization"] + parse["setupTelemetry()"] + factory["makeTelemetry()"] + end + + subgraph runtime["Runtime Components"] + tracer["TracerProvider"] + exporter["OTLP Exporter"] + processor["BatchProcessor"] + end + + subgraph collector["Collector Pipeline"] + recv["Receivers"] + proc["Processors"] + exp["Exporters"] + end + + cfgFile --> parse + cmake -->|"compile flag"| parse + parse --> factory + factory --> tracer + tracer --> processor + processor --> exporter + exporter -->|"OTLP"| recv + recv --> proc + proc --> exp + + style config fill:#e3f2fd,stroke:#1976d2 + style runtime fill:#e8f5e9,stroke:#388e3c + style collector fill:#fff3e0,stroke:#ff9800 +``` + +**Reading the diagram:** + +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. +- **Initialization**: `setupTelemetry()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects. +- **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. +- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) +- **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.). + +--- + +## 5.8 Grafana Integration + +> **APM** = Application Performance Monitoring + +Step-by-step instructions for integrating xrpld traces with Grafana. + +### 5.8.1 Data Source Configuration + +#### Tempo (Recommended) + +A Tempo datasource (`grafana/provisioning/datasources/tempo.yaml`, provisioned from `docker/telemetry/grafana/`) points at `http://tempo:3200` and enables `tracesToLogs` (linking to Loki on `service.name`/`tx_hash` and mapping `trace_id` → `traceID`), `serviceMap` against Prometheus, the node graph, and Loki search. + +#### Elastic APM + +Alternatively, an Elasticsearch datasource (`grafana/provisioning/datasources/elastic-apm.yaml`) of type `elasticsearch` points at `http://elasticsearch:9200` against the `apm-*` index, using `@timestamp` as the time field and mapping the log message/level fields. + +### 5.8.2 Dashboard Provisioning + +A dashboard provider (`grafana/provisioning/dashboards/dashboards.yaml`) loads the `xrpld` dashboard folder from disk (`/var/lib/grafana/dashboards/rippled`), polling for changes every 30s with deletion disabled. + +### 5.8.3 Example Dashboard: RPC Performance + +An example `xrpld RPC Performance` dashboard (uid `xrpld-rpc-performance`) sourced from Tempo via TraceQL provides four panels: RPC latency by command (heatmap), RPC error rate by command (timeseries), the top 10 slowest RPC commands by average duration (table), and a recent-traces table. + +### 5.8.4 Example Dashboard: Transaction Tracing + +An example `xrpld Transaction Tracing` dashboard (uid `xrpld-tx-tracing`) over Tempo provides three panels: transaction throughput (`tx.receive` rate, stat), cross-node relay count (average `span.relay_count` on `tx.relay`, timeseries), and a table of transaction validation errors (`tx.validate` with `status.code=error`). + +### 5.8.5 TraceQL Query Examples + +Common queries for xrpld traces: + +``` +# Find all traces for a specific transaction hash +{resource.service.name="xrpld" && span.tx_hash="ABC123..."} + +# Find slow RPC commands (>100ms) +{resource.service.name="xrpld" && name=~"rpc.command.*"} | duration > 100ms + +# Find consensus rounds taking >5 seconds +{resource.service.name="xrpld" && name="consensus.round"} | duration > 5s + +# Find failed transactions with error details +{resource.service.name="xrpld" && name="tx.validate" && status.code=error} + +# Find transactions relayed to many peers +{resource.service.name="xrpld" && name="tx.relay"} | span.relay_count > 10 + +# Compare latency across nodes +{resource.service.name="xrpld" && name="rpc.command.account_info"} | avg(duration) by (resource.service.instance.id) +``` + +### 5.8.6 Correlation with PerfLog + +To correlate OpenTelemetry traces with existing PerfLog data: + +**Step 1: Configure Loki to ingest PerfLog** + +Configure a Promtail scrape job (`promtail-config.yaml`) that tails `/var/log/rippled/perf*.log`, parses each JSON line, and promotes `trace_id`, `ledger_seq`, and `tx_hash` to Loki labels. + +**Step 2: Add trace_id to PerfLog entries** + +Modify PerfLog so its JSON output includes a `trace_id` field whenever a valid span is active: fetch the current span from the OpenTelemetry runtime context, and if its context is valid, render the trace ID as a 32-character lowercase hex string into the log entry. + +**Step 3: Configure Grafana trace-to-logs link** + +In the Tempo datasource, set the `tracesToLogs` derived field to link to Loki on the `trace_id` and `tx_hash` tags, with `filterByTraceID: true`. + +### 5.8.7 Correlation with Insight/OTel System Metrics + +To correlate traces with Beast Insight system metrics: + +**Step 1: Export Insight metrics to Prometheus** + +Beast Insight metrics are exported natively via OTLP to the OTel Collector, +which exposes them on the Prometheus endpoint alongside spanmetrics. Configure +the `[insight]` section of `xrpld.cfg` with `server=otel`, +`endpoint=http://localhost:4318/v1/metrics`, and `prefix=xrpld`; no separate +StatsD exporter or Prometheus scrape job is needed when using `server=otel`. + +**Step 2: Add exemplars to metrics** + +The OpenTelemetry SDK automatically adds exemplars (trace IDs) to metrics when using the Prometheus exporter, linking metric spikes to specific traces. + +**Step 3: Configure Grafana metric-to-trace link** + +In the Prometheus datasource, set `exemplarTraceIdDestinations` to map the `trace_id` exemplar to the Tempo datasource. + +**Step 4: Dashboard panel with exemplars** + +Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled. + +This allows clicking on metric data points to jump directly to the related trace. + +--- + +_Previous: [Implementation Strategy](./03-implementation-strategy.md)_ | _Next: [Implementation Phases](./06-implementation-phases.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md new file mode 100644 index 00000000000..504b9d73639 --- /dev/null +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -0,0 +1,2208 @@ +# Implementation Phases + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Configuration Reference](./05-configuration-reference.md) | [Observability Backends](./07-observability-backends.md) + +--- + +## 6.1 Phase Overview + +> **TxQ** = Transaction Queue + +```mermaid +gantt + title OpenTelemetry Implementation Timeline + dateFormat YYYY-MM-DD + axisFormat Week %W + + section Phase 1 + Core Infrastructure :p1, 2024-01-01, 2w + SDK Integration :p1a, 2024-01-01, 4d + Telemetry Interface :p1b, after p1a, 3d + Configuration & CMake :p1c, after p1b, 3d + Unit Tests :p1d, after p1c, 2d + Buffer & Integration :p1e, after p1d, 2d + + section Phase 2 + RPC Tracing :p2, after p1, 2w + HTTP Context Extraction :p2a, after p1, 2d + RPC Handler Instrumentation :p2b, after p2a, 4d + PathFinding Instrumentation :p2f, after p2b, 2d + TxQ Instrumentation :p2g, after p2f, 2d + WebSocket Support :p2c, after p2g, 2d + Integration Tests :p2d, after p2c, 2d + Buffer & Review :p2e, after p2d, 4d + + section Phase 3 + Transaction Tracing :p3, after p2, 2w + Protocol Buffer Extension :p3a, after p2, 2d + PeerImp Instrumentation :p3b, after p3a, 3d + Fee Escalation Instrumentation :p3f, after p3b, 2d + Relay Context Propagation :p3c, after p3f, 3d + Multi-node Tests :p3d, after p3c, 2d + Buffer & Review :p3e, after p3d, 4d + + section Phase 4 + Consensus Tracing :p4, after p3, 2w + Consensus Round Spans :p4a, after p3, 3d + Proposal Handling :p4b, after p4a, 3d + Establish Phase (4a) :p4f, after p4b, 3d + Validation Tests :p4c, after p4f, 4d + Buffer & Review :p4e, after p4c, 4d + + section Phase 5 + Documentation & Deploy :p5, after p4, 1w + + section Phase 6 + StatsD Metrics Bridge :p6, after p5, 1w + + section Phase 7 + Native OTel Metrics :p7, after p6, 2w + + section Phase 8 + Log-Trace Correlation :p8, after p7, 1w + + section Phase 9 (Future) + Internal Metric Gap Fill :p9, after p8, 2.5w + + section Phase 10 (Future) + Workload Validation :p10, after p9, 2w + + section Phase 11 (Future) + Third-Party Collection :p11, after p10, 3w +``` + +--- + +## 6.2 Phase 1: Core Infrastructure (Weeks 1-2) + +**Objective**: Establish foundational telemetry infrastructure + +### Tasks + +| Task | Description | +| ---- | ----------------------------------------------------- | +| 1.1 | Add OpenTelemetry C++ SDK to Conan/CMake | +| 1.2 | Implement `Telemetry` interface and factory | +| 1.3 | Implement `SpanGuard` RAII wrapper | +| 1.4 | Implement configuration parser | +| 1.5 | Integrate into `ApplicationImp` | +| 1.6 | Add conditional compilation (`XRPL_ENABLE_TELEMETRY`) | +| 1.7 | Create `NullTelemetry` no-op implementation | +| 1.8 | Unit tests for core infrastructure | + +### Exit Criteria + +- [ ] OpenTelemetry SDK compiles and links +- [ ] Telemetry can be enabled/disabled via config +- [ ] Basic span creation works +- [ ] No performance regression when disabled +- [ ] Unit tests passing + +--- + +## 6.3 Phase 2: RPC Tracing (Weeks 3-4) + +> **TxQ** = Transaction Queue + +**Objective**: Complete tracing for all RPC operations + +### Tasks + +| Task | Description | +| ---- | -------------------------------------------------------------------------- | +| 2.1 | Implement W3C Trace Context HTTP header extraction | +| 2.2 | Instrument `ServerHandler::onRequest()` | +| 2.3 | Instrument `RPCHandler::doCommand()` | +| 2.4 | Add RPC-specific attributes | +| 2.5 | Instrument WebSocket handler | +| 2.6 | PathFinding instrumentation (`pathfind.request`, `pathfind.compute` spans) | +| 2.7 | TxQ instrumentation (`txq.enqueue`, `txq.apply` spans) | +| 2.8 | Integration tests for RPC tracing | +| 2.9 | Performance benchmarks | +| 2.10 | Documentation | + +### Exit Criteria + +- [ ] All RPC commands traced +- [ ] Trace context propagates from HTTP headers +- [ ] WebSocket and HTTP both instrumented +- [ ] <1ms overhead per RPC call +- [ ] Integration tests passing + +--- + +## 6.4 Phase 3: Transaction Tracing (Weeks 5-6) + +**Objective**: Trace transaction lifecycle across network with deterministic cross-node correlation + +### Tasks + +| Task | Description | +| ---- | -------------------------------------------------------------- | +| 3.1 | Define `TraceContext` Protocol Buffer message | +| 3.2 | Implement protobuf context serialization | +| 3.3 | Instrument `PeerImp::handleTransaction()` | +| 3.4 | Instrument `NetworkOPs::submitTransaction()` | +| 3.5 | Instrument HashRouter integration | +| 3.6 | Fee escalation instrumentation (`fee.escalate` span) | +| 3.7 | Implement relay context propagation | +| 3.8 | Integration tests (multi-node) | +| 3.9 | Deterministic transaction trace ID (`trace_id = txHash[0:16]`) | +| 3.10 | Performance benchmarks | + +### Deterministic Trace ID (Task 3.9) + +Transaction spans use **deterministic trace IDs** derived from the transaction hash: +`trace_id = txHash[0:16]`. All nodes handling the same transaction independently +produce spans under the same trace_id. Protobuf `span_id` propagation (Task 3.7) +additionally provides parent-child relay ordering when available. See +[02-design-decisions.md §2.5.0](./02-design-decisions.md) for the design rationale +and [Phase3_taskList.md Task 3.9](./Phase3_taskList.md) for the full implementation spec. + +### Exit Criteria + +- [ ] Transaction traces span across nodes +- [ ] Trace context in Protocol Buffer messages +- [ ] HashRouter deduplication visible in traces +- [ ] Multi-node integration tests passing +- [ ] <5% overhead on transaction throughput +- [ ] Deterministic trace_id: all nodes produce same trace_id for same transaction +- [ ] Protobuf span_id propagation preserves parent-child ordering when available + +--- + +## 6.5 Phase 4: Consensus Tracing (Weeks 7-8) + +**Objective**: Full observability into consensus rounds + +### Tasks + +| Task | Description | Status | +| ---- | ---------------------------------------------- | ------------------ | +| 4.1 | Instrument `RCLConsensusAdaptor::startRound()` | ✅ Done (via 4a.2) | +| 4.2 | Instrument phase transitions | ✅ Done | +| 4.3 | Instrument proposal handling | ✅ Done | +| 4.4 | Instrument validation handling | ✅ Done | +| 4.5 | Add consensus-specific attributes | ✅ Done | +| 4.6 | Correlate with transaction traces | ✅ Done | +| 4.7 | Build verification and testing | ✅ Done | +| 4.8 | Validation span enrichment (ext. dashboard) | ❌ Not done | + +**Note**: The original plan doc listed tasks 4.7-4.11 as "Validator list tracing", +"Amendment voting tracing", "SHAMap sync tracing", "Multi-validator integration tests", +and "Performance validation". These were descoped and replaced by the tasklist's 4.7 +(build verification) and 4.8 (validation span enrichment). Validator, amendment, and +SHAMap tracing are not implemented. + +### Spans Produced + +| Span Name | Location | Attributes | +| --------------------------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.phase.open` | `Consensus.h` | _(none)_ | +| `consensus.proposal.send` | `RCLConsensus.cpp` | `consensus_round` | +| `consensus.ledger_close` | `RCLConsensus.cpp` | `ledger_seq`, `consensus_mode` | +| `consensus.accept` | `RCLConsensus.cpp` | `proposers`, `round_time_ms`, `quorum` | +| `consensus.accept.apply` | `RCLConsensus.cpp` | `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `ledger_seq`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction` | +| `consensus.validation.send` | `RCLConsensus.cpp` | `ledger_seq`, `proposing` | + +### Exit Criteria + +- [x] Complete consensus round traces +- [x] Phase transitions visible (open, establish, close, accept) +- [x] Proposals and validations traced — send and receive; relay deferred to Phase 4b +- [x] Close time agreement tracked (per `avCT_CONSENSUS_PCT`) +- [x] No impact on consensus timing +- [ ] Multi-validator test network validated +- [x] Transaction-consensus correlation (Task 4.6) — `tx.included` events in doAccept +- [ ] Validation span enrichment (Task 4.8) — not implemented + +### Implementation Status — Phase 4a Complete + +Phase 4a (establish-phase gap fill & cross-node correlation) adds: + +- **Deterministic trace ID** derived from `previousLedger.id()` so all validators + in the same round share the same `trace_id` (switchable via + `consensus_trace_strategy` config: `"deterministic"` or `"attribute"`). + See [Configuration Reference](./05-configuration-reference.md) for full + configuration options. +- **Round lifecycle spans**: `consensus.round` with round-to-round span links. +- **Establish phase**: `consensus.establish`, `consensus.update_positions` (with + `dispute.resolve` events), `consensus.check` (with threshold tracking). +- **Mode changes**: `consensus.mode_change` spans. +- **Validation**: `consensus.validation.send` with span link to round span + (thread-safe cross-thread access via `roundSpanContext_` snapshot). +- **Separation of concerns**: telemetry extracted to private helpers + (`startRoundTracing`, `createValidationSpan`, `startEstablishTracing`, + `updateEstablishTracing`, `endEstablishTracing`). + +See [Phase4_taskList.md](./Phase4_taskList.md) for the full spec and implementation notes. + +--- + +## 6.5a Phase 4a: Establish-Phase Gap Fill & Cross-Node Correlation + +**Objective**: Fill tracing gaps in the establish phase and establish cross-node +correlation using deterministic trace IDs derived from `previousLedger.id()`. + +**Approach**: Direct instrumentation in `Consensus.h` and `RCLConsensus.cpp`. +All spans use `SpanGuard` factory methods (`span()`, `hashSpan()`, `linkedSpan()`) +with `TraceCategory::Consensus` gating. No macros used — all tracing via direct +`SpanGuard` API calls. + +### Tasks + +| Task | Description | Effort | Risk | Status | +| ---- | ------------------------------------------------ | ------ | ------ | ------------------------ | +| 4a.0 | Prerequisites: extend SpanGuard & Telemetry APIs | 1d | Medium | ✅ Done (no macros) | +| 4a.1 | Adaptor `getTelemetry()` method | 0.5d | Low | ⏭️ Skipped (not needed) | +| 4a.2 | Switchable round span with deterministic traceID | 2d | High | ✅ Done | +| 4a.3 | Span members in `Consensus.h` | 0.5d | Medium | ✅ Done (with deviation) | +| 4a.4 | Instrument `phaseEstablish()` | 1d | Medium | ✅ Done | +| 4a.5 | Instrument `updateOurPositions()` | 1d | Medium | ✅ Done | +| 4a.6 | Instrument `haveConsensus()` (thresholds) | 1d | Medium | ✅ Done | +| 4a.7 | Instrument mode changes | 0.5d | Low | ✅ Done | +| 4a.8 | Reparent existing spans under round | 0.5d | Low | ✅ Done | +| 4a.9 | Build verification and testing | 1d | Low | ✅ Done | + +**Total Effort**: 9 days + +### Spans Produced + +| Span Name | Location | Key Attributes (actually set) | +| ---------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | `RCLConsensus.cpp` | `consensus_round_id`, `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy` | +| `consensus.establish` | `Consensus.h` | `converge_percent`, `establish_count`, `proposers` | +| `consensus.update_positions` | `Consensus.h` | `converge_percent`, `proposers`, `have_close_time_consensus`, `close_time_threshold`, `disputes_count`, `avalanche_threshold` | +| `consensus.check` | `Consensus.h` | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | +| `consensus.mode_change` | `RCLConsensus.cpp` | `mode_old`, `mode_new` | + +### Exit Criteria + +- [x] Establish phase internals traced (establish, update_positions, check spans) +- [x] Establish phase fully traced — `disputes_count`, `avalanche_threshold`, dispute `yays`/`nays` all implemented +- [x] Cross-node correlation works via deterministic trace_id +- [x] Strategy switchable via config (`deterministic` / `attribute`) +- [x] Consecutive rounds linked via follows-from spans +- [x] Build passes with telemetry ON and OFF +- [x] No impact on consensus timing + +See [Phase4_taskList.md](./Phase4_taskList.md) for full task details. + +--- + +## 6.5b Phase 4b: Cross-Node Propagation (Future) + +**Objective**: Wire `TraceContextPropagator` for P2P messages (proposals, +validations) to enable true distributed tracing between nodes. + +**Status**: Partially implemented. Send-side injection (proposals and +validations) and receive-side extraction (`consensus.{proposal,validation}. +receive` spans parented on the sender's context) are wired in Phase 4a. +Remaining Phase 4b work: relay spans in `share(RCLCxPeerPos)` and multi-node +validation of the propagation path. + +**Prerequisites**: Phase 4a complete and validated. + +See [Phase4_taskList.md § Phase 4b](./Phase4_taskList.md) for full design. + +--- + +## 6.6 Phase 5: Documentation & Deployment (Week 9) + +**Objective**: Production readiness + +### Tasks + +| Task | Description | Status | +| ---- | ----------------------------- | ------------------- | +| 5.1 | Operator runbook | Complete | +| 5.2 | Grafana dashboards | Complete | +| 5.3 | Alert definitions | Deferred — post-MVP | +| 5.4 | Collector deployment examples | Complete | +| 5.5 | Developer documentation | Complete | +| 5.6 | Training materials | Deferred — post-MVP | +| 5.7 | Final integration testing | Complete | + +--- + +## 6.7 Phase 6: StatsD Metrics Integration (Week 10) + +**Objective**: Bridge xrpld's existing `beast::insight` StatsD metrics into the OpenTelemetry collection pipeline, exposing 300+ pre-existing metrics alongside span-derived RED metrics in Prometheus/Grafana. + +### Background + +xrpld has a mature metrics framework (`beast::insight`) that emits StatsD-format metrics over UDP. These metrics cover node health, peer networking, RPC performance, job queue, and overlay traffic — data that **does not** overlap with the span-based instrumentation from Phases 1-5. By adding a StatsD receiver to the OTel Collector, both metric sources converge in Prometheus. + +### Metric Inventory + +| Category | Group | Type | Count | Key Metrics | +| --------------- | ------------------ | ------------- | ---------- | ----------------------------------------------------------------------------------------------------------- | +| Node State | `State_Accounting` | Gauge | 10 | `*_duration`, `*_transitions` per operating mode | +| Ledger | `LedgerMaster` | Gauge | 2 | `Validated_Ledger_Age`, `Published_Ledger_Age` | +| Ledger Fetch | — | Counter | 1 | `ledger_fetches` | +| Ledger History | `ledger.history` | Counter | 1 | `mismatch` | +| RPC | `rpc` | Counter+Event | 3 | `requests`, `time` (histogram), `size` (histogram) | +| Job Queue | `jobq` | Gauge+Event | 1 + 2×N | `job_count`, per-job `{name}` and `{name}_q` (emitted with the `jobq_` group prefix, e.g. `jobq_job_count`) | +| Peer Finder | `Peer_Finder` | Gauge | 2 | `Active_Inbound_Peers`, `Active_Outbound_Peers` | +| Overlay | `Overlay` | Gauge | 1 | `Peer_Disconnects` | +| Overlay Traffic | per-category | Gauge | 4×57 = 228 | `Bytes_In/Out`, `Messages_In/Out` per traffic category | +| Pathfinding | — | Event | 2 | `pathfind_fast`, `pathfind_full` (histograms) | +| I/O | — | Event | 1 | `ios_latency` (histogram) | +| Resource Mgr | — | Meter | 2 | `warn`, `drop` (rate counters) | +| Caches | per-cache | Gauge | 2×N | `{cache}.size`, `{cache}.hit_rate` | + +**Total**: ~255+ unique metrics (plus dynamic job-type and cache metrics) + +### Tasks + +| Task | Description | +| ---- | --------------------------------------------------------------------------------------------------------------- | +| 6.1 | **DEFERRED** Fix Meter wire format (`\|m` → `\|c`) in StatsDCollector.cpp — breaking change, tracked separately | +| 6.2 | Add `statsd` receiver to OTel Collector config | +| 6.3 | Expose UDP port 8125 in docker-compose.yml | +| 6.4 | Add `[insight]` config to integration test node configs | +| 6.5 | Create "Node Health" Grafana dashboard (16 panels) | +| 6.6 | Create "Network Traffic" Grafana dashboard (10 panels) | +| 6.7 | Create "RPC & Pathfinding (StatsD)" Grafana dashboard (8 panels) | +| 6.8 | Update integration test to verify StatsD metrics in Prometheus | +| 6.9 | Update TESTING.md and telemetry-runbook.md | + +### Wire Format Fix (Task 6.1) — DEFERRED + +The `StatsDMeterImpl` in `StatsDCollector.cpp` sends metrics with `|m` suffix, which is non-standard StatsD. The OTel StatsD receiver silently drops these. Fix: change `|m` to `|c` (counter), which is semantically correct since meters are increment-only counters. Only 2 metrics are affected (`warn`, `drop` in Resource Manager). + +**Status**: Deferred as a separate change — this is a breaking change for any StatsD backend that previously consumed the custom `|m` type. The Resource Warnings and Resource Drops dashboard panels will show no data until this fix is applied. + +### New Grafana Dashboards + +**Node Health** (`statsd-node-health.json`, uid: `xrpld-statsd-node-health`): + +- Validated/Published Ledger Age, Operating Mode Duration/Transitions, I/O Latency, Job Queue Depth, Ledger Fetch Rate, Ledger History Mismatches, Key Jobs Execution/Dequeue Time, FullBelowCache Size/Hit Rate, Ledger Publish Gap, State Duration Rate, All Jobs Detail + +**Network Traffic** (`statsd-network-traffic.json`, uid: `xrpld-statsd-network`): + +- Active Inbound/Outbound Peers, Peer Disconnects, Total Bytes/Messages In/Out, Transaction/Proposal/Validation Traffic, Top Traffic Categories, Duplicate Traffic, All Traffic Categories Detail + +**RPC & Pathfinding (StatsD)** (`statsd-rpc-pathfinding.json`, uid: `xrpld-statsd-rpc`): + +- RPC Request Rate, Response Time p95/p50, Response Size p95/p50, Pathfinding Fast/Full Duration, Resource Warnings/Drops, Response Time Heatmap + +### Exit Criteria + +- [ ] StatsD metrics visible in Prometheus (`curl localhost:9090/api/v1/query?query=ledgermaster_validated_ledger_age`) +- [ ] All 3 new Grafana dashboards load without errors +- [ ] Integration test verifies at least core StatsD metrics (ledger age, peer counts, RPC requests) +- [ ] ~~Meter metrics (`warn`, `drop`) flow correctly after `|m` → `|c` fix~~ — DEFERRED (breaking change, tracked separately; resolved by Phase 7's OTel Counter mapping) + +--- + +## 6.8 Phase 7: Native OTel Metrics Migration (Weeks 11-12) + +**Objective**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency and unifying traces and metrics into a single OTLP pipeline. + +### Motivation: Why Migrate from StatsD to Native OTel Metrics + +The Phase 6 StatsD bridge was a pragmatic first step, but it retains inherent limitations that native OTel export resolves. + +#### What We Gain + +1. **Unified telemetry pipeline** — Traces and metrics export via the same OTLP/HTTP endpoint to the same OTel Collector. One protocol, one endpoint, one config. Eliminates the split-brain architecture of "OTLP for traces, StatsD UDP for metrics." + +2. **Eliminates StatsD UDP limitations** — StatsD is fire-and-forget over UDP with no delivery guarantees, no backpressure, 1472-byte MTU packet fragmentation, and text-based encoding overhead. OTLP uses HTTP/gRPC with retries, binary protobuf encoding, and connection-level flow control. + +3. **Fixes the `|m` wire format issue** — The `StatsDMeterImpl` uses non-standard `|m` StatsD type that the OTel StatsD receiver silently drops. Native OTel counters eliminate this problem entirely (Phase 6 Task 6.1 — DEFERRED becomes resolved). + +4. **Richer metric semantics** — OTel Metrics SDK supports explicit histogram bucket boundaries, exemplars (linking metrics to traces), resource attributes, and metric views. StatsD has no concept of these. + +5. **Removes infrastructure dependency** — No more StatsD receiver needed in the OTel Collector. One less receiver to configure, monitor, and debug. Simplifies the collector YAML. + +6. **Metric-to-trace correlation** — OTel metrics and traces share the same resource attributes (service.name, service.instance.id). Grafana can link from a metric spike directly to the traces that caused it — impossible with StatsD-sourced metrics. + +7. **Production-grade export** — OTel's `PeriodicMetricReader` provides configurable export intervals, batch sizes, timeout handling, and graceful shutdown — all built into the SDK rather than hand-rolled in `StatsDCollectorImp`. + +#### What We Lose + +1. **StatsD ecosystem compatibility** — Operators using external StatsD-compatible backends (Datadog Agent, Graphite, Telegraph) will need to switch to OTLP-compatible backends or keep `server=statsd` as a fallback. + +2. **Simplicity of UDP** — StatsD's UDP fire-and-forget model is dead simple and has zero connection management. OTLP/HTTP requires a TCP connection, TLS negotiation (in production), and retry logic. The OTel SDK handles this, but it's more moving parts. + +3. **Slightly higher memory** — OTel SDK maintains internal aggregation state for metrics before export. StatsD just formats and sends strings. Expected overhead: ~1-2 MB additional for metric state. + +4. **Dependency on OTel C++ Metrics SDK stability** — The Metrics SDK is GA since 1.0 and on version 1.18.0, but it's less battle-tested than the tracing SDK in the C++ ecosystem. + +#### Decision + +The gains (unified pipeline, delivery guarantees, metric-trace correlation, simpler collector config) significantly outweigh the losses. `StatsDCollector` is retained as a fallback via `server=statsd` for operators who need StatsD ecosystem compatibility during the transition period. + +### Architecture + +#### Class Hierarchy (after Phase 7) + +``` +beast::insight::Collector (abstract interface — unchanged) + | + +-- StatsDCollector (existing — retained as fallback, deprecated) + | +-- StatsDCounterImpl -> StatsD |c over UDP + | +-- StatsDGaugeImpl -> StatsD |g over UDP + | +-- StatsDMeterImpl -> StatsD |m over UDP (non-standard) + | +-- StatsDEventImpl -> StatsD |ms over UDP + | +-- StatsDHookImpl -> 1s periodic callback + | + +-- NullCollector (existing — unchanged, used when disabled) + | +-- NullCounterImpl -> no-op + | +-- NullGaugeImpl -> no-op + | +-- NullMeterImpl -> no-op + | +-- NullEventImpl -> no-op + | +-- NullHookImpl -> no-op + | + +-- OTelCollector (NEW — Phase 7) + +-- OTelCounterImpl -> otel::Counter + +-- OTelGaugeImpl -> otel::ObservableGauge + +-- OTelMeterImpl -> otel::Counter + +-- OTelEventImpl -> otel::Histogram + +-- OTelHookImpl -> 1s periodic callback (same pattern) +``` + +#### Data Flow (after Phase 7) + +```mermaid +graph LR + subgraph xrpldNode["xrpld Node"] + A["Trace Macros
XRPL_TRACE_SPAN"] + B["beast::insight
OTelCollector"] + end + + subgraph collector["OTel Collector :4317 / :4318"] + direction TB + R1["OTLP Receiver
:4317 gRPC | :4318 HTTP"] + BP["Batch Processor"] + SM["SpanMetrics Connector"] + + R1 --> BP + BP --> SM + end + + subgraph backends["Trace Backends"] + D["Tempo"] + end + + subgraph metrics["Metrics Stack"] + E["Prometheus :9090
scrapes :8889
span-derived + native OTel metrics"] + end + + subgraph viz["Visualization"] + F["Grafana :3000"] + end + + A -->|"OTLP/HTTP :4318
(traces)"| R1 + B -->|"OTLP/HTTP :4318
(metrics)"| R1 + + BP -->|"OTLP/gRPC"| D + SM -->|"RED metrics"| E + R1 -->|"system metrics
(native OTLP)"| E + + E --> F + D --> F + + style A fill:#4a90d9,color:#fff,stroke:#2a6db5 + style B fill:#d9534f,color:#fff,stroke:#b52d2d + style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BP fill:#449d44,color:#fff,stroke:#2d6e2d + style SM fill:#449d44,color:#fff,stroke:#2d6e2d + style D fill:#f0ad4e,color:#000,stroke:#c78c2e + style E fill:#f0ad4e,color:#000,stroke:#c78c2e + style F fill:#5bc0de,color:#000,stroke:#3aa8c1 + style xrpldNode fill:#1a2633,color:#ccc,stroke:#4a90d9 + style collector fill:#1a3320,color:#ccc,stroke:#5cb85c + style backends fill:#332a1a,color:#ccc,stroke:#f0ad4e + style metrics fill:#332a1a,color:#ccc,stroke:#f0ad4e + style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de +``` + +**Key change**: StatsD receiver removed from collector. Both traces and metrics enter via OTLP receiver on the same port. + +#### Configuration + +```ini +# [insight] section — new "otel" server option +[insight] +server=otel # NEW: uses OTel OTLP metrics exporter +prefix=xrpld # metric name prefix (preserved) + +# Endpoint and auth inherited from [telemetry] section: +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +``` + +The `OTelCollector` reads the OTLP endpoint from `[telemetry]` config (replacing `/v1/traces` with `/v1/metrics` for the metrics exporter). No additional config keys needed. + +**Backward compatibility**: `server=statsd` continues to work exactly as before. + +See [Phase7_taskList.md](./Phase7_taskList.md) for detailed per-task breakdown. + +### Instrument Type Mapping + +| beast::insight | OTel Metrics SDK | Rationale | +| ---------------------- | -------------------------------- | ---------------------------------------------------------------- | +| Counter (int64, `\|c`) | `Counter` | Direct 1:1 mapping | +| Gauge (uint64, `\|g`) | `ObservableGauge` | Async callback matches existing Hook polling pattern | +| Meter (uint64, `\|m`) | `Counter` | Fixes non-standard wire format; meters are semantically counters | +| Event (ms, `\|ms`) | `Histogram` | Duration distributions with explicit bucket boundaries | +| Hook (1s callback) | `PeriodicMetricReader` alignment | Same 1s collection interval | + +### Tasks + +| Task | Description | +| ---- | ------------------------------------------------------------------------- | +| 7.1 | Add OTel Metrics SDK to build deps (conan/cmake) | +| 7.2 | Implement `OTelCollector` class (~400-500 lines) | +| 7.3 | Update `CollectorManager` — add `server=otel` | +| 7.4 | Update OTel Collector YAML (add metrics pipeline, remove StatsD receiver) | +| 7.5 | Preserve metric names in Prometheus (naming strategy) | +| 7.6 | Update Grafana dashboards (if names change) | +| 7.7 | Update integration tests | +| 7.8 | Update documentation (runbook, reference docs) | + +### Exit Criteria + +- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) +- [ ] `server=otel` is the default in development docker-compose +- [ ] `server=statsd` still works as a fallback +- [ ] Existing Grafana dashboards display data correctly +- [ ] Integration test passes with OTLP-only metrics pipeline +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) +- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant + +--- + +## 6.8 Phase 7: Native OTel Metrics Migration (Weeks 11-12) + +**Objective**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency and unifying traces and metrics into a single OTLP pipeline. + +### Motivation: Why Migrate from StatsD to Native OTel Metrics + +The Phase 6 StatsD bridge was a pragmatic first step, but it retains inherent limitations that native OTel export resolves. + +#### What We Gain + +1. **Unified telemetry pipeline** — Traces and metrics export via the same OTLP/HTTP endpoint to the same OTel Collector. One protocol, one endpoint, one config. Eliminates the split-brain architecture of "OTLP for traces, StatsD UDP for metrics." + +2. **Eliminates StatsD UDP limitations** — StatsD is fire-and-forget over UDP with no delivery guarantees, no backpressure, 1472-byte MTU packet fragmentation, and text-based encoding overhead. OTLP uses HTTP/gRPC with retries, binary protobuf encoding, and connection-level flow control. + +3. **Fixes the `|m` wire format issue** — The `StatsDMeterImpl` uses non-standard `|m` StatsD type that the OTel StatsD receiver silently drops. Native OTel counters eliminate this problem entirely (Phase 6 Task 6.1 — DEFERRED becomes resolved). + +4. **Richer metric semantics** — OTel Metrics SDK supports explicit histogram bucket boundaries, exemplars (linking metrics to traces), resource attributes, and metric views. StatsD has no concept of these. + +5. **Removes infrastructure dependency** — No more StatsD receiver needed in the OTel Collector. One less receiver to configure, monitor, and debug. Simplifies the collector YAML. + +6. **Metric-to-trace correlation** — OTel metrics and traces share the same resource attributes (service.name, service.instance.id). Grafana can link from a metric spike directly to the traces that caused it — impossible with StatsD-sourced metrics. + +7. **Production-grade export** — OTel's `PeriodicMetricReader` provides configurable export intervals, batch sizes, timeout handling, and graceful shutdown — all built into the SDK rather than hand-rolled in `StatsDCollectorImp`. + +#### What We Lose + +1. **StatsD ecosystem compatibility** — Operators using external StatsD-compatible backends (Datadog Agent, Graphite, Telegraph) will need to switch to OTLP-compatible backends or keep `server=statsd` as a fallback. + +2. **Simplicity of UDP** — StatsD's UDP fire-and-forget model is dead simple and has zero connection management. OTLP/HTTP requires a TCP connection, TLS negotiation (in production), and retry logic. The OTel SDK handles this, but it's more moving parts. + +3. **Slightly higher memory** — OTel SDK maintains internal aggregation state for metrics before export. StatsD just formats and sends strings. Expected overhead: ~1-2 MB additional for metric state. + +4. **Dependency on OTel C++ Metrics SDK stability** — The Metrics SDK is GA since 1.0 and on version 1.18.0, but it's less battle-tested than the tracing SDK in the C++ ecosystem. + +#### Decision + +The gains (unified pipeline, delivery guarantees, metric-trace correlation, simpler collector config) significantly outweigh the losses. `StatsDCollector` is retained as a fallback via `server=statsd` for operators who need StatsD ecosystem compatibility during the transition period. + +### Architecture + +#### Class Hierarchy (after Phase 7) + +``` +beast::insight::Collector (abstract interface — unchanged) + | + +-- StatsDCollector (existing — retained as fallback, deprecated) + | +-- StatsDCounterImpl -> StatsD |c over UDP + | +-- StatsDGaugeImpl -> StatsD |g over UDP + | +-- StatsDMeterImpl -> StatsD |m over UDP (non-standard) + | +-- StatsDEventImpl -> StatsD |ms over UDP + | +-- StatsDHookImpl -> 1s periodic callback + | + +-- NullCollector (existing — unchanged, used when disabled) + | +-- NullCounterImpl -> no-op + | +-- NullGaugeImpl -> no-op + | +-- NullMeterImpl -> no-op + | +-- NullEventImpl -> no-op + | +-- NullHookImpl -> no-op + | + +-- OTelCollector (NEW — Phase 7) + +-- OTelCounterImpl -> otel::Counter + +-- OTelGaugeImpl -> otel::ObservableGauge + +-- OTelMeterImpl -> otel::Counter + +-- OTelEventImpl -> otel::Histogram + +-- OTelHookImpl -> 1s periodic callback (same pattern) +``` + +#### Data Flow (after Phase 7) + +```mermaid +graph LR + subgraph xrpldNode["xrpld Node"] + A["Trace Macros
XRPL_TRACE_SPAN"] + B["beast::insight
OTelCollector"] + end + + subgraph collector["OTel Collector :4317 / :4318"] + direction TB + R1["OTLP Receiver
:4317 gRPC | :4318 HTTP"] + BP["Batch Processor"] + SM["SpanMetrics Connector"] + + R1 --> BP + BP --> SM + end + + subgraph backends["Trace Backends"] + D["Jaeger / Tempo"] + end + + subgraph metrics["Metrics Stack"] + E["Prometheus :9090
scrapes :8889
span-derived + native OTel metrics"] + end + + subgraph viz["Visualization"] + F["Grafana :3000"] + end + + A -->|"OTLP/HTTP :4318
(traces)"| R1 + B -->|"OTLP/HTTP :4318
(metrics)"| R1 + + BP -->|"OTLP/gRPC"| D + SM -->|"RED metrics"| E + R1 -->|"system metrics
(native OTLP)"| E + + E --> F + D --> F + + style A fill:#4a90d9,color:#fff,stroke:#2a6db5 + style B fill:#d9534f,color:#fff,stroke:#b52d2d + style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BP fill:#449d44,color:#fff,stroke:#2d6e2d + style SM fill:#449d44,color:#fff,stroke:#2d6e2d + style D fill:#f0ad4e,color:#000,stroke:#c78c2e + style E fill:#f0ad4e,color:#000,stroke:#c78c2e + style F fill:#5bc0de,color:#000,stroke:#3aa8c1 + style xrpldNode fill:#1a2633,color:#ccc,stroke:#4a90d9 + style collector fill:#1a3320,color:#ccc,stroke:#5cb85c + style backends fill:#332a1a,color:#ccc,stroke:#f0ad4e + style metrics fill:#332a1a,color:#ccc,stroke:#f0ad4e + style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de +``` + +**Key change**: StatsD receiver removed from collector. Both traces and metrics enter via OTLP receiver on the same port. + +#### Configuration + +```ini +# [insight] section — new "otel" server option +[insight] +server=otel # NEW: uses OTel OTLP metrics exporter +prefix=xrpld # metric name prefix (preserved) + +# Endpoint and auth inherited from [telemetry] section: +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +``` + +The `OTelCollector` reads the OTLP endpoint from `[telemetry]` config (replacing `/v1/traces` with `/v1/metrics` for the metrics exporter). No additional config keys needed. + +**Backward compatibility**: `server=statsd` continues to work exactly as before. + +See [Phase7_taskList.md](./Phase7_taskList.md) for detailed per-task breakdown. + +### Instrument Type Mapping + +| beast::insight | OTel Metrics SDK | Rationale | +| ---------------------- | -------------------------------- | ---------------------------------------------------------------- | +| Counter (int64, `\|c`) | `Counter` | Direct 1:1 mapping | +| Gauge (uint64, `\|g`) | `ObservableGauge` | Async callback matches existing Hook polling pattern | +| Meter (uint64, `\|m`) | `Counter` | Fixes non-standard wire format; meters are semantically counters | +| Event (ms, `\|ms`) | `Histogram` | Duration distributions with explicit bucket boundaries | +| Hook (1s callback) | `PeriodicMetricReader` alignment | Same 1s collection interval | + +### Tasks + +| Task | Description | +| ---- | ------------------------------------------------------------------------- | +| 7.1 | Add OTel Metrics SDK to build deps (conan/cmake) | +| 7.2 | Implement `OTelCollector` class (~400-500 lines) | +| 7.3 | Update `CollectorManager` — add `server=otel` | +| 7.4 | Update OTel Collector YAML (add metrics pipeline, remove StatsD receiver) | +| 7.5 | Preserve metric names in Prometheus (naming strategy) | +| 7.6 | Update Grafana dashboards (if names change) | +| 7.7 | Update integration tests | +| 7.8 | Update documentation (runbook, reference docs) | + +### Exit Criteria + +- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) +- [ ] `server=otel` is the default in development docker-compose +- [ ] `server=statsd` still works as a fallback +- [ ] Existing Grafana dashboards display data correctly +- [ ] Integration test passes with OTLP-only metrics pipeline +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) +- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant + +--- + +## 6.9 Phase 8: Log-Trace Correlation and Centralized Log Ingestion (Week 13) + +### Motivation + +xrpld's `beast::Journal` logs and OpenTelemetry traces are currently two disjoint observability signals. When investigating an issue, operators must manually correlate timestamps between log files and Tempo traces. Phase 8 bridges this gap by injecting trace context (`trace_id`, `span_id`) into every log line emitted within an active span, and ingesting those logs into Grafana Loki via the OTel Collector's filelog receiver. + +#### Gains + +1. **One-click trace-to-log navigation** — Click a trace in Tempo and immediately see the corresponding log lines in Loki, filtered by `trace_id`. +2. **Reverse lookup (log-to-trace)** — Loki derived fields make `trace_id` values clickable links back to Tempo. +3. **Unified observability** — All three pillars (traces, metrics, logs) flow through the same OTel Collector pipeline and are visible in a single Grafana instance. +4. **Zero new dependencies in xrpld** — Uses existing OTel SDK headers (`GetSpan`, `GetContext`) already linked in Phase 1. +5. **Negligible overhead** — The implementation checks the thread-local context value directly, avoiding heap allocation on the no-span path (~15-20ns). On the active-span path, total cost is ~50ns per log call. At typical logging rates, overhead is negligible. + +#### Losses / Risks + +1. **Log format change** — Existing log parsers that rely on a fixed format will need updating to handle the optional `trace_id=... span_id=...` fields. +2. **Loki resource usage** — Log ingestion adds storage and memory overhead to the observability stack (mitigated by retention policies). +3. **Filelog receiver complexity** — The regex parser must be kept in sync with the log format; a format change in `Logs::format()` could break parsing. + +#### Decision + +The correlation value far outweighs the risks. The log format change is backward-compatible (fields are appended only when a span is active), and the filelog receiver regex is straightforward to maintain. + +### Architecture + +Phase 8 has two independent sub-phases that can be developed in parallel: + +- **Phase 8a (code change)**: Modify `Logs::format()` in `src/libxrpl/basics/Log.cpp` to append `trace_id= span_id=` when the current thread has an active OTel span. Guarded by `#ifdef XRPL_ENABLE_TELEMETRY`. +- **Phase 8b (infra only)**: Add Loki to the Docker Compose stack, configure the OTel Collector's `filelog` receiver to tail xrpld's log file, parse out structured fields (timestamp, partition, severity, trace_id, span_id, message), and export to Loki via OTLP. Configure Grafana Tempo↔Loki bidirectional linking. + +#### Trace ID Injection Flow + +```mermaid +flowchart LR + subgraph xrpld["xrpld process"] + JLOG["JLOG(j.info())"] + Format["Logs::format()"] + OTelCtx["OTel Context
(thread-local)"] + JLOG --> Format + OTelCtx -.->|"GetSpan()→GetContext()"| Format + end + + subgraph output["Log Output"] + LogLine["2024-01-15T10:30:45.123Z
LedgerMaster:NFO
trace_id=abc123...
span_id=def456...
Validated ledger 42"] + end + + Format --> LogLine + + style xrpld fill:#1a237e,stroke:#0d1642,color:#fff + style output fill:#1b5e20,stroke:#0d3d14,color:#fff + style JLOG fill:#283593,stroke:#1a237e,color:#fff + style Format fill:#283593,stroke:#1a237e,color:#fff + style OTelCtx fill:#283593,stroke:#1a237e,color:#fff + style LogLine fill:#2e7d32,stroke:#1b5e20,color:#fff +``` + +#### Loki Ingestion Pipeline + +```mermaid +flowchart LR + subgraph collector["OTel Collector"] + FR["filelog receiver
tails debug.log"] + RP["regex_parser
extracts trace_id,
span_id, severity"] + BP["batch processor"] + LE["otlp/loki exporter"] + FR --> RP --> BP --> LE + end + + LogFile["xrpld
debug.log"] --> FR + LE --> Loki["Grafana Loki
:3100"] + Loki <-->|"derivedFields ↔
tracesToLogs"| Tempo["Grafana Tempo"] + + style collector fill:#e65100,stroke:#bf360c,color:#fff + style FR fill:#f57c00,stroke:#e65100,color:#fff + style RP fill:#f57c00,stroke:#e65100,color:#fff + style BP fill:#f57c00,stroke:#e65100,color:#fff + style LE fill:#f57c00,stroke:#e65100,color:#fff + style LogFile fill:#1a237e,stroke:#0d1642,color:#fff + style Loki fill:#4a148c,stroke:#2e0d57,color:#fff + style Tempo fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +### Tasks + +| Task | Description | +| ---- | ---------------------------------------------- | +| 8.1 | Inject trace_id into Logs::format() | +| 8.2 | Add Loki to Docker Compose stack | +| 8.3 | Add filelog receiver to OTel Collector | +| 8.4 | Configure Grafana trace-to-log correlation | +| 8.5 | Update integration tests | +| 8.6 | Update documentation (runbook, reference docs) | + +**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3–8.6 are sequential. + +### Exit Criteria + +- [ ] Log lines within active spans contain `trace_id= span_id=` +- [ ] Log lines outside spans have no trace context (no empty fields) +- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver +- [ ] Grafana Tempo → Loki one-click correlation works +- [ ] Grafana Loki → Tempo reverse lookup works via derived field +- [ ] Integration test verifies trace_id presence in logs +- [ ] No performance regression from trace_id injection (< 0.1% overhead) + +--- + +## 6.8.2 Phase 9: Internal Metric Instrumentation Gap Fill (Weeks 14-15) — Future Enhancement + +> **Status**: Planned, not yet implemented. + +### Motivation + +Phases 1-8 establish trace spans, StatsD metrics bridge, native OTel metrics, and log-trace correlation. However, ~68 metrics that exist inside xrpld's `get_counts`, `server_info`, TxQ, PerfLog, and `CountedObject` systems have **no time-series export path**. These are the metrics that exchanges, payment processors, analytics providers, validators, and researchers need most — NodeStore I/O performance, cache hit rates, per-RPC-method counters, transaction queue depth, fee escalation levels, and live object instance counts. + +### Architecture + +Hybrid approach — two instrumentation strategies based on proximity to existing code: + +```mermaid +flowchart TB + subgraph xrpld["xrpld process"] + subgraph existing["Existing beast::insight registrations"] + NS["NodeStore I/O
(Database.cpp)"] + end + subgraph newreg["New OTel MetricsRegistry"] + CR["Cache Hit Rates
(async gauge callbacks)"] + TQ["TxQ Metrics
(async gauge callbacks)"] + PL["PerfLog RPC/Job
(counters + histograms)"] + CO["CountedObjects
(async gauge callbacks)"] + LF["Load Factors
(async gauge callbacks)"] + end + end + + subgraph export["Export Pipelines"] + BI["beast::insight
OTelCollector (Phase 7)"] + OS["OTel Metrics SDK
PeriodicMetricReader"] + end + + NS --> BI + CR --> OS + TQ --> OS + PL --> OS + CO --> OS + LF --> OS + + BI --> OTLP["OTLP/HTTP :4318
/v1/metrics"] + OS --> OTLP + + style xrpld fill:#1a2633,color:#ccc,stroke:#4a90d9 + style existing fill:#2a4a6b,color:#fff,stroke:#4a90d9 + style newreg fill:#2a4a6b,color:#fff,stroke:#4a90d9 + style export fill:#1a3320,color:#ccc,stroke:#5cb85c + style NS fill:#4a90d9,color:#fff,stroke:#2a6db5 + style CR fill:#5cb85c,color:#fff,stroke:#3d8b3d + style TQ fill:#5cb85c,color:#fff,stroke:#3d8b3d + style PL fill:#5cb85c,color:#fff,stroke:#3d8b3d + style CO fill:#5cb85c,color:#fff,stroke:#3d8b3d + style LF fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BI fill:#449d44,color:#fff,stroke:#2d6e2d + style OS fill:#449d44,color:#fff,stroke:#2d6e2d + style OTLP fill:#f0ad4e,color:#000,stroke:#c78c2e +``` + +- **beast::insight extensions** (blue): NodeStore I/O metrics added near existing `Database.cpp` registrations — exported via Phase 7's `OTelCollector`. +- **OTel MetricsRegistry** (green): New centralized class using `ObservableGauge` async callbacks for cache, TxQ, PerfLog, CountedObjects, and load factors — polled at 10s intervals by `PeriodicMetricReader`. + +### Third-Party Consumer Context + +| Consumer Category | Key Metrics They Need From Phase 9 | +| ---------------------- | --------------------------------------------------------------- | +| Exchanges | Fee escalation levels, TxQ depth, settlement latency | +| Payment Processors | Load factors, io_latency, transaction throughput | +| Analytics Providers | NodeStore I/O, cache hit rates, counted objects | +| Validators / Operators | Per-job execution times, PerfLog RPC counters, consensus timing | +| Academic Researchers | Consensus performance time-series, fee market dynamics | +| Institutional Custody | Server health scores, reserve calculations, node availability | + +### Tasks + +| Task | Description | +| ---- | ----------------------------------------- | +| 9.1 | NodeStore I/O metrics | +| 9.2 | Cache hit rate metrics + MetricsRegistry | +| 9.3 | TxQ metrics | +| 9.4 | PerfLog per-RPC metrics | +| 9.5 | PerfLog per-job metrics | +| 9.6 | Counted object instance metrics | +| 9.7 | Fee escalation & load factor metrics | +| 9.7a | push_metrics.py parity gauges | +| 9.8 | New Grafana dashboards (2 new, 2 updated) | +| 9.9 | Update documentation | +| 9.10 | Integration tests | + +See [Phase9_taskList.md](./Phase9_taskList.md) for detailed per-task breakdown. + +### Exit Criteria + +- [ ] All ~68 new metrics visible in Prometheus via OTLP pipeline +- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK +- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue) +- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) +- [ ] Documentation updated with full new metric inventory + +--- + +## 6.8.3 Phase 10: Synthetic Workload Generation & Telemetry Validation (Weeks 16-17) + +> **Status**: In progress. + +### Motivation + +Before the telemetry stack (Phases 1-9) can be considered production-ready, we need automated proof that all spans, attributes, metrics, Grafana dashboards, and log-trace correlation work correctly under realistic load. This phase establishes a reusable CI-integrated validation suite and performance benchmark baseline. + +### Architecture + +The validation uses a **2-node** validator cluster running as local processes alongside a Docker Compose telemetry stack (Collector, Tempo, Prometheus, Grafana). Two nodes are sufficient for consensus rounds and peer-to-peer span validation while minimizing CI resource usage. + +```mermaid +flowchart LR + subgraph harness["2-Node Validator Cluster (local processes)"] + direction TB + V1["Validator 1"] ~~~ V2["Validator 2"] + end + + subgraph telemetry["Docker Compose Telemetry Stack"] + direction TB + COL["OTel Collector
(OTLP + StatsD)"] + JAE["Tempo
(trace search)"] + PROM["Prometheus
(metrics)"] + GRAF["Grafana
(dashboards)"] + end + + subgraph generators["Workload Generators"] + RPC["RPC Load Generator
(configurable RPS,
command distribution)"] + TX["Transaction Submitter
(10 tx types via
WebSocket command API)"] + end + + subgraph validation["Validation Suite"] + SV["Span Validator
(Tempo API)"] + MV["Metric Validator
(Prometheus API,
all 26 metrics required)"] + DV["Dashboard Validator
(Grafana API)"] + BM["Benchmark Suite
(CPU, memory, latency
ON vs OFF comparison)"] + end + + generators --> harness + harness --> telemetry + telemetry --> validation + + style harness fill:#1a2633,color:#ccc,stroke:#4a90d9 + style telemetry fill:#1a2633,color:#ccc,stroke:#4a90d9 + style generators fill:#1a3320,color:#ccc,stroke:#5cb85c + style validation fill:#332a1a,color:#ccc,stroke:#f0ad4e + style V1 fill:#4a90d9,color:#fff,stroke:#2a6db5 + style V2 fill:#4a90d9,color:#fff,stroke:#2a6db5 + style COL fill:#4a90d9,color:#fff,stroke:#2a6db5 + style JAE fill:#4a90d9,color:#fff,stroke:#2a6db5 + style PROM fill:#4a90d9,color:#fff,stroke:#2a6db5 + style GRAF fill:#4a90d9,color:#fff,stroke:#2a6db5 + style RPC fill:#5cb85c,color:#fff,stroke:#3d8b3d + style TX fill:#5cb85c,color:#fff,stroke:#3d8b3d + style SV fill:#f0ad4e,color:#000,stroke:#c78c2e + style MV fill:#f0ad4e,color:#000,stroke:#c78c2e + style DV fill:#f0ad4e,color:#000,stroke:#c78c2e + style BM fill:#f0ad4e,color:#000,stroke:#c78c2e +``` + +### Key Implementation Details + +- **Transaction submitter and RPC load generator** both use xrpld's native WebSocket command format (`{"command": ...}`) — not JSON-RPC format. Response data lives inside `"result"` with `"status"` at the top level. +- **Node config** requires `[signing_support] true` for server-side signing, and `[ips]` (not `[ips_fixed]`) to ensure peer connections count in `peer_finder_active_*` metrics. +- **Metric validation** uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale StatsD gauges. Every metric in `expected_metrics.json` must have > 0 series. +- **StatsD gauge fix**: `StatsDGaugeImpl` initializes `m_dirty = true` so all gauges emit their initial value on first flush. Without this, gauges starting at 0 that never change (e.g. `jobq_job_count`) would be invisible in Prometheus. +- **I/O latency fix**: `io_latency_sampler` emits unconditionally on first sample, then applies the 10 ms threshold. This ensures `ios_latency` is registered in Prometheus even in low-load CI environments. +- **tx.receive span**: Sets default attributes (`xrpl.tx.suppressed = false`, `xrpl.tx.status = "new"`) on span creation so they are always present. The suppressed/bad code paths override these when applicable. + +### Tasks + +| Task | Description | +| ---- | -------------------------------------- | +| 10.1 | Multi-node test harness (5 validators) | +| 10.2 | RPC load generator | +| 10.3 | Transaction submitter (6+ tx types) | +| 10.4 | Telemetry validation suite | +| 10.5 | Performance benchmark suite | +| 10.6 | CI integration | +| 10.7 | Documentation | + +See [Phase10_taskList.md](./Phase10_taskList.md) for detailed per-task breakdown. + +### Validation Check Inventory (71 Checks) + +The validation suite (`validate_telemetry.py`) runs exactly 71 checks, broken down as: + +- **1 service registration** — `xrpld` exists in Tempo +- **17 span existence** — `rpc.request`, `rpc.process`, `rpc.ws_message`, `rpc.command.*`, `tx.process`, `tx.receive`, `tx.apply`, `consensus.proposal.send`, `consensus.ledger_close`, `consensus.accept`, `consensus.validation.send`, `consensus.accept.apply`, `ledger.build`, `ledger.validate`, `ledger.store`, `peer.proposal.receive`, `peer.validation.receive` +- **14 span attribute** — required attributes on the 14 spans that define them (22 unique attributes total) +- **2 span hierarchies** — `rpc.process` -> `rpc.command.*`, `ledger.build` -> `tx.apply` (1 skipped: `rpc.request` -> `rpc.process`, cross-thread) +- **1 span duration bounds** — all spans > 0 and < 60 s +- **26 metric existence** — 4 SpanMetrics (`span_calls_total`, `span_duration_milliseconds_{bucket,count,sum}`), 6 StatsD gauges (`ledgermaster_validated_ledger_age`, `published_ledger_age`, `state_accounting_full_duration`, `peer_finder_active_{inbound,outbound}_peers`, `jobq_job_count`), 2 StatsD counters (`rpc_requests_total`, `ledger_fetches_total`), 3 StatsD histograms (`rpc_time`, `rpc_size`, `ios_latency`), 4 overlay traffic (`total_bytes_{in,out}`, `total_messages_{in,out}`), 7 Phase 9 OTLP (`nodestore_state`, `cache_metrics`, `txq_metrics`, `rpc_method_{started,finished}_total`, `object_count`, `load_factor_metrics`) +- **10 dashboard loads** — `rpc-performance`, `transaction-overview`, `consensus-health`, `ledger-operations`, `peer-network`, `node-health`, `network-traffic`, `rpc-pathfinding`, `overlay-traffic-detail`, `ledger-data-sync` + +See [Phase10_taskList.md](./Phase10_taskList.md) for the full numbered check-by-check enumeration. + +### Current Status + +**Working** (71/71 checks pass in CI): +All 17 spans, 26 metrics, 10 dashboards, 14 attribute checks, 2 hierarchies, and duration bounds validated. + +**Not implemented or not available in CI**: + +1. `rpc.request` -> `rpc.process` parent-child hierarchy — skipped (cross-thread context propagation) +2. Log-trace correlation validation (Loki) — not included in checks +3. Full 255+ StatsD metric coverage — only 26 representative metrics validated +4. Sustained load / backpressure testing — not implemented +5. `docs/telemetry-runbook.md` updates — not done +6. `09-data-collection-reference.md` "Validation" section — not done +7. **Automated cross-CI baseline persistence** — the regression gate reads a + committed baseline; baseline updates flow through a manual PR refresh, not + an artifact promoted from `develop` (FU-2). + +### Exit Criteria + +- [x] 2-node validator cluster starts and reaches consensus +- [x] Validation suite confirms all required spans, attributes, and metrics (71/71 checks) +- [x] All 10 Grafana dashboards render data +- [ ] Benchmark shows < 3% CPU overhead, < 5MB memory overhead +- [x] CI workflow runs validation on telemetry branch changes +- [x] OTel-driven regression gate: captures per-span/per-RPC/per-job timings + from Prometheus and compares against a committed baseline + +--- + +## 6.8.4 Phase 11: Third-Party Data Collection Pipelines (Weeks 18-20) — Future Enhancement + +> **Status**: Planned, not yet implemented. + +### Motivation + +xrpld has no native Prometheus/OTLP metrics export for data accessible only via JSON-RPC (`server_info`, `get_counts`, `fee`, `peers`, `validators`, `feature`). Every external consumer — exchanges, payment processors, analytics providers, validators, compliance firms, DeFi protocols, researchers, custodians, and CBDC platforms — must build custom JSON-RPC polling and conversion pipelines. This phase centralizes that work into a reusable custom OTel Collector receiver. + +### Architecture + +```mermaid +flowchart LR + subgraph receiver["Custom OTel Collector Receiver (Go)"] + direction TB + SI["server_info
collector"] + GC["get_counts
collector"] + FE["fee
collector"] + PE["peers
collector"] + VA["validators
collector"] + DX["DEX/AMM
collector
(optional)"] + end + + xrpld["xrpld
Admin RPC
:5005"] -->|"JSON-RPC
poll every 30s"| receiver + + receiver -->|"xrpl_* metrics"| PROM["Prometheus
:9090"] + receiver -->|"OTLP export"| OTLP["Any OTLP-
compatible
backend"] + + PROM --> GF["Grafana
4 new dashboards"] + PROM --> AL["Prometheus
Alerting Rules"] + + style receiver fill:#1a3320,color:#ccc,stroke:#5cb85c + style SI fill:#5cb85c,color:#fff,stroke:#3d8b3d + style GC fill:#5cb85c,color:#fff,stroke:#3d8b3d + style FE fill:#5cb85c,color:#fff,stroke:#3d8b3d + style PE fill:#5cb85c,color:#fff,stroke:#3d8b3d + style VA fill:#5cb85c,color:#fff,stroke:#3d8b3d + style DX fill:#449d44,color:#fff,stroke:#2d6e2d + style xrpld fill:#4a90d9,color:#fff,stroke:#2a6db5 + style PROM fill:#f0ad4e,color:#000,stroke:#c78c2e + style OTLP fill:#f0ad4e,color:#000,stroke:#c78c2e + style GF fill:#5bc0de,color:#000,stroke:#3aa8c1 + style AL fill:#d9534f,color:#fff,stroke:#b52d2d +``` + +### Third-Party Consumer Gap Analysis + +| Consumer Category | Data Unlocked by Phase 11 | +| ---------------------- | ------------------------------------------------------------ | +| Exchanges | Real-time fee estimates, TxQ capacity, server health scores | +| Payment Processors | Settlement latency percentiles, corridor health | +| Analytics Providers | Validator metrics, network topology, amendment voting status | +| DeFi / AMM | AMM pool TVL, DEX order book depth, trade volumes | +| Validators / Operators | Per-peer latency, version distribution, UNL health, alerting | +| Compliance | Transaction volume trends, network growth metrics | +| Academic Researchers | Consensus performance time-series, decentralization metrics | +| CBDC / Tokenization | Token supply tracking, trust line adoption, freeze status | +| Institutional Custody | Multi-sig status, escrow tracking, reserve calculations | +| Wallet Providers | Server health for node selection, fee prediction data | + +### Tasks + +| Task | Description | +| ----- | ------------------------------------- | +| 11.1 | OTel Collector receiver scaffold (Go) | +| 11.2 | server_info / server_state collector | +| 11.3 | get_counts collector | +| 11.4 | Peer topology collector | +| 11.5 | Validator & amendment collector | +| 11.6 | Fee & TxQ collector | +| 11.7 | DEX & AMM collector (optional) | +| 11.8 | Prometheus alerting rules | +| 11.9 | New Grafana dashboards (4) | +| 11.10 | Integration with Phase 10 validation | +| 11.11 | Documentation | + +See [Phase11_taskList.md](./Phase11_taskList.md) for detailed per-task breakdown. + +### Exit Criteria + +- [ ] Custom OTel Collector receiver exports all `xrpl_*` metrics to Prometheus +- [ ] 4 new Grafana dashboards operational (Validator Health, Network Topology, Fee Market, DEX/AMM) +- [ ] Prometheus alerting rules fire correctly for simulated failures +- [ ] Receiver handles xrpld restart/unavailability gracefully +- [ ] Go receiver has unit tests with >80% coverage + +--- + +## 6.9 Risk Assessment + +```mermaid +quadrantChart + title Risk Assessment Matrix + x-axis Low Impact --> High Impact + y-axis Low Likelihood --> High Likelihood + quadrant-1 Mitigate Immediately + quadrant-2 Plan Mitigation + quadrant-3 Accept Risk + quadrant-4 Monitor Closely + + SDK Compat: [0.2, 0.18] + Protocol Chg: [0.75, 0.72] + Perf Overhead: [0.58, 0.42] + Context Prop: [0.4, 0.55] + Memory Leaks: [0.85, 0.25] +``` + +### Risk Details + +| Risk | Likelihood | Impact | Mitigation | +| ------------------------------------ | ---------- | ------ | --------------------------------------- | +| Protocol changes break compatibility | Medium | High | Use high field numbers, optional fields | +| Performance overhead unacceptable | Medium | Medium | Sampling, conditional compilation | +| Context propagation complexity | Medium | Medium | Phased rollout, extensive testing | +| SDK compatibility issues | Low | Medium | Pin SDK version, fallback to no-op | +| Memory leaks in long-running nodes | Low | High | Memory profiling, bounded queues | + +--- + +## 6.10 Success Metrics + +| Metric | Target | Measurement | +| ------------------------ | -------------------------------------------------------------- | --------------------- | +| Trace coverage | >95% of transaction code paths (independent of sampling ratio) | Sampling verification | +| CPU overhead | <3% | Benchmark tests | +| Memory overhead | <10 MB | Memory profiling | +| Latency impact (p99) | <2% | Performance tests | +| Trace completeness | >99% spans with required attrs | Validation script | +| Cross-node trace linkage | >90% of multi-hop transactions | Integration tests | + +--- + +## 6.11 Quick Wins and Crawl-Walk-Run Strategy + +> **TxQ** = Transaction Queue + +This section outlines a prioritized approach to maximize ROI with minimal initial investment. + +### 6.11.1 Crawl-Walk-Run Overview + +
+ +```mermaid +flowchart TB + subgraph crawl["🐢 CRAWL (Week 1-2)"] + direction LR + c1[Core SDK Setup] ~~~ c2[RPC Tracing Only] ~~~ c3[PathFinding + TxQ Tracing] ~~~ c4[Single Node] + end + + subgraph walk["🚶 WALK (Week 3-5)"] + direction LR + w1[Transaction Tracing] ~~~ w2[Fee Escalation Tracing] ~~~ w3[Cross-Node Context] ~~~ w4[Basic Dashboards] + end + + subgraph run["🏃 RUN (Week 6-9)"] + direction LR + r1[Consensus Tracing] ~~~ r2[Establish Phase
& Cross-Node Correlation] ~~~ r3[StatsD Integration] ~~~ r4[Production Deploy] + end + + crawl --> walk --> run + + style crawl fill:#1b5e20,stroke:#0d3d14,color:#fff + style walk fill:#bf360c,stroke:#8c2809,color:#fff + style run fill:#0d47a1,stroke:#082f6a,color:#fff + style c1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style c2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style c3 fill:#1b5e20,stroke:#0d3d14,color:#fff + style c4 fill:#1b5e20,stroke:#0d3d14,color:#fff + style w1 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style w2 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style w3 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style w4 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style r1 fill:#0d47a1,stroke:#082f6a,color:#fff + style r2 fill:#0d47a1,stroke:#082f6a,color:#fff + style r3 fill:#0d47a1,stroke:#082f6a,color:#fff + style r4 fill:#0d47a1,stroke:#082f6a,color:#fff +``` + +
+ +**Reading the diagram:** + +- **CRAWL (Weeks 1-2)**: Minimal investment -- set up the SDK, instrument RPC and PathFinding/TxQ handlers, and verify on a single node. Delivers immediate latency visibility. +- **WALK (Weeks 3-5)**: Expand to transaction lifecycle tracing, fee escalation, cross-node context propagation, and basic Grafana dashboards. This is where distributed tracing starts working. +- **RUN (Weeks 6-9)**: Full consensus instrumentation, establish-phase gap fill, cross-node correlation, StatsD integration, and production deployment with sampling and alerting. +- **Arrows (crawl → walk → run)**: Each phase builds on the prior one; you cannot skip ahead because later phases depend on infrastructure established earlier. + +### 6.11.2 Quick Wins (Immediate Value) + +| Quick Win | Value | When to Deploy | +| ------------------------------ | ------ | -------------- | +| **RPC Command Tracing** | High | Week 2 | +| **RPC Latency Histograms** | High | Week 2 | +| **Error Rate Dashboard** | Medium | Week 2 | +| **Transaction Submit Tracing** | High | Week 3 | +| **Consensus Round Duration** | Medium | Week 6 | + +### 6.11.3 CRAWL Phase (Weeks 1-2) + +**Goal**: Get basic tracing working with minimal code changes. + +**What You Get**: + +- RPC request/response traces for all commands +- Latency breakdown per RPC command +- PathFinding and TxQ tracing (directly impacts RPC latency) +- Error visibility with stack traces +- Basic Grafana dashboard + +**Code Changes**: ~15 lines in `ServerHandler.cpp`, ~40 lines in new telemetry module + +**Why Start Here**: + +- RPC is the lowest-risk, highest-visibility component +- PathFinding and TxQ are RPC-adjacent and directly affect latency +- Immediate value for debugging client issues +- No cross-node complexity +- Single file modification to existing code + +### 6.11.4 WALK Phase (Weeks 3-5) + +**Goal**: Add transaction lifecycle tracing across nodes. + +**What You Get**: + +- End-to-end transaction traces from submit to relay +- Fee escalation tracing within the transaction pipeline +- Cross-node correlation (see transaction path) +- HashRouter deduplication visibility +- Relay latency metrics + +**Code Changes**: ~120 lines across 4 files, plus protobuf extension + +**Why Do This Second**: + +- Builds on RPC tracing (transactions submitted via RPC) +- Fee escalation is integral to the transaction processing pipeline +- Moderate complexity (requires context propagation) +- High value for debugging transaction issues + +### 6.11.5 RUN Phase (Weeks 6-9) + +**Goal**: Full observability including consensus. + +**What You Get**: + +- Complete consensus round visibility +- Phase transition timing +- Validator proposal tracking +- ~~Validator list and manifest tracing~~ — descoped +- ~~Amendment voting tracing~~ — descoped +- ~~SHAMap sync tracing~~ — descoped +- Full end-to-end traces (client → RPC → TX → consensus → ledger) — partial (tx-consensus correlation not yet done) + +**Code Changes**: ~100 lines across 3 consensus files + +**Why Do This Last**: + +- Highest complexity (consensus is critical path) +- Validator, amendment, and SHAMap components were descoped (lower priority) +- Requires thorough testing +- Lower relative value (consensus issues are rarer) + +### 6.11.6 ROI Prioritization Matrix + +```mermaid +quadrantChart + title Implementation ROI Matrix + x-axis Low Effort --> High Effort + y-axis Low Value --> High Value + quadrant-1 Quick Wins - Do First + quadrant-2 Major Projects - Plan Carefully + quadrant-3 Nice to Have - Optional + quadrant-4 Time Sinks - Avoid + + RPC Tracing: [0.15, 0.92] + TX Submit Trace: [0.3, 0.78] + TX Relay Trace: [0.5, 0.88] + Consensus Trace: [0.72, 0.72] + Peer Msg Trace: [0.85, 0.3] + Ledger Acquire: [0.55, 0.52] +``` + +--- + +## 6.12 Definition of Done + +> **TxQ** = Transaction Queue | **HA** = High Availability + +Clear, measurable criteria for each phase. + +### 6.12.1 Phase 1: Core Infrastructure + +| Criterion | Measurement | Target | +| --------------- | ---------------------------------------------------------- | ---------------------------- | +| SDK Integration | `cmake --build` succeeds with `-DXRPL_ENABLE_TELEMETRY=ON` | ✅ Compiles | +| Runtime Toggle | `enabled=0` produces zero overhead | <0.1% CPU difference | +| Span Creation | Unit test creates and exports span | Span appears in Tempo | +| Configuration | All config options parsed correctly | Config validation tests pass | +| Documentation | Developer guide exists | PR approved | + +**Definition of Done**: All criteria met, PR merged, no regressions in CI. + +### 6.12.2 Phase 2: RPC Tracing + +| Criterion | Measurement | Target | +| ------------------ | ---------------------------------- | -------------------------- | +| Coverage | All RPC commands instrumented | 100% of commands | +| Context Extraction | traceparent header propagates | Integration test passes | +| Attributes | Command, status, duration recorded | Validation script confirms | +| Performance | RPC latency overhead | <1ms p99 | +| Dashboard | Grafana dashboard deployed | Screenshot in docs | + +**Definition of Done**: RPC traces visible in Tempo for all commands, dashboard shows latency distribution. + +### 6.12.3 Phase 3: Transaction Tracing + +| Criterion | Measurement | Target | +| --------------------- | ------------------------------------------------- | -------------------------------------------------------- | +| Local Trace | Submit → validate → TxQ traced | Single-node test passes | +| Cross-Node | Context propagates via protobuf | Multi-node test passes | +| Deterministic TraceID | Same trace_id on all nodes for same tx | Multi-node test: query by txHash[0:16] returns all spans | +| Relay Ordering | Protobuf span_id propagation creates parent-child | Tempo trace tree shows relay chain | +| Graceful Degradation | Old peer drops trace_context | Spans still grouped by deterministic trace_id | +| Relay Visibility | relay_count attribute correct | Spot check 100 txs | +| HashRouter | Deduplication visible in trace | Duplicate txs show suppressed=true | +| Performance | TX throughput overhead | <5% degradation | + +**Definition of Done**: Transaction traces span 3+ nodes in test network with deterministic trace_id correlation, parent-child ordering via protobuf propagation, and performance within bounds. + +### 6.12.4 Phase 4: Consensus Tracing + +| Criterion | Measurement | Target | +| -------------------- | ----------------------------- | ------------------------- | +| Round Tracing | startRound creates root span | Unit test passes | +| Phase Visibility | All phases have child spans | Integration test confirms | +| Proposer Attribution | Proposer ID in attributes | Spot check 50 rounds | +| Timing Accuracy | Phase durations match PerfLog | <5% variance | +| No Consensus Impact | Round timing unchanged | Performance test passes | + +**Definition of Done**: Consensus rounds fully traceable, no impact on consensus timing. + +### 6.12.5 Phase 5: Production Deployment + +| Criterion | Measurement | Target | +| ------------ | ---------------------------- | -------------------------- | +| Collector HA | Multiple collectors deployed | No single point of failure | +| Sampling | Tail sampling configured | 10% base + errors + slow | +| Retention | Data retained per policy | 7 days hot, 30 days warm | +| Alerting | Alerts configured | Error spike, high latency | +| Runbook | Operator documentation | Approved by ops team | +| Training | Team trained | Session completed | + +**Definition of Done**: Telemetry running in production, operators trained, alerts active. + +### 6.12.6 Success Metrics Summary + +| Phase | Primary Metric | Secondary Metric | Deadline | Status | +| -------- | ------------------------------------------------------------------ | --------------------------- | -------------- | ------------------ | +| Phase 1 | SDK compiles and runs | Zero overhead when disabled | End of Week 2 | Active | +| Phase 2 | 100% RPC coverage | <1ms latency overhead | End of Week 4 | Active | +| Phase 3 | Cross-node traces work | <5% throughput impact | End of Week 6 | Active | +| Phase 4 | Consensus fully traced | No consensus timing impact | End of Week 8 | Active | +| Phase 5 | Production deployment | Operators trained | End of Week 9 | Active | +| Phase 6 | StatsD metrics in Prometheus | 3 dashboards operational | End of Week 10 | Active | +| Phase 7 | All metrics via OTLP | No StatsD dependency | End of Week 12 | Active | +| Phase 8 | trace_id in logs + Loki | Tempo↔Loki correlation | End of Week 13 | Active | +| Phase 9 | 68+ new internal metrics in Prom | 2 new dashboards | End of Week 15 | Future Enhancement | +| Phase 10 | Full telemetry stack validated; OTel-sourced regression gate in CI | < 3% CPU overhead proven | End of Week 17 | Future Enhancement | +| Phase 11 | Third-party metrics via receiver | 4 new dashboards + alerting | End of Week 20 | Future Enhancement | + +--- + +## 6.13 Recommended Implementation Order + +Based on ROI analysis, implement in this exact order: + +```mermaid +flowchart TB + subgraph week1["Week 1"] + t1[1. OpenTelemetry SDK
Conan/CMake integration] + t2[2. Telemetry interface
SpanGuard, config] + end + + subgraph week2["Week 2"] + t3[3. RPC ServerHandler
instrumentation] + t4[4. Basic Tempo setup
for testing] + end + + subgraph week3["Week 3"] + t5[5. Transaction submit
tracing] + t6[6. Grafana dashboard
v1] + end + + subgraph week4["Week 4"] + t7[7. Protobuf context
extension] + t8[8. PeerImp tx.relay
instrumentation] + end + + subgraph week5["Week 5"] + t9[9. Multi-node
integration tests] + t10[10. Performance
benchmarks] + end + + subgraph week6_8["Weeks 6-8"] + t11[11. Consensus
instrumentation] + t12[12. Full integration
testing] + end + + subgraph week9["Week 9"] + t13[13. Production
deployment] + t14[14. Documentation
& training] + end + + t1 --> t2 --> t3 --> t4 + t4 --> t5 --> t6 + t6 --> t7 --> t8 + t8 --> t9 --> t10 + t10 --> t11 --> t12 + t12 --> t13 --> t14 + + style week1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style week2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style week3 fill:#bf360c,stroke:#8c2809,color:#fff + style week4 fill:#bf360c,stroke:#8c2809,color:#fff + style week5 fill:#bf360c,stroke:#8c2809,color:#fff + style week6_8 fill:#0d47a1,stroke:#082f6a,color:#fff + style week9 fill:#4a148c,stroke:#2e0d57,color:#fff + style t1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t3 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t4 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t5 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t6 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t7 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t8 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t9 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t10 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t11 fill:#0d47a1,stroke:#082f6a,color:#fff + style t12 fill:#0d47a1,stroke:#082f6a,color:#fff + style t13 fill:#4a148c,stroke:#2e0d57,color:#fff + style t14 fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +**Reading the diagram:** + +- **Week 1 (tasks 1-2)**: Foundation work -- integrate the OpenTelemetry SDK via Conan/CMake and build the `Telemetry` interface with `SpanGuard` and config parsing. +- **Week 2 (tasks 3-4)**: First observable output -- instrument `ServerHandler` for RPC tracing and stand up Tempo so developers can see traces immediately. +- **Weeks 3-5 (tasks 5-10)**: Transaction lifecycle -- add submit tracing, build the first Grafana dashboard, extend protobuf for cross-node context, instrument `PeerImp` relay, then validate with multi-node integration tests and performance benchmarks. +- **Weeks 6-8 (tasks 11-12)**: Consensus deep-dive -- instrument consensus rounds and phases, then run full integration testing across all instrumented paths. +- **Week 9 (tasks 13-14)**: Go-live -- deploy to production with sampling/alerting configured, and deliver documentation and operator training. +- **Arrow chain (t1 → ... → t14)**: Strict sequential dependency; each task's output is a prerequisite for the next. + +--- + +--- + +## Appendix: External Dashboard Parity + +> Cross-phase plan for reaching parity with the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). Previously a standalone design spec; merged here so the phase plan is self-contained. + +> **Date**: 2026-03-30 +> **Status**: Draft +> **Source**: [realgrapedrop/xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard) +> **Jira Epic**: RIPD-5060 + +### Summary + +Integrate 29 missing metrics, 18 alert rules, and enriched span attributes from the community `xrpl-validator-dashboard` into xrpld's native OpenTelemetry instrumentation. Changes are distributed across phases 2, 3, 4, 6, 7, 9, 10, and 11 of the OTel PR chain. + +### Gap Analysis + +#### Coverage Breakdown (86 external metrics) + +| Status | Count | Notes | +| ----------------- | ----- | -------------------------------------------------------------- | +| Already covered | 30 | peer_count, load_factor, io_latency, uptime, overlay traffic | +| Partially covered | 3 | state_value encoding, NuDB granularity, validation_quorum | +| Missing | 29 | Validation agreement, ledger economy, peer quality, UNL health | +| N/A (external) | 24 | Monitor health, realtime duplicates, system metrics | + +#### Missing Metrics by Category + +| Category | Metrics | Count | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | +| Validation Agreement | `validations_sent_total`, `validations_checked_total`, `validation_agreements_total`, `validation_missed_total`, `validation_agreement_pct_1h/24h`, `validation_agreements_1h/24h`, `validation_missed_1h/24h`, `validation_event` | 11 | +| Ledger Economy | `ledgers_closed_total`, `ledger_age_seconds`, `base_fee_xrp`, `reserve_base_xrp`, `reserve_inc_xrp`, `transaction_rate` | 6 | +| State Tracking | `time_in_current_state_seconds`, `state_changes_total`, `validator_state_info` | 3 | +| Peer Quality | `peers_insane`, `peer_latency_p90_ms` | 2 | +| Validator Health | `amendment_blocked`, `unl_expiry_days` | 2 | +| Upgrade Awareness | `peers_higher_version_pct`, `upgrade_recommended` | 2 | +| Storage / Other | `ledger_nudb_bytes`, `jq_trans_overflow_total`, `initial_sync_duration_seconds` | 3 | + +#### Alert Rules (18 total, from external dashboard) + +| Group | Count | Rules | +| ----------- | ----- | ----------------------------------------------------------------------------------------------------------------------- | +| Critical | 8 | Agreement <90%, not proposing, unhealthy state, amendment blocked, UNL expiring, IO latency, load factor, peer count <5 | +| Network | 3 | Peer drop >10%/30%, P90 latency + disconnect correlation | +| Performance | 7 | CPU >80%, memory >90%, disk >85%, job queue overflow, upgrade recommended, tx rate drop, stale ledger | + +--- + +### Branch-to-Change Mapping + +#### Phase 2 — `pratik/otel-phase2-rpc-tracing` + +> **Ref**: Adds to existing Phase 2 task list. Consumed by Phase 7 (MetricsRegistry) and Phase 10 (validation checks). + +**Task 2.8: RPC Span Attribute Enrichment** + +Add node-level health context to every `rpc.command.*` span so operators can correlate RPC behavior with node state. + +New span attributes on `rpc.command.*`: + +| Attribute | Type | Source | Value Example | +| ----------------------------- | ------ | ------------------------------------ | --------------------- | +| `xrpl.node.amendment_blocked` | bool | `app_.getOPs().isAmendmentBlocked()` | `true` | +| `xrpl.node.server_state` | string | `app_.getOPs().strOperatingMode()` | `"full"`, `"syncing"` | + +**File**: `src/xrpld/rpc/detail/RPCHandler.cpp` (in the `rpc.command.*` span creation block, after existing setAttribute calls) + +**Rationale**: RPC is the operator's primary interaction point. When a node is amendment-blocked or degraded, every RPC response is suspect. Tagging spans with this state enables Jaeger queries like `{name=~"rpc.command.*"} | xrpl.node.amendment_blocked = true` to find all RPCs served during a blocked period. + +**Exit Criteria**: + +- [ ] `rpc.command.server_info` spans carry `xrpl.node.amendment_blocked` and `xrpl.node.server_state` attributes +- [ ] No measurable latency impact (attribute values are cached atomics, not computed per-call) + +--- + +#### Phase 3 — `pratik/otel-phase3-tx-tracing` + +> **Ref**: Adds to existing Phase 3 task list. Consumed by Phase 10 (validation checks). + +**Task 3.7: Transaction Span Peer Version Attribute** + +Add the relaying peer's xrpld version to transaction receive spans to enable version-mismatch correlation. + +New span attribute on `tx.receive`: + +| Attribute | Type | Source | Value Example | +| ------------------- | ------ | -------------------- | --------------- | +| `xrpl.peer.version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` | + +**File**: `src/xrpld/overlay/detail/PeerImp.cpp` (in the `tx.receive` span block, after existing `xrpl.peer.id` setAttribute) + +**Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues during network upgrades. + +**Exit Criteria**: + +- [ ] `tx.receive` spans carry `xrpl.peer.version` attribute with a non-empty version string +- [ ] Attribute is omitted (not empty-string) when `getVersion()` returns empty + +--- + +#### Phase 4 — `pratik/otel-phase4-consensus-tracing` + +> **Ref**: Adds to existing Phase 4 task list. Provides the span-level foundation that Phase 7 (ValidationTracker) builds upon. Consumed by Phase 10 (validation checks). + +**Task 4.8: Consensus Validation Span Enrichment** + +Add ledger hash and validation type to validation spans on both send and receive paths. This enables trace-level agreement analysis — filter by ledger hash to see which validators agreed. + +New span attributes on `consensus.validation.send`: + +| Attribute | Type | Source | Value Example | +| ----------------------------- | ------ | --------------------------------------- | --------------------------- | +| `xrpl.validation.ledger_hash` | string | Ledger hash from `validate()` call args | `"A1B2C3..."` (64-char hex) | +| `xrpl.validation.full` | bool | Whether this is a full validation | `true` | + +New span attributes on `peer.validation.receive`: + +| Attribute | Type | Source | Value Example | +| ---------------------------------- | ------ | ------------------------------------- | --------------------------- | +| `xrpl.peer.validation.ledger_hash` | string | From deserialized STValidation object | `"A1B2C3..."` (64-char hex) | +| `xrpl.peer.validation.full` | bool | From STValidation flags | `true` | + +New span attributes on `consensus.accept`: + +| Attribute | Type | Source | Value Example | +| ------------------------------------ | ----- | ---------------------------------------- | ------------- | +| `xrpl.consensus.validation_quorum` | int64 | `app_.validators().quorum()` | `28` | +| `xrpl.consensus.proposers_validated` | int64 | `result.proposers` from consensus result | `35` | + +**Files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` (validation.send and accept spans) +- `src/xrpld/overlay/detail/PeerImp.cpp` (peer.validation.receive span) + +**Rationale**: The external dashboard's most valuable feature is validation agreement tracking. By recording the ledger hash on both outgoing and incoming validation spans, we create the raw data for agreement analysis at the trace level. Phase 7's ValidationTracker builds the metric-level aggregation on top of this. + +**Exit Criteria**: + +- [ ] `consensus.validation.send` spans carry `xrpl.validation.ledger_hash` and `xrpl.validation.full` +- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full` +- [ ] `consensus.accept` spans carry `xrpl.consensus.validation_quorum` and `xrpl.consensus.proposers_validated` +- [ ] Ledger hash attributes match between send and receive for the same ledger + +--- + +#### Phase 6 — `pratik/otel-phase6-statsd` + +> **Ref**: Adds to existing Phase 6 scope. No separate task list file exists for Phase 6 per project convention. + +**Addition: Bridge `peerDisconnectsCharges_` metric** + +The overlay already tracks resource-limit disconnects via `OverlayImpl::Stats::peerDisconnectsCharges_` (a `beast::insight::Gauge`). This metric is registered but not included in the StatsD bridge mapping. + +**What to do**: + +- Ensure `overlay_peer_disconnects_charges` appears in the StatsD-to-Prometheus metric name mapping +- Verify the metric appears in Prometheus after StatsD bridge is active + +**File**: `src/xrpld/overlay/detail/OverlayImpl.cpp` + +**Prometheus name**: `overlay_peer_disconnects_charges` + +--- + +#### Phase 7 — `pratik/otel-phase7-native-metrics` + +> **Ref**: Adds to existing Phase 7 task list. This is the largest addition. Depends on Phase 4 span attributes for validation tracking context. Consumed by Phase 9 (dashboards), Phase 10 (validation), Phase 11 (alerts). + +**Task 7.8: ValidationTracker — Validation Agreement Computation** + +The most valuable missing component. A stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows. + +**Architecture**: + +``` + + consensus.validation.send ─────> ValidationTracker ──────> MetricsRegistry + (records our validation (reconciles after (exports agreement + for ledger X) 8s grace period) gauges every 10s) + + ledger.validate ───────────────> ValidationTracker + (records which ledger (marks ledger X as + network validated) agreed or missed) +``` + +**Design**: + +```cpp +/// Tracks validation agreement between this node and network consensus. +/// +/// ValidationTracker +/// ├── recordOurValidation(ledgerHash, ledgerSeq) // called when we send +/// ├── recordNetworkValidation(ledgerHash, seq) // called on ledger validate +/// ├── reconcile() // called periodically (timer) +/// ├── agreementPct1h() -> double // 0.0-100.0 +/// ├── agreementPct24h() -> double +/// ├── agreements1h() -> uint64_t +/// ├── missed1h() -> uint64_t +/// ├── agreements24h() -> uint64_t +/// ├── missed24h() -> uint64_t +/// ├── totalAgreements() -> uint64_t +/// ├── totalMissed() -> uint64_t +/// ├── totalValidationsSent() -> uint64_t +/// └── totalValidationsChecked() -> uint64_t // all network validations seen +class ValidationTracker +{ + // Ring buffer of pending ledger events (max 1000) + struct LedgerEvent { + uint256 ledgerHash; + LedgerIndex seq; + TimePoint closeTime; + bool weValidated = false; // did we send a validation for this ledger? + bool networkValidated = false; // did network validate this ledger? + bool reconciled = false; // has 8s grace period elapsed? + bool agreed = false; // after reconciliation: did we agree? + }; + + // Sliding window deques for pre-computed window stats + struct WindowEvent { + TimePoint time; + bool agreed; + }; + std::deque window1h_; // events in last 1 hour + std::deque window24h_; // events in last 24 hours + + // Reconciliation: 8s grace period after ledger close. + // If our validation hasn't arrived by then, mark as missed. + // 5-minute late repair: if a late validation arrives, correct the miss. + static constexpr auto kGracePeriod = std::chrono::seconds(8); + static constexpr auto kLateRepairWindow = std::chrono::minutes(5); +}; +``` + +**Recording sites** (modifications to consensus code from Phase 7 branch): + +| Hook Point | File | What to Record | +| ---------------------------- | ------------------- | ----------------------------------------------------------------------- | +| `validate()` in `doAccept()` | RCLConsensus.cpp | `tracker.recordOurValidation(ledgerHash, seq)` | +| `onValidation()` callback | RCLValidations path | `tracker.recordNetworkValidation(...)` — increment `validationsChecked` | +| LedgerMaster fully-validated | LedgerMaster.cpp | `tracker.recordNetworkValidation(validatedHash, seq)` | + +**Key new files**: + +- `src/xrpld/telemetry/ValidationTracker.h` +- `src/xrpld/telemetry/detail/ValidationTracker.cpp` + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker) +- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks) +- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook) + +**Exit Criteria**: + +- [ ] `ValidationTracker` correctly tracks agreement with 8s grace period +- [ ] 5-minute late repair corrects false-positive misses +- [ ] Thread-safe (atomics + mutex for window deques) +- [ ] Rolling windows correctly evict stale entries +- [ ] Unit tests for: normal agreement, missed validation, late repair, window eviction + +--- + +**Task 7.9: Validator Health Observable Gauges** + +New MetricsRegistry observable gauge for amendment, UNL, and quorum health. + +| Gauge Name | Label `metric=` | Type | Source | +| ------------------ | ------------------- | ------ | ------------------------------------------------- | +| `validator_health` | `amendment_blocked` | int64 | `app_.getOPs().isAmendmentBlocked()` → 0/1 | +| | `unl_blocked` | int64 | `app_.getOPs().isUNLBlocked()` → 0/1 | +| | `unl_expiry_days` | double | `app_.validators().expires()` → days until expiry | +| | `validation_quorum` | int64 | `app_.validators().quorum()` | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` (new gauge callback in `registerAsyncGauges()`) + +**Exit Criteria**: + +- [ ] All 4 label values emitted every 10s +- [ ] `unl_expiry_days` is negative when expired, positive when active +- [ ] Values visible in Prometheus + +--- + +**Task 7.10: Peer Quality Observable Gauges** + +New MetricsRegistry observable gauge for peer health aggregates. + +| Gauge Name | Label `metric=` | Type | Source | +| -------------- | -------------------------- | ------ | ------------------------------------------ | +| `peer_quality` | `peer_latency_p90_ms` | double | Iterate peers, compute P90 from `latency_` | +| | `peers_insane_count` | int64 | Count peers with `tracking_ == diverged` | +| | `peers_higher_version_pct` | double | Compare `getVersion()` to own version | +| | `upgrade_recommended` | int64 | 1 if `peers_higher_version_pct > 60%` | + +**Implementation note**: The callback iterates `app_.overlay().foreach(...)` to collect per-peer latency and version data. This runs every 10s on the metrics reader thread — acceptable overhead for ~50-200 peers. + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] P90 latency computed correctly (sort peer latencies, pick 90th percentile) +- [ ] Insane count matches `peers` RPC output +- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1") +- [ ] Values visible in Prometheus + +--- + +**Task 7.11: Ledger Economy Observable Gauges** + +New MetricsRegistry observable gauge for fee and ledger metrics. + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | -------------------- | ------ | ----------------------------------------- | +| `ledger_economy` | `base_fee_xrp` | double | `app_.getFeeTrack().getBaseFee()` → drops | +| | `reserve_base_xrp` | double | From validated ledger fee settings | +| | `reserve_inc_xrp` | double | From validated ledger fee settings | +| | `ledger_age_seconds` | double | `now - lastValidatedCloseTime` | +| | `transaction_rate` | double | Derived: tx count delta / time delta | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] Fee values match `server_info` RPC output +- [ ] `ledger_age_seconds` increases monotonically between ledger closes, resets on close +- [ ] `transaction_rate` is smoothed (rolling average, not instantaneous) + +--- + +**Task 7.12: State Tracking Observable Gauges** + +New MetricsRegistry observable gauge for node state duration. + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | ------------------------------- | ------ | ------------------------------------------------ | +| `state_tracking` | `state_value` | int64 | 0-7 numeric encoding matching external dashboard | +| | `time_in_current_state_seconds` | double | `now - lastModeChangeTime` | + +**State value encoding**: + +xrpld's `OperatingMode` enum maps 0-4 (DISCONNECTED through FULL). The external dashboard extends this to 0-6 by combining operating mode with consensus participation: + +| Value | State | Source | +| ----- | ------------ | ----------------------------------------------------------- | +| 0 | disconnected | `OperatingMode::DISCONNECTED` | +| 1 | connected | `OperatingMode::CONNECTED` | +| 2 | syncing | `OperatingMode::SYNCING` | +| 3 | tracking | `OperatingMode::TRACKING` | +| 4 | full | `OperatingMode::FULL` and not validating | +| 5 | validating | `OperatingMode::FULL` and `mConsensus.validating()` is true | +| 6 | proposing | `OperatingMode::FULL` and consensus mode is `proposing` | + +**Note**: Values 5-6 require checking both `OperatingMode` and `ConsensusMode`. The callback should derive these from `app_.getOPs().getOperatingMode()` combined with `mConsensus.mode()`. If operating mode is FULL and consensus is proposing → 6; if FULL and validating → 5; otherwise use the raw OperatingMode enum value. + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] `state_value` matches external dashboard encoding +- [ ] `time_in_current_state_seconds` resets on mode change + +--- + +**Task 7.13: Storage Detail Observable Gauge** + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | --------------- | ----- | ---------------------------------------- | +| `storage_detail` | `nudb_bytes` | int64 | NuDB backend file size (filesystem stat) | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] NuDB file size reported in bytes +- [ ] Gracefully returns 0 if NuDB not configured + +--- + +**Task 7.14: New Synchronous Counters** + +New counters incremented at event sites. Declared in MetricsRegistry, recording sites added in consensus/overlay/network code. + +| Counter Name | Increment Site | Source File | +| ----------------------------- | -------------------------------- | --------------------- | +| `ledgers_closed_total` | `onAccept()` in consensus | RCLConsensus.cpp | +| `validations_sent_total` | `validate()` in consensus | RCLConsensus.cpp | +| `validations_checked_total` | Network validation received | LedgerMaster.cpp | +| `validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `validation_missed_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `state_changes_total` | `setMode()` in NetworkOPs | NetworkOPs.cpp | +| `jq_trans_overflow_total` | Job queue overflow path | JobQueue.cpp | + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (counter declarations) +- `src/xrpld/app/consensus/RCLConsensus.cpp` (recording: ledgers_closed, validations_sent) +- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (recording: validations_checked) +- `src/xrpld/app/misc/NetworkOPs.cpp` (recording: state_changes) + +**Exit Criteria**: + +- [ ] All 7 counters monotonically increase during normal operation +- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s) +- [ ] Values visible in Prometheus + +--- + +**Task 7.15: Validation Agreement Observable Gauge** + +Reads from the `ValidationTracker` (Task 7.8) to export rolling window stats. + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | ------------------- | ------ | --------------------------- | +| `validation_agreement` | `agreement_pct_1h` | double | `tracker.agreementPct1h()` | +| | `agreements_1h` | int64 | `tracker.agreements1h()` | +| | `missed_1h` | int64 | `tracker.missed1h()` | +| | `agreement_pct_24h` | double | `tracker.agreementPct24h()` | +| | `agreements_24h` | int64 | `tracker.agreements24h()` | +| | `missed_24h` | int64 | `tracker.missed24h()` | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] Agreement percentages in range [0.0, 100.0] +- [ ] Window stats match manual count from validation counters +- [ ] Percentages stabilize after 1h/24h of operation + +--- + +#### Phase 9 — `pratik/otel-phase9-metric-gap-fill` + +> **Ref**: Adds to existing Phase 9 task list. Depends on Phase 7 gauges/counters. Consumed by Phase 10 (dashboard load checks). + +**Task 9.11: Validator Health Dashboard** + +New Grafana dashboard: `validator-health.json` + +| Panel | Type | PromQL | +| -------------------------- | ---------- | -------------------------------------------------------- | +| Agreement % (1h) | stat | `validation_agreement{metric="agreement_pct_1h"}` | +| Agreement % (24h) | stat | `validation_agreement{metric="agreement_pct_24h"}` | +| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | +| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | +| Validation Rate | stat | `rate(validations_sent_total[5m]) * 60` | +| Validations Checked Rate | stat | `rate(validations_checked_total[5m]) * 60` | +| Amendment Blocked | stat | `validator_health{metric="amendment_blocked"}` | +| UNL Expiry (days) | stat | `validator_health{metric="unl_expiry_days"}` | +| Validation Quorum | stat | `validator_health{metric="validation_quorum"}` | +| State Value Timeline | timeseries | `state_tracking{metric="state_value"}` | +| Time in Current State | stat | `state_tracking{metric="time_in_current_state_seconds"}` | +| State Changes Rate | stat | `rate(state_changes_total[1h])` | +| Ledgers Closed Rate | stat | `rate(ledgers_closed_total[5m]) * 60` | + +**Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes. + +--- + +**Task 9.12: Peer Quality Dashboard** + +New Grafana dashboard: `peer-quality.json` + +| Panel | Type | PromQL | +| ---------------------- | ---------- | -------------------------------------------------------- | +| P90 Peer Latency | timeseries | `peer_quality{metric="peer_latency_p90_ms"}` | +| Insane/Diverged Peers | stat | `peer_quality{metric="peers_insane_count"}` | +| Higher Version Peers % | stat | `peer_quality{metric="peers_higher_version_pct"}` | +| Upgrade Recommended | stat | `peer_quality{metric="upgrade_recommended"}` | +| Resource Disconnects | timeseries | `overlay_peer_disconnects_charges` | +| Inbound vs Outbound | bargauge | `peer_finder_active_inbound_peers`, `..._outbound_peers` | + +--- + +**Task 9.13: Ledger Economy Dashboard Panels** + +Add a "Ledger Economy" row to the existing `node-health.json` dashboard: + +| Panel | Type | PromQL | +| -------------------- | ---------- | --------------------------------------------- | +| Base Fee (drops) | stat | `ledger_economy{metric="base_fee_xrp"}` | +| Reserve Base (drops) | stat | `ledger_economy{metric="reserve_base_xrp"}` | +| Reserve Inc (drops) | stat | `ledger_economy{metric="reserve_inc_xrp"}` | +| Ledger Age | stat | `ledger_economy{metric="ledger_age_seconds"}` | +| Transaction Rate | timeseries | `ledger_economy{metric="transaction_rate"}` | + +--- + +#### Phase 10 — `pratik/otel-phase10-workload-validation` + +> **Ref**: Adds to existing Phase 10 task list. Validates all additions from Phases 2-9. + +**Task 10.6: External Dashboard Parity Validation Checks** + +Add checks to `validate_telemetry.py` for all new span attributes and metrics. + +**New span attribute checks (~8)**: + +| Span Name | New Attribute | +| --------------------------- | ------------------------------------ | +| `rpc.command.server_info` | `xrpl.node.amendment_blocked` | +| `rpc.command.server_info` | `xrpl.node.server_state` | +| `tx.receive` | `xrpl.peer.version` | +| `consensus.validation.send` | `xrpl.validation.ledger_hash` | +| `consensus.validation.send` | `xrpl.validation.full` | +| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | +| `consensus.accept` | `xrpl.consensus.validation_quorum` | +| `consensus.accept` | `xrpl.consensus.proposers_validated` | + +**New metric existence checks (~13)**: + +| Metric Name | +| -------------------------------------------------- | +| `validation_agreement{metric="agreement_pct_1h"}` | +| `validation_agreement{metric="agreement_pct_24h"}` | +| `validator_health{metric="amendment_blocked"}` | +| `validator_health{metric="unl_expiry_days"}` | +| `peer_quality{metric="peer_latency_p90_ms"}` | +| `peer_quality{metric="peers_insane_count"}` | +| `ledger_economy{metric="base_fee_xrp"}` | +| `ledger_economy{metric="transaction_rate"}` | +| `state_tracking{metric="state_value"}` | +| `ledgers_closed_total` | +| `validations_sent_total` | +| `state_changes_total` | +| `storage_detail{metric="nudb_bytes"}` | + +**New dashboard load checks (~3)**: + +| Dashboard | +| ----------------------- | +| `validator-health` | +| `peer-quality` | +| `node-health` (updated) | + +**New metric value sanity checks (~4)**: + +| Check | Condition | +| ----------------------------- | ----------------- | +| `validation_agreement_pct_1h` | in [0, 100] | +| `unl_expiry_days` | > 0 (not expired) | +| `peer_latency_p90_ms` | > 0 (peers exist) | +| `state_value` | in [0, 7] | + +**Total new checks: ~28** (bringing total from 73 to ~101) + +--- + +#### Phase 11 — (future branch) + +> **Ref**: Adds to existing Phase 11 task list. Depends on Phase 7 metrics and Phase 9 dashboards. + +**Task 11.9: Alert Rules from External Dashboard** + +Port 18 alert rules from the external `xrpl-validator-dashboard` to Grafana alerting provisioning. + +**Critical Group** (8 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------- | --- | +| Agreement Below 90% | `validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | +| Not Proposing | `state_tracking{metric="state_value"} < 6` | 10s | +| Unhealthy State | `state_tracking{metric="state_value"} < 4` | 10s | +| Amendment Blocked | `validator_health{metric="amendment_blocked"} == 1` | 1m | +| UNL Expiring | `validator_health{metric="unl_expiry_days"} < 14` | 1h | +| High IO Latency | `histogram_quantile(0.95, ios_latency_bucket) > 50` | 1m | +| High Load Factor | `load_factor_metrics{metric="load_factor"} > 1000` | 1m | +| Peer Count Critical | `server_info{metric="peers"} < 5` | 1m | + +**Network Group** (3 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------------- | ----------------------------------------------------------- | --- | +| Peer Drop >10% | `delta(server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | +| Peer Drop >30% | Same formula, threshold -30 | 30s | +| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | + +**Performance Group** (7 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------ | --- | +| CPU High | Per-core CPU > 80% | 2m | +| Memory Critical | Memory usage > 90% | 1m | +| Disk Warning | Disk usage > 85% | 2m | +| Job Queue Overflow | `rate(jq_trans_overflow_total[5m]) > 0` | 1m | +| Upgrade Recommended | `peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | +| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | +| Stale Ledger | `ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | + +**Notification channels**: Template configs for Email/SMTP, Discord, Slack, PagerDuty. + +**Files**: + +- `docker/telemetry/grafana/alerting/alert-rules.yaml` (new or extend existing) +- `docker/telemetry/grafana/alerting/contact-points.yaml` +- `docker/telemetry/grafana/alerting/notification-policies.yaml` + +--- + +**Task 11.10: Dual-Datasource Architecture Documentation** + +Document the external dashboard's "fast path" pattern as a future optimization for real-time panels: + +- **Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval. +- **Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow. +- **Decision**: Document as a future option, not implement now. Current 10s interval is acceptable for v1. + +**File**: `OpenTelemetryPlan/Phase11_taskList.md` (documentation task, no code) + +--- + +### Documentation Updates + +#### `docs/telemetry-runbook.md` (on Phase 9 branch) + +Add new sections after "Phase 9: OTel Metrics Alerting Rules": + +1. **Validator Health Monitoring** — explains agreement tracking, amendment blocked, UNL expiry, with example PromQL queries +2. **Peer Quality Monitoring** — explains P90 latency, insane peers, version awareness +3. **Ledger Economy Monitoring** — explains fee/reserve gauges, transaction rate, ledger age +4. **Validation Agreement Explained** — operator-facing explanation of the reconciliation algorithm (8s grace, 5m late repair), what "missed" means, and when to worry + +#### `OpenTelemetryPlan/09-data-collection-reference.md` (on Phase 9 branch) + +Add new metric tables in a "Phase 7+: External Dashboard Parity" section covering all 29 new metrics with their gauge names, label values, types, and sources. + +--- + +### Cross-Phase Dependency Chain + +``` +Phase 2 (span attrs: amendment_blocked, server_state) +Phase 3 (span attrs: peer.version) +Phase 4 (span attrs: validation.ledger_hash, validation.full, quorum) +Phase 6 (StatsD bridge: peerDisconnectsCharges) + │ + ├── all above rebase into ──> + │ +Phase 7 (ValidationTracker + 7 gauges + 7 counters + agreement gauge) + │ +Phase 9 (3 dashboards + ledger economy panels + runbook + data-collection-ref) + │ +Phase 10 (28 new validation checks in validate_telemetry.py) + │ +Phase 11 (18 alert rules + dual-datasource docs) +``` + +### Rebase Strategy + +After committing changes to each branch (starting from Phase 2): + +1. Commit on `pratik/otel-phase2-rpc-tracing` +2. Rebase `phase3` onto `phase2`, resolve conflicts (task list files only — low risk) +3. Commit on `phase3`, rebase `phase4` onto `phase3` +4. Continue through chain: 4 → 5 → 5b → 6 → 7 → 8 → 9 → 10 +5. Force-push-with-lease all affected branches + +Since these are documentation-only changes (task list .md files), merge conflicts should be minimal — each file is unique to its branch. + +_Previous: [Configuration Reference](./05-configuration-reference.md)_ | _Next: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md new file mode 100644 index 00000000000..4ebb6028fd2 --- /dev/null +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -0,0 +1,407 @@ +# Observability Backend Recommendations + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Implementation Phases](./06-implementation-phases.md) | [Appendix](./08-appendix.md) + +--- + +## 7.1 Development/Testing Backends + +> **OTLP** = OpenTelemetry Protocol + +| Backend | Pros | Cons | Use Case | +| ---------- | ----------------------------------- | ---------------------- | ------------------- | +| **Tempo** | Cost-effective, Grafana integration | Requires Grafana stack | Local dev, CI, Prod | +| **Zipkin** | Simple, lightweight | Basic features | Quick prototyping | + +### Quick Start with Tempo + +```bash +# Start Tempo with OTLP support +docker run -d --name tempo \ + -p 3200:3200 \ + -p 4317:4317 \ + -p 4318:4318 \ + grafana/tempo:2.6.1 +``` + +--- + +## 7.2 Production Backends + +> **APM** = Application Performance Monitoring + +| Backend | Pros | Cons | Use Case | +| ----------------- | ----------------------------------------- | ---------------------- | --------------------------- | +| **Grafana Tempo** | Cost-effective, Grafana integration | Requires Grafana stack | Most production deployments | +| **Elastic APM** | Full observability stack, log correlation | Resource intensive | Existing Elastic users | +| **Honeycomb** | Excellent query, high cardinality | SaaS cost | Deep debugging needs | +| **Datadog APM** | Full platform, easy setup | SaaS cost | Enterprise with budget | + +### Backend Selection Flowchart + +```mermaid +flowchart TD + start[Select Backend] --> budget{Budget
Constraints?} + + budget -->|Yes| oss[Open Source] + budget -->|No| saas{Prefer
SaaS?} + + oss --> existing{Existing
Stack?} + existing -->|Grafana| tempo[Grafana Tempo] + existing -->|Elastic| elastic[Elastic APM] + existing -->|None| tempo + + saas -->|Yes| enterprise{Enterprise
Support?} + saas -->|No| oss + + enterprise -->|Yes| datadog[Datadog APM] + enterprise -->|No| honeycomb[Honeycomb] + + tempo --> final[Configure Collector] + elastic --> final + honeycomb --> final + datadog --> final + + style start fill:#0f172a,stroke:#020617,color:#fff + style budget fill:#334155,stroke:#1e293b,color:#fff + style oss fill:#1e293b,stroke:#0f172a,color:#fff + style existing fill:#334155,stroke:#1e293b,color:#fff + style saas fill:#334155,stroke:#1e293b,color:#fff + style enterprise fill:#334155,stroke:#1e293b,color:#fff + style final fill:#0f172a,stroke:#020617,color:#fff + style tempo fill:#1b5e20,stroke:#0d3d14,color:#fff + style elastic fill:#bf360c,stroke:#8c2809,color:#fff + style honeycomb fill:#0d47a1,stroke:#082f6a,color:#fff + style datadog fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +**Reading the diagram:** + +- **Budget Constraints? (Yes)**: Leads to open-source options. If you already run Grafana or Elastic, pick the matching backend; otherwise default to Grafana Tempo. +- **Budget Constraints? (No) → Prefer SaaS?**: If you want a managed service, choose between Datadog (enterprise support) and Honeycomb (developer-focused). If not, fall back to open-source. +- **Terminal nodes (Tempo / Elastic / Honeycomb / Datadog)**: Each represents a concrete backend choice, all of which feed into the same final step. +- **Configure Collector**: Regardless of backend, you always finish by configuring the OTel Collector to export to your chosen destination. + +--- + +## 7.3 Recommended Production Architecture + +> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring | **HA** = High Availability + +```mermaid +flowchart TB + subgraph validators["Validator Nodes"] + v1[xrpld
Validator 1] + v2[xrpld
Validator 2] + end + + subgraph stock["Stock Nodes"] + s1[xrpld
Stock 1] + s2[xrpld
Stock 2] + end + + subgraph collector["OTel Collector Cluster"] + c1[Collector
DC1] + c2[Collector
DC2] + end + + subgraph backends["Storage Backends"] + tempo[(Grafana
Tempo)] + elastic[(Elastic
APM)] + archive[(S3/GCS
Archive)] + end + + subgraph ui["Visualization"] + grafana[Grafana
Dashboards] + end + + v1 -->|OTLP| c1 + v2 -->|OTLP| c1 + s1 -->|OTLP| c2 + s2 -->|OTLP| c2 + + c1 --> tempo + c1 --> elastic + c2 --> tempo + c2 --> archive + + tempo --> grafana + elastic --> grafana + + %% Note: simplified single-collector-per-DC topology shown for clarity + + style validators fill:#b71c1c,stroke:#7f1d1d,color:#ffffff + style stock fill:#0d47a1,stroke:#082f6a,color:#ffffff + style collector fill:#bf360c,stroke:#8c2809,color:#ffffff + style backends fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style ui fill:#4a148c,stroke:#2e0d57,color:#ffffff +``` + +**Reading the diagram:** + +- **Validator / Stock Nodes**: All xrpld nodes emit trace data via OTLP. Validators and stock nodes are grouped separately because they may reside in different network zones. +- **Collector Cluster (DC1, DC2)**: Regional collectors receive OTLP from nodes in their datacenter, apply processing (sampling, enrichment), and fan out to multiple backends. Enrichment includes deployment-tier tagging: each collector stamps `deployment.environment` and (as a fallback) `xrpl.network.type` so one Grafana stack can filter data from many collectors by tier. +- **Storage Backends**: Tempo and Elastic provide queryable trace storage; S3/GCS Archive provides long-term cold storage for compliance or post-incident analysis. +- **Grafana Dashboards**: The single visualization layer that queries both Tempo and Elastic, giving operators a unified view of all traces. +- **Data flow direction**: Nodes → Collectors → Storage → Grafana. Each arrow represents a network hop; minimizing collector-to-backend hops reduces latency. + +> **Note**: Production deployments should use multiple collector instances behind a load balancer for high availability. The diagram shows a simplified single-collector topology for clarity. + +--- + +## 7.4 Architecture Considerations + +### 7.4.1 Collector Placement + +| Strategy | Description | Pros | Cons | +| ------------- | -------------------- | ------------------------ | ----------------------- | +| **Sidecar** | Collector per node | Isolation, simple config | Resource overhead | +| **DaemonSet** | Collector per host | Shared resources | Complexity | +| **Gateway** | Central collector(s) | Centralized processing | Single point of failure | + +**Recommendation**: Use **Gateway** pattern with regional collectors for xrpld networks: + +- One collector cluster per datacenter/region +- Tail-based sampling at collector level +- Multiple export destinations for redundancy + +### 7.4.2 Sampling Strategy + +```mermaid +flowchart LR + subgraph head["Head Sampling (Node)"] + hs[Node-level head sampling
fixed at 100%
not configurable] + end + + subgraph tail["Tail Sampling (Collector)"] + ts1[Keep all errors] + ts2[Keep slow >5s] + ts3[Keep 10% rest] + end + + head --> tail + + ts1 --> final[Final Traces] + ts2 --> final + ts3 --> final + + style head fill:#0d47a1,stroke:#082f6a,color:#fff + style tail fill:#1b5e20,stroke:#0d3d14,color:#fff + style hs fill:#0d47a1,stroke:#082f6a,color:#fff + style ts1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style ts2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style ts3 fill:#1b5e20,stroke:#0d3d14,color:#fff + style final fill:#bf360c,stroke:#8c2809,color:#fff +``` + +**Reading the diagram:** + +- **Head Sampling (Node)**: xrpld pins head sampling at 100% (sample everything) and does not expose a configurable ratio. This is intentional: a per-node ratio would let different nodes make divergent keep/drop decisions for the same distributed trace, producing broken/partial traces. xrpld uses a `ParentBased` sampler so spans inheriting a remote parent honor the upstream decision. Volume reduction is delegated to the collector's tail sampling. +- **Tail Sampling (Collector)**: The second filter -- the collector inspects completed traces and applies rules: keep all errors, keep anything slower than 5 seconds, and keep 10% of the remainder. +- **Arrow head → tail**: All head-sampled traces flow to the collector, where tail sampling further reduces volume while preserving the most valuable data. +- **Final Traces**: The output after both sampling stages; this is what gets stored and queried. The two-stage approach balances cost with debuggability. + +### 7.4.3 Data Retention + +| Environment | Hot Storage | Warm Storage | Cold Archive | +| ----------- | ----------- | ------------ | ------------ | +| Development | 24 hours | N/A | N/A | +| Staging | 7 days | N/A | N/A | +| Production | 7 days | 30 days | many years | + +--- + +## 7.5 Integration Checklist + +- [ ] Choose primary backend (Tempo recommended for cost/features) +- [ ] Deploy collector cluster with high availability +- [ ] Configure tail-based sampling for error/latency traces +- [ ] Set up Grafana dashboards for trace visualization +- [ ] Configure alerts for trace anomalies +- [ ] Establish data retention policies +- [ ] Test trace correlation with logs and metrics + +--- + +## 7.6 Grafana Dashboard Examples + +Pre-built dashboards for xrpld observability. + +### 7.6.1 Consensus Health Dashboard + +A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all driven by TraceQL: + +- **Consensus Round Duration** (timeseries, ms): average `consensus.round` span duration per node instance, with yellow/red thresholds at 4s/5s. +- **Phase Duration Breakdown** (barchart): average duration of `consensus.phase.*` spans grouped by span name. +- **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. +- **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. + +Each panel's TraceQL query is described inline in its bullet above. + +### 7.6.2 Node Overview Dashboard + +A Tempo-backed dashboard (uid `xrpld-node-overview`) with four panels: + +- **Active Nodes** (stat): count of distinct `resource.service.instance.id` values seen for the `xrpld` service. +- **Total Transactions (1h)** (stat): count of `tx.receive` spans. +- **Error Rate** (gauge, percent): ratio of `status.code=error` spans to all spans, with yellow/red thresholds at 1%/5%. +- **Service Map** (nodeGraph): Tempo-generated service dependency graph. + +### 7.6.3 Alert Rules + +Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts`, evaluated every 1m) against the Tempo datasource: + +- **Consensus Round Slow** (warning, `for: 5m`): fires when average `consensus.round` duration exceeds 5s. + + ``` + {resource.service.name="xrpld" && name="consensus.round"} | avg(duration) > 5s + ``` + +- **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: + + ``` + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) + / + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) + > 0.05 + ``` + +- **Transaction Throughput Drop** (warning, `for: 10m`): fires when the `tx.receive` span rate falls below 10/s. + + ``` + {resource.service.name="xrpld" && name="tx.receive"} | rate() < 10 + ``` + +> **Note**: The Consensus Round Slow and Transaction Throughput Drop rules use TraceQL aggregates (`avg(duration)`, `rate()`), which require Tempo 2.3+ with TraceQL metrics enabled. Verify aggregate query support in your Tempo version before provisioning. The RPC Error Rate Spike rule instead queries Prometheus span metrics (collector `spanmetrics` connector), so it needs that connector enabled in the collector pipeline. + +--- + +## 7.7 PerfLog and Insight Correlation + +> **OTLP** = OpenTelemetry Protocol + +How to correlate OpenTelemetry traces with existing xrpld observability. + +### 7.7.1 Correlation Architecture + +```mermaid +flowchart TB + subgraph xrpld["xrpld Node"] + otel[OpenTelemetry
Spans] + perflog[PerfLog
JSON Logs] + insight[Beast Insight
StatsD Metrics] + end + + subgraph collectors["Data Collection"] + otelc[OTel Collector] + promtail[Promtail/Fluentd] + statsd[StatsD Exporter] + end + + subgraph storage["Storage"] + tempo[(Tempo)] + loki[(Loki)] + prom[(Prometheus)] + end + + subgraph grafana["Grafana"] + traces[Trace View] + logs[Log View] + metrics[Metrics View] + corr[Correlation
Panel] + end + + otel -->|OTLP| otelc --> tempo + perflog -->|JSON| promtail --> loki + insight -->|StatsD| statsd --> prom + + tempo --> traces + loki --> logs + prom --> metrics + + traces --> corr + logs --> corr + metrics --> corr + + style xrpld fill:#0d47a1,stroke:#082f6a,color:#fff + style collectors fill:#bf360c,stroke:#8c2809,color:#fff + style storage fill:#1b5e20,stroke:#0d3d14,color:#fff + style grafana fill:#4a148c,stroke:#2e0d57,color:#fff + style otel fill:#0d47a1,stroke:#082f6a,color:#fff + style perflog fill:#0d47a1,stroke:#082f6a,color:#fff + style insight fill:#0d47a1,stroke:#082f6a,color:#fff + style otelc fill:#bf360c,stroke:#8c2809,color:#fff + style promtail fill:#bf360c,stroke:#8c2809,color:#fff + style statsd fill:#bf360c,stroke:#8c2809,color:#fff + style tempo fill:#1b5e20,stroke:#0d3d14,color:#fff + style loki fill:#1b5e20,stroke:#0d3d14,color:#fff + style prom fill:#1b5e20,stroke:#0d3d14,color:#fff + style traces fill:#4a148c,stroke:#2e0d57,color:#fff + style logs fill:#4a148c,stroke:#2e0d57,color:#fff + style metrics fill:#4a148c,stroke:#2e0d57,color:#fff + style corr fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +**Reading the diagram:** + +- **xrpld Node (three sources)**: A single node emits three independent data streams -- OpenTelemetry spans, PerfLog JSON logs, and Beast Insight StatsD metrics. +- **Data Collection layer**: Each stream has its own collector -- OTel Collector for spans, Promtail/Fluentd for logs, and a StatsD exporter for metrics. They operate independently. +- **Storage layer (Tempo, Loki, Prometheus)**: Each data type lands in a purpose-built store optimized for its query patterns (trace search, log grep, metric aggregation). +- **Grafana Correlation Panel**: The key integration point -- Grafana queries all three stores and links them via shared fields (`trace_id`, `tx_hash`, `ledger_seq`), enabling a single-pane debugging experience. + +### 7.7.2 Correlation Fields + +| Source | Field | Link To | Purpose | +| ----------- | ------------------- | ------------- | -------------------------- | +| **Trace** | `trace_id` | Logs | Find log entries for trace | +| **Trace** | `tx_hash` | Logs, Metrics | Find TX-related data | +| **Trace** | `ledger_seq` | Logs | Find ledger-related logs | +| **PerfLog** | `trace_id` (new) | Traces | Jump to trace from log | +| **PerfLog** | `ledger_seq` | Traces | Find consensus trace | +| **Insight** | `exemplar.trace_id` | Traces | Jump from metric spike | + +### 7.7.3 Example: Debugging a Slow Transaction + +**Step 1: Find the trace** + +``` +# In Grafana Explore with Tempo +{resource.service.name="xrpld" && span.tx_hash="ABC123..."} +``` + +**Step 2: Get the trace_id from the trace view** + +``` +Trace ID: 4bf92f3577b34da6a3ce929d0e0e4736 +``` + +**Step 3: Find related PerfLog entries** + +``` +# In Grafana Explore with Loki +{job="xrpld"} |= "4bf92f3577b34da6a3ce929d0e0e4736" +``` + +**Step 4: Check Insight metrics for the time window** + +``` +# In Grafana with Prometheus +rate(xrpld_tx_applied_total[1m]) + @ timestamp_from_trace +``` + +### 7.7.4 Unified Dashboard Example + +A single dashboard (uid `xrpld-unified`) that ties traces, metrics, and logs together across the Tempo, Prometheus, and Loki datasources: + +- **Transaction Latency (Traces)** (timeseries, Tempo): `histogram_over_time(duration)` of `tx.receive` spans. +- **Transaction Rate (Metrics)** (timeseries, Prometheus): `rate(xrpld_tx_received_total[5m])` per instance, with a data link that opens the matching `tx.receive` traces in Tempo. +- **Recent Logs** (logs, Loki): `{job="xrpld"} | json`. +- **Trace Search** (table, Tempo): all `xrpld` traces, with per-row data links on `traceID` that jump to the trace in Tempo and to the correlated logs in Loki (`{job="xrpld"} |= ""`). + +The cross-datasource data links are what make this a single-pane debugging view; the correlation fields they rely on are listed in section 7.7.2. + +--- + +_Previous: [Implementation Phases](./06-implementation-phases.md)_ | _Next: [Appendix](./08-appendix.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md new file mode 100644 index 00000000000..53d7b01e42d --- /dev/null +++ b/OpenTelemetryPlan/08-appendix.md @@ -0,0 +1,262 @@ +# Appendix + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Observability Backends](./07-observability-backends.md) + +--- + +## 8.1 Glossary + +> **OTLP** = OpenTelemetry Protocol | **TxQ** = Transaction Queue + +| Term | Definition | +| --------------------- | ---------------------------------------------------------- | +| **Span** | A unit of work with start/end time, name, and attributes | +| **Trace** | A collection of spans representing a complete request flow | +| **Trace ID** | 128-bit unique identifier for a trace | +| **Span ID** | 64-bit unique identifier for a span within a trace | +| **Context** | Carrier for trace/span IDs across boundaries | +| **Propagator** | Component that injects/extracts context | +| **Sampler** | Decides which traces to record | +| **Exporter** | Sends spans to backend | +| **Collector** | Receives, processes, and forwards telemetry | +| **OTLP** | OpenTelemetry Protocol (wire format) | +| **W3C Trace Context** | Standard HTTP headers for trace propagation | +| **Baggage** | Key-value pairs propagated across service boundaries | +| **Resource** | Entity producing telemetry (service, host, etc.) | +| **Instrumentation** | Code that creates telemetry data | + +### xrpld-Specific Terms + +| Term | Definition | +| ----------------- | ------------------------------------------------------------- | +| **Overlay** | P2P network layer managing peer connections | +| **Consensus** | XRP Ledger consensus algorithm (RCL) | +| **Proposal** | Validator's suggested transaction set for a ledger | +| **Validation** | Validator's signature on a closed ledger | +| **HashRouter** | Component for transaction deduplication | +| **JobQueue** | Thread pool for asynchronous task execution | +| **PerfLog** | Existing performance logging system in xrpld | +| **Beast Insight** | Existing metrics framework in xrpld | +| **PathFinding** | Payment path computation engine for cross-currency payments | +| **TxQ** | Transaction queue managing fee-based prioritization | +| **LoadManager** | Dynamic fee escalation based on network load | +| **SHAMap** | SHA-256 hash-based map (Merkle trie variant) for ledger state | + +### Phase 9–11 Terms + +| Term | Definition | +| --------------------------- | ------------------------------------------------------------------------- | +| **MetricsRegistry** | Centralized class for OTel async gauge registrations (Phase 9) | +| **ObservableGauge** | OTel Metrics SDK async instrument polled via callback at fixed intervals | +| **PeriodicMetricReader** | OTel SDK component that invokes gauge callbacks at configurable intervals | +| **CountedObject** | xrpld template that tracks live instance counts via atomic counters | +| **TxQ** | Transaction queue managing fee escalation and ordering | +| **Load Factor** | Combined multiplier affecting transaction cost (local, cluster, network) | +| **OTel Collector Receiver** | Custom Go plugin that polls xrpld RPC and emits OTel metrics (Phase 11) | + +--- + +## 8.2 Span Hierarchy Visualization + +> **TxQ** = Transaction Queue + +```mermaid +flowchart TB + subgraph trace["Trace: Transaction Lifecycle"] + rpc["rpc.request
(entry point)"] + validate["tx.validate"] + relay["tx.relay
(parent span)"] + + subgraph peers["Peer Spans"] + p1["peer.send
Peer A"] + p2["peer.send
Peer B"] + p3["peer.send
Peer C"] + end + + subgraph pathfinding["PathFinding Spans"] + pathfind["pathfind.request"] + pathcomp["pathfind.compute"] + end + + consensus["consensus.round"] + apply["tx.apply"] + + subgraph txqueue["TxQ Spans"] + txq["txq.enqueue"] + txqApply["txq.apply"] + end + + feeCalc["fee.escalate"] + end + + subgraph validators["Validator Spans"] + valFetch["validator.list.fetch"] + valManifest["validator.manifest"] + end + + rpc --> validate + rpc --> pathfind + pathfind --> pathcomp + validate --> relay + relay --> p1 + relay --> p2 + relay --> p3 + p1 -.->|"context propagation"| consensus + consensus --> apply + apply --> txq + txq --> txqApply + txq --> feeCalc + + style trace fill:#0f172a,stroke:#020617,color:#fff + style peers fill:#1e3a8a,stroke:#172554,color:#fff + style pathfinding fill:#134e4a,stroke:#0f766e,color:#fff + style txqueue fill:#064e3b,stroke:#047857,color:#fff + style validators fill:#4c1d95,stroke:#6d28d9,color:#fff + style rpc fill:#1d4ed8,stroke:#1e40af,color:#fff + style validate fill:#047857,stroke:#064e3b,color:#fff + style relay fill:#047857,stroke:#064e3b,color:#fff + style p1 fill:#0e7490,stroke:#155e75,color:#fff + style p2 fill:#0e7490,stroke:#155e75,color:#fff + style p3 fill:#0e7490,stroke:#155e75,color:#fff + style consensus fill:#fef3c7,stroke:#fde68a,color:#1e293b + style apply fill:#047857,stroke:#064e3b,color:#fff + style pathfind fill:#0e7490,stroke:#155e75,color:#fff + style pathcomp fill:#0e7490,stroke:#155e75,color:#fff + style txq fill:#047857,stroke:#064e3b,color:#fff + style txqApply fill:#047857,stroke:#064e3b,color:#fff + style feeCalc fill:#047857,stroke:#064e3b,color:#fff + style valFetch fill:#6d28d9,stroke:#4c1d95,color:#fff + style valManifest fill:#6d28d9,stroke:#4c1d95,color:#fff +``` + +**Reading the diagram:** + +- **rpc.request (blue, top)**: The entry point — every traced transaction starts as an RPC call; this root span is the parent of all downstream work. +- **tx.validate and pathfind.request (green/teal, first fork)**: The RPC request fans out into transaction validation and, for cross-currency payments, a PathFinding branch (`pathfind.request` -> `pathfind.compute`). +- **tx.relay -> Peer Spans (teal, middle)**: After validation, the transaction is relayed to peers A, B, and C in parallel; each `peer.send` is a sibling child span showing fan-out across the network. +- **context propagation (dashed arrow)**: The dotted line from `peer.send Peer A` to `consensus.round` represents the trace context crossing a node boundary — the receiving validator picks up the same `trace_id` and continues the trace. +- **consensus.round -> tx.apply -> TxQ Spans (green, lower)**: Once consensus accepts the transaction, it is applied to the ledger; the TxQ spans (`txq.enqueue`, `txq.apply`, `fee.escalate`) capture queue depth and fee escalation behavior. +- **Validator Spans (purple, detached)**: `validator.list.fetch` and `validator.manifest` are independent workflows for UNL management — they run on their own traces and are linked to consensus via Span Links, not parent-child relationships. + +--- + +## 8.3 References + +> **OTLP** = OpenTelemetry Protocol + +### OpenTelemetry Resources + +1. [OpenTelemetry C++ SDK](https://github.com/open-telemetry/opentelemetry-cpp) +2. [OpenTelemetry Specification](https://opentelemetry.io/docs/specs/otel/) +3. [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) +4. [OTLP Protocol Specification](https://opentelemetry.io/docs/specs/otlp/) + +### Standards + +5. [W3C Trace Context](https://www.w3.org/TR/trace-context/) +6. [W3C Baggage](https://www.w3.org/TR/baggage/) +7. [Protocol Buffers](https://protobuf.dev/) + +### xrpld Resources + +8. [xrpld Source Code](https://github.com/XRPLF/rippled) +9. [XRP Ledger Documentation](https://xrpl.org/docs/) +10. [xrpld Overlay README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/README.md) +11. [xrpld RPC README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/README.md) +12. [xrpld Consensus README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/README.md) + +--- + +## 8.4 Version History + +| Version | Date | Author | Changes | +| ------- | ---------- | ------ | -------------------------------------------------------------- | +| 1.0 | 2026-02-12 | - | Initial implementation plan | +| 1.1 | 2026-02-13 | - | Refactored into modular documents | +| 1.2 | 2026-03-09 | - | Added Phases 9–11 (future enhancement plans) | +| 1.3 | 2026-03-24 | - | Review fixes: accuracy corrections, cross-document consistency | + +--- + +## 8.5 Document Index + +### Plan Documents + +| Document | Description | +| -------------------------------------------------------------------- | -------------------------------------------------- | +| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) | Master overview and executive summary | +| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md) | Distributed tracing concepts and OTel primer | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | xrpld architecture and trace points | +| [02-design-decisions.md](./02-design-decisions.md) | SDK selection, exporters, span conventions | +| [03-implementation-strategy.md](./03-implementation-strategy.md) | Directory structure, performance analysis | +| [05-configuration-reference.md](./05-configuration-reference.md) | xrpld config, CMake, Collector configs | +| [06-implementation-phases.md](./06-implementation-phases.md) | Timeline, tasks, risks, success metrics | +| [07-observability-backends.md](./07-observability-backends.md) | Backend selection and architecture | +| [08-appendix.md](./08-appendix.md) | Glossary, references, version history | +| [secure-OTel.md](./secure-OTel.md) | Threat model and hardening (mTLS, peer validation) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Span/metric/dashboard inventory | + +### Task Lists + +| Document | Description | +| -------------------------------------------------------------------------- | --------------------------------------------------- | +| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation | +| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing | +| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing | +| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing | +| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests | +| [Phase7_taskList.md](./Phase7_taskList.md) | Native OTel metrics migration | +| [Phase8_taskList.md](./Phase8_taskList.md) | Log-trace correlation | +| [Phase9_taskList.md](./Phase9_taskList.md) | Internal metric instrumentation gap fill (future) | +| [Phase10_taskList.md](./Phase10_taskList.md) | Synthetic workload generation & validation (future) | +| [Phase11_taskList.md](./Phase11_taskList.md) | Third-party data collection pipelines (future) | + +> **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md). + +--- + +## 8.6 Phase 9–11 Cross-Reference Guide + +This guide maps Phase 9–11 content to its location across the documentation. + +### Phase 9: Internal Metric Instrumentation Gap Fill + +| Content | Location | +| ------------------------------- | ------------------------------------------------------------------------ | +| Plan & architecture | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md) | +| Task list (10 tasks) | [Phase9_taskList.md](./Phase9_taskList.md) | +| Future metric definitions (~50) | [09-data-collection-reference.md §5b](./09-data-collection-reference.md) | +| New class: `MetricsRegistry` | `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (planned) | +| New dashboards | `fee-market`, `job-queue` (planned) | + +**Metric categories**: NodeStore I/O, Cache Hit Rates, TxQ, PerfLog Per-RPC, PerfLog Per-Job, Counted Objects, Fee Escalation & Load Factors. + +### Phase 10: Synthetic Workload Generation & Telemetry Validation + +| Content | Location | +| -------------------- | ------------------------------------------------------------------------ | +| Plan & architecture | [06-implementation-phases.md §6.8.3](./06-implementation-phases.md) | +| Task list (7 tasks) | [Phase10_taskList.md](./Phase10_taskList.md) | +| Validation inventory | [09-data-collection-reference.md §5c](./09-data-collection-reference.md) | +| Test harness | `docker/telemetry/docker-compose.workload.yaml` (planned) | +| CI workflow | `.github/workflows/telemetry-validation.yml` (planned) | + +**Validates**: 16 spans, 22 attributes, 300+ metrics, 10 dashboards, log-trace correlation. + +### Phase 11: Third-Party Data Collection Pipelines + +| Content | Location | +| --------------------------------- | ------------------------------------------------------------------------ | +| Plan & architecture | [06-implementation-phases.md §6.8.4](./06-implementation-phases.md) | +| Task list (11 tasks) | [Phase11_taskList.md](./Phase11_taskList.md) | +| External metric definitions (~30) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) | +| Custom OTel Collector receiver | `docker/telemetry/otel-rippled-receiver/` (planned) | +| Prometheus alerting rules (11) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) | +| New dashboards (4) | Validator Health, Network Topology, Fee Market (External), DEX & AMM | + +**Consumer categories**: Exchanges, Payment Processors, DeFi/AMM, NFT Marketplaces, Analytics Providers, Wallets, Compliance, Academic Researchers, Institutional Custody, CBDC Bridge Operators. + +--- + +_Previous: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md new file mode 100644 index 00000000000..99fd2fc2705 --- /dev/null +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -0,0 +1,1343 @@ +# Observability Data Collection Reference + +> **Audience**: Developers and operators. This is the single source of truth for all telemetry data collected by xrpld's observability stack. +> +> **Related docs**: [docs/telemetry-runbook.md](../docs/telemetry-runbook.md) (operator runbook with alerting and troubleshooting) | [03-implementation-strategy.md](./03-implementation-strategy.md) (code structure and performance optimization) | [04-code-samples.md](./04-code-samples.md) (C++ instrumentation examples) + +## Data Flow Overview + +```mermaid +graph LR + subgraph xrpldNode["xrpld Node"] + A["Trace Macros
XRPL_TRACE_SPAN
(OTLP/HTTP exporter)"] + B["beast::insight
OTel native metrics
(OTLP/HTTP exporter)"] + C["MetricsRegistry
OTel SDK metrics
(OTLP/HTTP exporter)"] + end + + subgraph collector["OTel Collector :4317 / :4318"] + direction TB + R1["OTLP Receiver
:4317 gRPC | :4318 HTTP
(traces + metrics)"] + BP["Batch Processor
timeout 1s, batch 100"] + SM["SpanMetrics Connector
derives RED metrics
from trace spans"] + + R1 --> BP + BP --> SM + end + + subgraph backends["Trace Backend"] + D["Grafana Tempo :3200
TraceQL search &
S3/GCS long-term storage"] + end + + subgraph metrics["Metrics Stack"] + E["Prometheus :9090
scrapes :8889
span-derived + system metrics"] + end + + subgraph viz["Visualization"] + F["Grafana :3000
13 dashboards"] + end + + A -->|"OTLP/HTTP :4318
(traces + attributes)"| R1 + B -->|"OTLP/HTTP :4318
(gauges, counters, histograms)"| R1 + C -->|"OTLP/HTTP :4318
(counters, histograms,
observable gauges)"| R1 + + BP -->|"OTLP/gRPC :4317"| D + + SM -->|"span_calls_total
span_duration_ms
(6 dimension labels)"| E + R1 -->|"gauges, counters,
histograms (OTLP)"| E + + E -->|"Prometheus
data source"| F + D -->|"Tempo
data source"| F + + style A fill:#4a90d9,color:#fff,stroke:#2a6db5 + style B fill:#4a90d9,color:#fff,stroke:#2a6db5 + style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BP fill:#449d44,color:#fff,stroke:#2d6e2d + style SM fill:#449d44,color:#fff,stroke:#2d6e2d + style D fill:#f0ad4e,color:#000,stroke:#c78c2e + style E fill:#f0ad4e,color:#000,stroke:#c78c2e + style F fill:#5bc0de,color:#000,stroke:#3aa8c1 + style xrpldNode fill:#1a2633,color:#ccc,stroke:#4a90d9 + style collector fill:#1a3320,color:#ccc,stroke:#5cb85c + style backends fill:#332a1a,color:#ccc,stroke:#f0ad4e + style metrics fill:#332a1a,color:#ccc,stroke:#f0ad4e + style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de +``` + +There are two independent telemetry pipelines entering a single **OTel Collector** via the same OTLP receiver: + +1. **OpenTelemetry Traces** — Distributed spans with attributes, exported via OTLP/HTTP (:4318) to the collector's **OTLP Receiver**. The **Batch Processor** groups spans (1s timeout, batch size 100) before forwarding to trace backends. The **SpanMetrics Connector** derives RED metrics (rate, errors, duration) from every span and feeds them into the metrics pipeline. +2. **beast::insight OTel Metrics** — System-level gauges, counters, and histograms exported natively via OTLP/HTTP (:4318) to the same **OTLP Receiver**. These are batched and exported to Prometheus alongside span-derived metrics. The StatsD UDP transport has been replaced by native OTLP; `server=statsd` remains available as a fallback. + +**Trace backend** — The collector exports traces via OTLP/gRPC to: + +- **Grafana Tempo** — Preferred trace backend. Supports TraceQL queries at `:3200`, S3/GCS object storage for cost-effective long-term trace retention, and integrates natively with Grafana. + +> **Further reading**: [00-tracing-fundamentals.md](./00-tracing-fundamentals.md) for core OpenTelemetry concepts (traces, spans, context propagation, sampling). [07-observability-backends.md](./07-observability-backends.md) for production backend selection, collector placement, and sampling strategies. + +--- + +## 1. OpenTelemetry Spans + +### 1.1 Complete Span Inventory (~37 spans) + +> **See also**: [02-design-decisions.md §2.3](./02-design-decisions.md#23-span-naming-conventions) for naming conventions and the full span catalog with rationale. [04-code-samples.md §4.6](./04-code-samples.md#46-span-flow-visualization) for span flow diagrams. + +> **Span names vs. attribute keys**: span names use dotted `subsystem.operation` +> form (e.g. `rpc.http_request`). Span _attribute_ keys use the bare/underscore +> form from the 2026-05-13 naming redesign (e.g. `tx_hash`, not `xrpl.tx.hash`). +> The dotted `xrpl.*` form is reserved for OTel **resource** attributes set once +> at startup. See §1.2 for the full attribute inventory. + +#### RPC Spans + +Controlled by `trace_rpc=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| -------------------- | ------------------ | ----------------- | ------------------------------------------------------------------------ | +| `rpc.http_request` | — | ServerHandler.cpp | Top-level HTTP JSON-RPC request entry point | +| `rpc.ws_message` | — | ServerHandler.cpp | WebSocket message handling (one per inbound frame) | +| `rpc.ws_upgrade` | — | ServerHandler.cpp | WebSocket upgrade handshake (records handshake failures) | +| `rpc.process` | `rpc.http_request` | ServerHandler.cpp | RPC processing pipeline (single or batch request) | +| `rpc.command.` | `rpc.process` | RPCHandler.cpp | Per-command span (e.g., `rpc.command.server_info`, `rpc.command.ledger`) | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"rpc.http_request|rpc.command.*"}` + +**Grafana dashboard**: _RPC Performance_ (`rpc-performance`) + +#### gRPC Spans + +Controlled by `trace_rpc=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ------------------- | ------ | -------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `grpc.` | — | GRPCServer.cpp | One flat span per gRPC method (e.g., `grpc.GetLedger`, `grpc.GetLedgerData`, `grpc.GetLedgerDiff`, `grpc.GetLedgerEntry`) | + +The method name is embedded in the span name (formed at the call site as +`grpc.`), so dashboards break out per-method latency and error +rates without TraceQL attribute filters. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"grpc.*"}` + +**Grafana dashboard**: _RPC Performance_ (`rpc-performance`) + +#### Transaction Spans + +Controlled by `trace_transactions=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| --------------- | -------------- | --------------- | ----------------------------------------------------------------- | +| `tx.process` | — | NetworkOPs.cpp | Transaction submission entry point (local or peer-relayed) | +| `tx.receive` | — | PeerImp.cpp | Raw transaction received from peer overlay (before deduplication) | +| `tx.apply` | `ledger.build` | BuildLedger.cpp | Transaction set applied to new ledger during consensus | +| `tx.preflight` | — | applySteps.cpp | Stateless checks stage (`stage=preflight`) | +| `tx.preclaim` | — | applySteps.cpp | Ledger-aware checks stage before fee claim (`stage=preclaim`) | +| `tx.transactor` | — | Transactor.cpp | Apply stage — the transactor runs (`stage=apply`) | + +The three apply-pipeline spans share a deterministic `trace_id` derived from +`txID[0:16]`, so preflight, preclaim, and transactor for one transaction group +under a single trace even though they run sequentially and often on different +threads. A transaction that hard-fails preflight or preclaim never reaches the +later spans — the `stage` attribute identifies where it stopped. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"tx.process|tx.receive"}` +or, for the apply pipeline: `{resource.service.name="xrpld" && name=~"tx.preflight|tx.preclaim|tx.transactor"}` + +**Grafana dashboard**: _Transaction Overview_ (`transaction-overview`) + +#### Transaction Queue (TxQ) Spans + +Controlled by `trace_transactions=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ------------------ | ------------- | ----------- | --------------------------------------------------- | +| `txq.enqueue` | `tx.process` | TxQ.cpp | Enqueue decision when a tx is submitted | +| `txq.apply_direct` | `txq.enqueue` | TxQ.cpp | Direct apply attempt that bypasses the queue | +| `txq.batch_clear` | `txq.enqueue` | TxQ.cpp | Batch clear of an account's queued txs | +| `txq.accept` | — | TxQ.cpp | Ledger-close accept loop (drains the queue) | +| `txq.accept_tx` | `txq.accept` | TxQ.cpp | Per-queued-transaction apply inside the accept loop | +| `txq.cleanup` | — | TxQ.cpp | Post-close cleanup of expired queue entries | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"txq.*"}` + +**Grafana dashboard**: _Transaction Overview_ (`transaction-overview`) + +#### Consensus Spans + +Controlled by `trace_consensus=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ------------------------------ | ------------------ | ---------------- | ------------------------------------------------------------------- | +| `consensus.round` | — (root) | RCLConsensus.cpp | Root span for one consensus round (deterministic trace per round) | +| `consensus.phase.open` | `consensus.round` | Consensus.h | Open phase — collecting transactions before close | +| `consensus.proposal.send` | `consensus.round` | RCLConsensus.cpp | Node broadcasts its transaction set proposal | +| `consensus.ledger_close` | `consensus.round` | RCLConsensus.cpp | Ledger close event triggered by consensus | +| `consensus.establish` | `consensus.round` | Consensus.h | Establish phase — converging on the transaction set | +| `consensus.update_positions` | `consensus.round` | Consensus.h | Position update with per-dispute vote details | +| `consensus.check` | `consensus.round` | Consensus.h | Consensus threshold check (agree/disagree tally) | +| `consensus.accept` | `consensus.round` | RCLConsensus.cpp | Consensus accepts a ledger (round complete) | +| `consensus.accept.apply` | `consensus.accept` | RCLConsensus.cpp | Ledger application with close-time details (jtACCEPT thread) | +| `consensus.validation.send` | `consensus.round` | RCLConsensus.cpp | Validation message sent after ledger accepted (follows-from link) | +| `consensus.mode_change` | `consensus.round` | RCLConsensus.cpp | Operating-mode transition during the round | +| `consensus.proposal.receive` | (context) | PeerImp.cpp | Proposal received from a peer (context-propagated into the round) | +| `consensus.validation.receive` | (context) | PeerImp.cpp | Validation received from a peer (context-propagated into the round) | + +The `.receive` spans are created per-message in the overlay and joined to the +round trace via context propagation rather than direct parenting. The +`consensus.validation.send` span uses a follows-from link off the round. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"consensus.*"}` + +**Grafana dashboard**: _Consensus Health_ (`consensus-health`) + +#### Ledger Spans + +Controlled by `trace_ledger=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ----------------- | ------ | ----------------- | ---------------------------------------------- | +| `ledger.build` | — | BuildLedger.cpp | Build new ledger from accepted transaction set | +| `ledger.validate` | — | LedgerMaster.cpp | Ledger promoted to validated status | +| `ledger.store` | — | LedgerMaster.cpp | Ledger stored to database/history | +| `ledger.acquire` | — | InboundLedger.cpp | Fetch a missing ledger from peers | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"ledger.*"}` + +**Grafana dashboard**: _Ledger Operations_ (`ledger-operations`) + +#### Peer Spans + +Controlled by `trace_peer` in `[telemetry]` config. **Enabled by default** (high volume). + +| Span Name | Parent | Source File | Description | +| ------------------------- | ------ | ----------- | ------------------------------------- | +| `peer.proposal.receive` | — | PeerImp.cpp | Consensus proposal received from peer | +| `peer.validation.receive` | — | PeerImp.cpp | Validation message received from peer | + +A `—` parent means the span is a fresh trace root (`kConsumer`): it is started +via `SpanGuard::rootSpan()` at the inbound-message entry point and never +inherits an ambient span left active on the peer thread, so it does not nest +under an unrelated transaction's trace. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"peer.*"}` + +**Grafana dashboard**: _Peer Network_ (`peer-network`) + +#### PathFind Spans + +Controlled by `trace_rpc=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| --------------------- | ------------------ | --------------- | ---------------------------------------------------------- | +| `pathfind.request` | `rpc.command.*` | PathRequest.cpp | `path_find` / `ripple_path_find` RPC entry | +| `pathfind.compute` | `pathfind.request` | PathRequest.cpp | Path computation for one request (`PathRequest::doUpdate`) | +| `pathfind.discover` | `pathfind.compute` | Pathfinder.cpp | Graph exploration (one per RPC call) | +| `pathfind.update_all` | — | PathRequest.cpp | Async recomputation of all active requests at ledger close | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"pathfind.*"}` + +--- + +### 1.2 Complete Attribute Inventory (bare/underscore keys) + +> **See also**: [02-design-decisions.md §2.4.2](./02-design-decisions.md#242-span-attributes-by-category) for attribute design rationale and privacy considerations. + +Every span can carry key-value attributes that provide context for filtering and +aggregation. Per the 2026-05-13 naming redesign, span-attribute keys use the +**bare** field name (the span name already carries the domain), or the +`_` underscore form where a bare name would collide (e.g. +`rpc_status`, `grpc_status`, `tx_status`, `txq_status`). + +> **Dotted keys are resource attributes, never span attributes:** +> +> - `xrpl.network.id` and `xrpl.network.type` are **resource** attributes set +> once at startup on the OTel resource — not span attributes. They appear on +> every span's resource scope, queried as `{resource.xrpl.network.id=...}`. +> - The ledger hash uses the bare `ledger_hash` key on every span that records +> it (both `consensus.validation.send` and `peer.validation.receive`) — there +> is no dotted span attribute. + +#### RPC Attributes + +| Attribute | Type | Set On | Description | +| ---------------------- | ------- | --------------------------------- | ------------------------------------------------ | +| `command` | string | `rpc.command.*`, `rpc.ws_message` | RPC command name (e.g., `server_info`, `ledger`) | +| `version` | int64 | `rpc.command.*` | API version number | +| `rpc_role` | string | `rpc.command.*` | Caller role: `"admin"` or `"user"` | +| `rpc_status` | string | `rpc.command.*` | Result: `"success"` or `"error"` | +| `request_payload_size` | int64 | `rpc.http_request` | Bytes of inbound request payload | +| `is_batch` | boolean | `rpc.process` | `true` if the request is a JSON-RPC batch | +| `batch_size` | int64 | `rpc.process` | Number of sub-requests in a batch | +| `load_type` | string | `rpc.command.*` | Resource cost category after execution | + +**Tempo query**: `{span.command="server_info"}` to find all `server_info` calls. + +**Prometheus label**: `command` (used as a SpanMetrics dimension). + +#### gRPC Attributes + +| Attribute | Type | Set On | Description | +| ------------- | ------ | ------------------- | ------------------------------------ | +| `method` | string | `grpc.` | gRPC method name (e.g., `GetLedger`) | +| `grpc_role` | string | `grpc.` | Caller role: `"admin"` or `"user"` | +| `grpc_status` | string | `grpc.` | Result: `"success"` or `"error"` | + +**Tempo query**: `{span.method="GetLedger"}` or `{name="grpc.GetLedger"}`. + +**Prometheus labels**: `method`, `grpc_role`, `grpc_status` (SpanMetrics dimensions). + +#### Transaction Attributes + +| Attribute | Type | Set On | Description | +| -------------- | ------- | ------------------------------------------------------------ | --------------------------------------------------------------------- | +| `tx_hash` | string | `tx.process`, `tx.receive` | Transaction hash (hex-encoded) | +| `local` | boolean | `tx.process` | `true` if locally submitted, `false` if peer-relayed | +| `path` | string | `tx.process` | Submission path: `"sync"` or `"async"` | +| `tx_type` | string | `tx.process`, `tx.preflight`, `tx.preclaim`, `tx.transactor` | Transaction type name (e.g., `Payment`) | +| `fee` | int64 | `tx.process` | Transaction fee in drops | +| `sequence` | int64 | `tx.process` | Transaction sequence number | +| `suppressed` | boolean | `tx.receive` | `true` if transaction was suppressed (duplicate) | +| `tx_status` | string | `tx.receive` | Transaction status (e.g., `"known_bad"`) | +| `peer_id` | int64 | `tx.receive` | Peer identifier (also set on peer spans) | +| `peer_version` | string | `tx.receive` | Peer protocol version string | +| `stage` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | Apply-pipeline stage: `preflight`, `preclaim`, or `apply` | +| `ter_result` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | Engine result token for that stage (e.g., `tesSUCCESS`, `terPRE_SEQ`) | +| `applied` | boolean | `tx.transactor` | `true` if the transaction was applied to the ledger | + +**Tempo query**: `{span.tx_hash=""}` to trace a specific transaction across nodes. + +**Prometheus labels**: `local`, `suppressed`, `tx_type`, `ter_result`, `stage` (SpanMetrics dimensions). + +#### Transaction Queue (TxQ) Attributes + +| Attribute | Type | Set On | Description | +| -------------------- | ------- | ------------------------------ | ----------------------------------------------------------- | +| `tx_hash` | string | `txq.enqueue`, `txq.accept_tx` | Transaction hash | +| `tx_type` | string | `txq.enqueue` | Transaction type name | +| `txq_status` | string | `txq.enqueue`, `txq.accept_tx` | Queue outcome (e.g. `queued`, `applied_direct`, `rejected`) | +| `fee_level_paid` | int64 | `txq.enqueue` | Fee level paid by the queued tx | +| `required_fee_level` | int64 | `txq.enqueue` | Minimum fee level for inclusion | +| `num_cleared` | int64 | `txq.batch_clear` | Entries cleared in a batch | +| `queue_size` | int64 | `txq.accept` | Current TxQ depth | +| `ledger_changed` | boolean | `txq.accept` | Whether the ledger changed since last attempt | +| `ter_code` | int64 | `txq.accept_tx` | Transaction engine result code | +| `retries_remaining` | int64 | `txq.accept_tx` | Retries left before discard | +| `ledger_seq` | int64 | `txq.cleanup` | Ledger sequence number | +| `expired_count` | int64 | `txq.cleanup` | Number of expired entries cleared | + +**Prometheus label**: `txq_status` (SpanMetrics dimension). + +#### Consensus Attributes + +| Attribute | Type | Set On | Description | +| -------------------------- | ------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `consensus_ledger_id` | string | `consensus.round` | Previous-ledger id anchoring the round | +| `ledger_seq` | int64 | `consensus.round`, `consensus.ledger_close`, `consensus.accept.apply`, `consensus.validation.send` | Ledger sequence number | +| `consensus_mode` | string | `consensus.round`, `consensus.ledger_close` | Node mode: `"Proposing"`, `"Observing"`, `"Wrong"`, etc. | +| `consensus_round_id` | int64 | `consensus.round` | Round identifier | +| `consensus_phase` | string | `consensus.round` | Current phase name (updated on each transition) | +| `trace_strategy` | string | `consensus.round` | Trace-id strategy (`deterministic` / `random`) | +| `previous_ledger_seq` | int64 | `consensus.round` | Sequence of the previous ledger | +| `previous_proposers` | int64 | `consensus.round` | Proposer count in the previous round | +| `previous_round_time_ms` | int64 | `consensus.round` | Duration of the previous round | +| `consensus_round` | int64 | `consensus.proposal.send` | Proposal sequence number for the broadcast proposal | +| `is_bow_out` | boolean | `consensus.proposal.send` | Whether the proposal is a bow-out (resigning the round) | +| `tx_count_open` | int64 | `consensus.ledger_close` | Transactions in the open ledger at close | +| `close_time_resolution_ms` | int64 | `consensus.ledger_close` | Close-time rounding granularity | +| `converge_percent` | int64 | `consensus.establish`, `consensus.update_positions` | Convergence percentage | +| `establish_count` | int64 | `consensus.establish` | Establish-phase iteration count | +| `proposers` | int64 | `consensus.establish`, `consensus.update_positions`, `consensus.accept` | Number of proposers | +| `disputes_count` | int64 | `consensus.establish`, `consensus.update_positions` | Number of disputed transactions | +| `tx_id` | string | `consensus.update_positions` | Disputed transaction id (per-dispute event) | +| `dispute_our_vote` | boolean | `consensus.update_positions` | Our vote on the disputed tx | +| `dispute_yays` | int64 | `consensus.update_positions` | Yes votes on the disputed tx | +| `dispute_nays` | int64 | `consensus.update_positions` | No votes on the disputed tx | +| `agree_count` | int64 | `consensus.check` | Agreeing proposer count | +| `disagree_count` | int64 | `consensus.check` | Disagreeing proposer count | +| `threshold_percent` | int64 | `consensus.check` | Agreement threshold percentage | +| `consensus_result` | string | `consensus.check` | Check outcome | +| `quorum` | int64 | `consensus.check`, `consensus.accept` | Quorum required | +| `round_time_ms` | int64 | `consensus.accept`, `consensus.accept.apply` | Total consensus round duration in milliseconds | +| `consensus_state` | string | `consensus.accept.apply` | Consensus outcome: `"finished"` or `"moved_on"` | +| `close_time` | int64 | `consensus.accept.apply` | Agreed-upon ledger close time (epoch seconds) | +| `close_time_correct` | boolean | `consensus.accept.apply` | Whether validators agreed on close time | +| `close_resolution_ms` | int64 | `consensus.accept.apply` | Close-time rounding granularity in milliseconds | +| `proposing` | boolean | `consensus.accept.apply`, `consensus.validation.send` | Whether this node was a proposer | +| `parent_close_time` | int64 | `consensus.accept.apply` | Parent ledger close time | +| `close_time_self` | int64 | `consensus.accept.apply` | This node's close-time vote | +| `close_time_vote_bins` | string | `consensus.accept.apply` | Distribution of close-time votes | +| `resolution_direction` | string | `consensus.accept.apply` | Whether close resolution increased/decreased/unchanged | +| `tx_count` | int64 | `consensus.accept.apply` | Transactions in the accepted set | +| `ledger_hash` | string | `consensus.validation.send` | Full hash of the validated ledger (shared with peer) | +| `full_validation` | boolean | `consensus.validation.send` | Whether this is a full validation | +| `validation_sign_time` | int64 | `consensus.validation.send` | Validation signing time | +| `mode_old` | string | `consensus.mode_change` | Operating mode before the transition | +| `mode_new` | string | `consensus.mode_change` | Operating mode after the transition | + +**Tempo query**: `{span.consensus_mode="Proposing"}` to find rounds where the node was proposing. + +**Prometheus labels**: `consensus_mode`, `consensus_state`, `consensus_phase`, `consensus_result`, `consensus_stalled`, `mode_new`, `close_time_correct` (SpanMetrics dimensions). + +#### Ledger Attributes + +| Attribute | Type | Set On | Description | +| --------------------- | ------- | ------------------------------------------------- | ------------------------------------------------ | +| `ledger_seq` | int64 | `ledger.build`, `ledger.validate`, `ledger.store` | Ledger sequence number | +| `close_time` | int64 | `ledger.build` | Ledger close time (epoch seconds) | +| `close_time_correct` | boolean | `ledger.build` | Whether close time was agreed upon by validators | +| `close_resolution_ms` | int64 | `ledger.build` | Close time rounding granularity in milliseconds | +| `tx_count` | int64 | `tx.apply` | Transactions applied to the ledger | +| `tx_failed` | int64 | `tx.apply` | Failed transactions in the apply set | +| `validations` | int64 | `ledger.validate` | Number of validations received for this ledger | +| `acquire_reason` | string | `ledger.acquire` | Why the ledger fetch was triggered | +| `timeouts` | int64 | `ledger.acquire` | Number of fetch timeouts | +| `peer_count` | int64 | `ledger.acquire` | Peers queried during the fetch | +| `outcome` | string | `ledger.acquire` | Fetch outcome | + +The apply-step span `tx.apply` (child of `ledger.build`) carries `tx_count`/`tx_failed`; +the parent `ledger.build` carries `ledger_seq` and the close-time attributes. +`ledger.acquire` (InboundLedger) also sets `ledger_seq`. + +**Tempo query**: `{span.ledger_seq=12345}` to find all spans for a specific ledger. + +#### Peer Attributes + +| Attribute | Type | Set On | Description | +| -------------------- | ------- | ---------------------------------------------------------------- | ---------------------------------------------------- | +| `peer_id` | int64 | `tx.receive`, `peer.proposal.receive`, `peer.validation.receive` | Peer identifier | +| `proposal_trusted` | boolean | `peer.proposal.receive` | Whether the proposal came from a trusted validator | +| `validation_trusted` | boolean | `peer.validation.receive` | Whether the validation came from a trusted validator | +| `full_validation` | boolean | `peer.validation.receive` | Whether the validation is a full validation | +| `ledger_hash` | string | `peer.validation.receive` | Validated ledger hash (shared with consensus spans) | + +**Prometheus labels**: `proposal_trusted`, `validation_trusted` (SpanMetrics dimensions). + +#### PathFind Attributes + +| Attribute | Type | Set On | Description | +| ------------------------- | ------- | --------------------- | ---------------------------------------- | +| `pathfind_source_account` | string | `pathfind.request` | Originating account for the path search | +| `pathfind_dest_account` | string | `pathfind.request` | Destination account | +| `pathfind_fast` | boolean | `pathfind.compute` | Whether fast pathfinding mode is enabled | +| `pathfind_search_level` | int64 | `pathfind.discover` | Depth of graph exploration | +| `pathfind_num_paths` | int64 | `pathfind.discover` | Total paths produced | +| `pathfind_ledger_index` | int64 | `pathfind.update_all` | Target ledger index | +| `pathfind_num_requests` | int64 | `pathfind.update_all` | Active requests recomputed | + +--- + +### 1.3 SpanMetrics — Derived Prometheus Metrics + +> **See also**: [01-architecture-analysis.md](./01-architecture-analysis.md) §1.8.2 for how span-derived metrics map to operational insights. + +The OTel Collector's SpanMetrics connector automatically generates RED (Rate, Errors, Duration) metrics from every span. No custom metrics code in xrpld is needed. + +| Prometheus Metric | Type | Description | +| ----------------------------------- | --------- | ------------------------------------------------------------------------------ | +| `span_calls_total` | Counter | Total span invocations | +| `span_duration_milliseconds_bucket` | Histogram | Latency distribution (buckets: 1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000 ms) | +| `span_duration_milliseconds_count` | Histogram | Observation count | +| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | + +**Standard labels on every metric**: `span_name`, `status_code`, `service_name`, `span_kind` + +**Additional dimension labels** (configured in `otel-collector-config.yaml`). +The Prometheus label is the **bare span-attribute key verbatim** — the +SpanMetrics connector does not rewrite or prefix it: + +| Prometheus Label / Span Attribute | Type | Applies To | +| --------------------------------- | ------- | ---------------------------------------------- | +| `command` | string | `rpc.command.*` | +| `rpc_status` | string | `rpc.command.*` | +| `consensus_mode` | string | `consensus.round`, `consensus.ledger_close` | +| `close_time_correct` | boolean | `consensus.accept.apply` | +| `local` | boolean | `tx.process` | +| `suppressed` | boolean | `tx.receive` | +| `proposal_trusted` | boolean | `peer.proposal.receive` | +| `validation_trusted` | boolean | `peer.validation.receive` | +| `tx_type` | string | `tx.*`, `txq.enqueue` | +| `ter_result` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | +| `stage` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | +| `txq_status` | string | `txq.enqueue`, `txq.accept_tx` | +| `consensus_state` | string | `consensus.accept.apply` | +| `load_type` | string | `rpc.command.*` | +| `is_batch` | boolean | `rpc.process` | +| `mode_new` | string | `consensus.mode_change` | +| `consensus_stalled` | boolean | `consensus.check` | +| `consensus_phase` | string | `consensus.round` | +| `consensus_result` | string | `consensus.check` | +| `method` | string | `grpc.` | +| `grpc_role` | string | `grpc.` | +| `grpc_status` | string | `grpc.` | + +The `stage` dimension (3 values: `preflight`, `preclaim`, `apply`) turns the +apply-pipeline spans into per-stage RED metrics with no native instruments — the +_Transaction Overview_ dashboard charts rate, p95 latency, and failure rate by stage. + +> **Sampling caveat**: xrpld head sampling is fixed at 1.0 (every trace is +> recorded), so span-derived metrics are not undercounted at the node. If the +> collector is configured with tail sampling, span-derived metrics reflect only +> the retained traces, whereas native StatsD/meter metrics do not sample. +> Account for any collector-side tail sampling when reading absolute stage rates. + +**Where to query**: Prometheus → `span_calls_total{span_name="rpc.command.server_info"}` + +--- + +## 2. System Metrics (beast::insight — OTel native) + +> **See also**: [02-design-decisions.md](./02-design-decisions.md) for the beast::insight coexistence design. [06-implementation-phases.md](./06-implementation-phases.md) for the Phase 6/7 metric inventory. +> +> **Migration complete**: Phase 7 replaced the StatsD UDP transport with native OTel Metrics SDK export via OTLP/HTTP. The `beast::insight::Collector` interface and all metric names are preserved — only the wire protocol changed. `[insight] server=statsd` remains as a fallback. + +These are system-level metrics emitted by xrpld's `beast::insight` framework via OTel OTLP/HTTP. They cover operational data that doesn't map to individual trace spans. + +### Configuration + +```ini +# Recommended: native OTel metrics via OTLP/HTTP +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +``` + +Fallback (StatsD): + +```ini +[insight] +server=statsd +address=127.0.0.1:8125 +prefix=xrpld +``` + +### 2.1 Gauges + +| Prometheus Metric | Source File | Description | Typical Range | +| ------------------------------------------- | --------------------- | ----------------------------------------- | ------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h | Seconds since last validated ledger | 0–10 (healthy), >30 (stale) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h | Seconds since last published ledger | 0–10 (healthy) | +| `state_accounting_disconnected_duration` | NetworkOPs.cpp | Cumulative seconds in Disconnected state | Monotonic | +| `state_accounting_connected_duration` | NetworkOPs.cpp | Cumulative seconds in Connected state | Monotonic | +| `state_accounting_syncing_duration` | NetworkOPs.cpp | Cumulative seconds in Syncing state | Monotonic | +| `state_accounting_tracking_duration` | NetworkOPs.cpp | Cumulative seconds in Tracking state | Monotonic | +| `state_accounting_full_duration` | NetworkOPs.cpp | Cumulative seconds in Full state | Monotonic (should dominate) | +| `state_accounting_disconnected_transitions` | NetworkOPs.cpp | Count of transitions to Disconnected | Low | +| `state_accounting_connected_transitions` | NetworkOPs.cpp | Count of transitions to Connected | Low | +| `state_accounting_syncing_transitions` | NetworkOPs.cpp | Count of transitions to Syncing | Low | +| `state_accounting_tracking_transitions` | NetworkOPs.cpp | Count of transitions to Tracking | Low | +| `state_accounting_full_transitions` | NetworkOPs.cpp | Count of transitions to Full | Low (should be 1 after startup) | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp | Active inbound peer connections | 0–85 | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp | Active outbound peer connections | 10–21 | +| `overlay_peer_disconnects` | OverlayImpl.cpp | Cumulative peer disconnection count | Low growth | +| `overlay_peer_disconnects_charges` | OverlayImpl.cpp | Disconnects due to resource limit charges | Low growth (subset of above) | +| `jobq_job_count` | JobQueue.cpp | Current job queue depth (group `jobq`) | 0–100 (healthy) | + +**Grafana dashboard**: _Node Health_ (`node-health`) + +### 2.2 Counters + +| Prometheus Metric | Source File | Description | +| ------------------------- | ------------------ | --------------------------------------------- | +| `rpc_requests` | ServerHandler.cpp | Total RPC requests received | +| `ledger_fetches` | InboundLedgers.cpp | Inbound ledger fetch attempts | +| `ledger_history_mismatch` | LedgerHistory.cpp | Ledger hash mismatches detected | +| `warn` | Logic.h | Resource manager warnings issued | +| `drop` | Logic.h | Resource manager drops (connections rejected) | + +**Note**: With `server=otel`, `warn` and `drop` are properly exported as OTel Counter instruments. The previous StatsD `|m` type limitation no longer applies. + +**Grafana dashboard**: _RPC & Pathfinding_ (`rpc-pathfinding`) + +### 2.3 Histograms (Event timers) + +| Prometheus Metric | Source File | Unit | Description | +| ----------------- | ----------------- | ---- | ------------------------------ | +| `rpc_time` | ServerHandler.cpp | ms | RPC response time distribution | +| `rpc_size` | ServerHandler.cpp | ms\* | RPC response size (see note) | +| `ios_latency` | Application.cpp | ms | I/O service loop latency | +| `pathfind_fast` | PathRequests.h | ms | Fast pathfinding duration | +| `pathfind_full` | PathRequests.h | ms | Full pathfinding duration | + +Quantiles collected: 0th, 50th, 90th, 95th, 99th, 100th percentile. + +\* **`rpc_size` instrument mismatch (known issue):** response size in bytes is +recorded through the millisecond-scaled event histogram (`makeEvent`), so it is +exported as `rpc_size_milliseconds_bucket` with time-scaled boundaries that top +out at 5000. Byte values above ~5 KB saturate in the last bucket, so the +percentiles are not true byte sizes. The _RPC & Pathfinding_ panel is flagged +accordingly. A dedicated byte-unit histogram is needed to fix this; tracked +separately. + +**Grafana dashboards**: _Node Health_ (`ios_latency`), _RPC & Pathfinding_ (`rpc_time`, `rpc_size`, `pathfind_*`) + +### 2.4 Overlay Traffic Metrics + +For each of the 45+ overlay traffic categories (defined in `TrafficCount.h`), four gauges are emitted: + +- `{category}_bytes_in` +- `{category}_bytes_out` +- `{category}_messages_in` +- `{category}_messages_out` + +**Key categories**: + +| Category | Description | +| ----------------------------------------------------------------- | -------------------------- | +| `total` | All traffic aggregated | +| `overhead` / `overhead_overlay` | Protocol overhead | +| `transactions` / `transactions_duplicate` | Transaction relay | +| `proposals` / `proposals_untrusted` / `proposals_duplicate` | Consensus proposals | +| `validations` / `validations_untrusted` / `validations_duplicate` | Consensus validations | +| `ledger_data_get` / `ledger_data_share` | Ledger data exchange | +| `ledger_data_Transaction_Node_get/share` | Transaction node data | +| `ledger_data_Account_State_Node_get/share` | Account state node data | +| `ledger_data_Transaction_Set_candidate_get/share` | Transaction set candidates | +| `getObject` / `haveTxSet` / `ledgerData` | Object requests | +| `ping` / `status` | Keepalive and status | +| `set_get` | Set requests | + +**Grafana dashboards**: _Network Traffic_ (`network-traffic`), _Overlay Traffic Detail_ (`overlay-traffic-detail`), _Ledger Data & Sync_ (`ledger-data-sync`) + +--- + +## 3. Grafana Dashboard Reference + +> **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8 for Grafana data source provisioning (Tempo, Prometheus) and TraceQL query examples. + +### 3.1 Span-Derived Dashboards (5) + +| Dashboard | UID | Data Source | Key Panels | +| -------------------- | ---------------------- | ------------------------ | ---------------------------------------------------------------------------------- | +| RPC Performance | `rpc-performance` | Prometheus (SpanMetrics) | Request rate by command, p95 latency by command, error rate, heatmap, top commands | +| Transaction Overview | `transaction-overview` | Prometheus (SpanMetrics) | Processing rate, latency p95/p50, local vs relay split, apply duration, heatmap | +| Consensus Health | `consensus-health` | Prometheus (SpanMetrics) | Round duration p95/p50, proposals rate, close duration, mode timeline, heatmap | +| Ledger Operations | `ledger-operations` | Prometheus (SpanMetrics) | Build rate, build duration, validation rate, store rate, build vs close comparison | +| Peer Network | `peer-network` | Prometheus (SpanMetrics) | Proposal receive rate, validation receive rate, trusted vs untrusted breakdown | + +### 3.2 System Metrics Dashboards (5) + +| Dashboard | UID | Data Source | Key Panels | +| ---------------------- | ------------------------ | ----------------- | --------------------------------------------------------------------------------- | +| Node Health | `node-health` | Prometheus (OTLP) | Ledger age, operating mode, I/O latency, job queue, fetch rate | +| Network Traffic | `network-traffic` | Prometheus (OTLP) | Active peers, disconnects, bytes in/out, messages in/out, traffic by category | +| RPC & Pathfinding | `rpc-pathfinding` | Prometheus (OTLP) | RPC rate, response time/size, pathfinding duration, resource warnings/drops | +| Overlay Traffic Detail | `overlay-traffic-detail` | Prometheus (OTLP) | Squelch, overhead, validator lists, set get/share, have/requested tx, proof paths | +| Ledger Data & Sync | `ledger-data-sync` | Prometheus (OTLP) | Ledger data exchange, legacy ledger share/get, getobject by type, traffic heatmap | + +### 3.3 Deployment-Tier Template Variables + +Every dashboard carries seven filtering template variables (each variable name +matches its Prometheus label), letting one Grafana stack be sliced by tier and +by perf-comparison run: + +| Variable | Source label | Description | +| ------------------------- | ------------------------ | ---------------------------------------------------------------- | +| `$node` | `service_instance_id` | Filter by xrpld node instance | +| `$service_name` | `service_name` | Filter by service (`service.name`, e.g. `xrpld`) | +| `$deployment_environment` | `deployment_environment` | Filter by deployment tier (`local` / `test` / `ci` / `prod`) | +| `$xrpl_network_type` | `xrpl_network_type` | Filter by network (`mainnet` / `testnet` / `devnet` / `perf`) | +| `$xrpl_work_item` | `xrpl_work_item` | Filter by perf-iac work item / ticket (e.g. `RIPD-7455`) | +| `$xrpl_branch` | `xrpl_branch` | Filter by comparison side (`baseline::` / `test:…`) | +| `$xrpl_node_role` | `xrpl_node_role` | Filter by node role (`validator` / `peer`) | + +The last three are populated only during perf-iac comparison runs (stamped as +resource attributes by perf-iac's own alloy pipeline, not the repo collector). +Outside those runs the labels are absent; the filters default to **All**, which +matches series lacking the label so every dashboard still renders. + +See [telemetry-runbook.md](../docs/telemetry-runbook.md) "Deployment Tiers" +for how the tier attributes are set and reach metrics. + +### 3.4 Accessing the Dashboards + +1. Open Grafana at **http://localhost:3000** +2. Navigate to **Dashboards → xrpld** folder +3. All 10 dashboards are auto-provisioned from `docker/telemetry/grafana/dashboards/` + +--- + +## 4. Tempo Trace Search Guide + +> **See also**: [08-appendix.md](./08-appendix.md) §8.2 for span hierarchy visualizations. [05-configuration-reference.md](./05-configuration-reference.md) §5.8.5 for TraceQL query examples. + +### Finding Traces by Type + +| What to Find | Tempo TraceQL Query | +| ------------------------ | ------------------------------------------------------------------------------ | +| All RPC calls | `{resource.service.name="xrpld" && name="rpc.http_request"}` | +| Specific RPC command | `{resource.service.name="xrpld" && name="rpc.command.server_info"}` | +| Slow RPC calls | `{resource.service.name="xrpld" && name=~"rpc.command.*"} \| duration > 100ms` | +| Failed RPC calls | `{span.rpc_status="error"}` | +| gRPC method calls | `{resource.service.name="xrpld" && name="grpc.GetLedger"}` | +| Specific transaction | `{span.tx_hash=""}` | +| Local transactions only | `{span.local=true}` | +| Consensus rounds | `{resource.service.name="xrpld" && name="consensus.round"}` | +| Rounds by mode | `{span.consensus_mode="Proposing"}` | +| Specific ledger | `{span.ledger_seq=12345}` | +| Peer proposals (trusted) | `{span.proposal_trusted=true}` | + +### Trace Structure + +A typical RPC trace shows the span hierarchy: + +``` +rpc.http_request (ServerHandler) + └── rpc.process (ServerHandler) + └── rpc.command.server_info (RPCHandler) +``` + +A consensus round groups its lifecycle spans under a single root +(`consensus.round`); the build/ledger spans run as their own trees: + +``` +consensus.round (root — one per round) + ├── consensus.phase.open (open phase) + ├── consensus.proposal.send (broadcast proposal) + ├── consensus.ledger_close (close event) + ├── consensus.establish (establish phase) + ├── consensus.update_positions (position updates) + ├── consensus.check (threshold check) + ├── consensus.accept (accept result) + │ └── consensus.accept.apply (apply, jtACCEPT thread) + └── consensus.validation.send (send validation, follows-from link) + +ledger.build (build new ledger) + └── tx.apply (apply transaction set) +ledger.validate (promote to validated) +ledger.store (persist to DB) +``` + +--- + +## 5. Prometheus Query Examples + +> **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8.7 for correlating Prometheus system metrics with trace-derived metrics. + +### Span-Derived Metrics + +```promql +# RPC request rate by command (last 5 minutes) +sum by (command) (rate(span_calls_total{span_name=~"rpc.command.*"}[5m])) + +# RPC p95 latency by command +histogram_quantile(0.95, sum by (le, command) (rate(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m]))) + +# Consensus round duration p95 +histogram_quantile(0.95, sum by (le) (rate(span_duration_milliseconds_bucket{span_name="consensus.round"}[5m]))) + +# Transaction processing rate (local vs relay) +sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m])) + +# Trusted vs untrusted proposal rate +sum by (proposal_trusted) (rate(span_calls_total{span_name="peer.proposal.receive"}[5m])) +``` + +### StatsD Metrics + +```promql +# Validated ledger age (should be < 10s) +ledgermaster_validated_ledger_age + +# Active peer count +peer_finder_active_inbound_peers + peer_finder_active_outbound_peers + +# RPC response time p95 +histogram_quantile(0.95, rpc_time_bucket) + +# Total network bytes in (rate) +rate(total_bytes_in[5m]) + +# Operating mode (should be "Full" after startup) +state_accounting_full_duration +``` + +--- + +## 5a. Log-Trace Correlation (Phase 8) + +> **Plan details**: [06-implementation-phases.md §6.8.1](./06-implementation-phases.md) — motivation, architecture, Mermaid diagrams +> **Task breakdown**: [Phase8_taskList.md](./Phase8_taskList.md) — per-task implementation details + +Phase 8 injects OTel trace context into xrpld's `Logs::format()` output, enabling log-trace correlation. When a log line is emitted within an active OTel span, the trace and span identifiers are automatically appended after the severity field: + +### Log Format + +``` + : trace_id=<32hex> span_id=<16hex> +``` + +Example: + +``` +2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +``` + +- **`trace_id=`** — 32-character lowercase hex trace identifier. Links to the distributed trace in Tempo. +- **`span_id=`** — 16-character lowercase hex span identifier. Identifies the specific span within the trace. +- **Only present** when the log is emitted within an active OTel span. Log lines outside of traced code paths have no trace context fields. + +### Implementation + +The trace context injection is implemented in `Logs::format()` (`src/libxrpl/basics/Log.cpp`), guarded by `#ifdef XRPL_ENABLE_TELEMETRY`. It checks the thread-local runtime context value directly (via `RuntimeContext::GetCurrent().GetValue(kSpanKey)`) to avoid the heap allocation that `GetSpan()` performs on the no-span path. On threads without an active span, the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, total cost is ~50ns per log call. + +### Log Ingestion Pipeline + +``` +xrpld debug.log -> OTel Collector filelog receiver -> regex_parser -> Loki exporter -> Grafana Loki +``` + +The OTel Collector's `filelog` receiver tails `debug.log` files and uses a `regex_parser` operator to extract structured fields: + +| Field | Type | Description | +| ----------- | -------- | -------------------------------------------------------- | +| `timestamp` | datetime | Log timestamp | +| `partition` | string | Log partition (e.g., `LedgerMaster`, `PeerImp`) | +| `severity` | string | Severity code (`TRC`, `DBG`, `NFO`, `WRN`, `ERR`, `FTL`) | +| `trace_id` | string | 32-hex trace identifier (optional) | +| `span_id` | string | 16-hex span identifier (optional) | +| `message` | string | Log message body | + +### Grafana Correlation + +Bidirectional linking between logs and traces is configured via Grafana datasource provisioning: + +- **Tempo -> Loki** (`tracesToLogs`): Clicking "Logs for this trace" on a Tempo trace view filters Loki logs by `trace_id`, showing all log lines from that trace. +- **Loki -> Tempo** (`derivedFields`): A regex-based derived field on the Loki datasource extracts `trace_id` from log lines and renders it as a clickable link to the corresponding trace in Tempo. + +### Loki Backend + +Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. + +### LogQL Query Examples + +```logql +# Find all logs for a specific trace +{job="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" + +# Error logs with trace context +{job="xrpld"} |= "ERR" |= "trace_id=" + +# Logs from a specific partition with trace context +{job="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P[a-f0-9]+)` | trace_id != "" + +# Count traced log lines over time +count_over_time({job="xrpld"} |= "trace_id=" [5m]) +``` + +--- + +## 5b. Internal Metric Gap Fill (Phase 9) + +> **Status**: Implemented. +> **Plan details**: [06-implementation-phases.md §6.8.2](./06-implementation-phases.md) — motivation, architecture, third-party context +> **Task breakdown**: [Phase9_taskList.md](./Phase9_taskList.md) — per-task implementation details + +Phase 9 fills the metrics that exist inside xrpld but previously lacked time-series export. It +uses a hybrid approach: `beast::insight` extensions for NodeStore I/O plus OTel `ObservableGauge` +async callbacks for new categories. + +> **Authoritative metric names live in [§ Phase 9: OTel SDK-Exported Metrics](#phase-9-otel-sdk-exported-metrics-metricsregistry) below.** +> Most internal metrics are emitted as **labeled** gauges — one instrument carrying many logical +> values via a `metric` label (e.g. `cache_metrics{metric="sle_hit_rate"}`, +> `txq_metrics{metric="txq_count"}`, `load_factor_metrics{metric="load_factor"}`, +> `nodestore_state{metric="node_reads_total"}`) — not the flat per-name form. Query the +> labeled names; the flat names (`cache_sle_hit_rate`, `txq_count`, …) are **not** emitted. + +#### Server Info (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------------- | ----- | -------- | -------------------------------------------- | +| `server_info{metric="server_state"}` | Gauge | `metric` | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | Gauge | `metric` | Seconds since server start | +| `server_info{metric="peers"}` | Gauge | `metric` | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | Gauge | `metric` | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | Gauge | `metric` | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | Gauge | `metric` | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | Gauge | `metric` | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | Gauge | `metric` | Last close convergence time (milliseconds) | + +#### Build Info (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| ----------------------------- | ----- | --------- | --------------------------------- | +| `build_info{version=""}` | Gauge | `version` | Info-style metric, always value 1 | + +#### Complete Ledger Ranges (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ----- | --------------- | --------------------------- | +| `complete_ledgers{bound="start",index=""}` | Gauge | `bound`,`index` | Start of contiguous range N | +| `complete_ledgers{bound="end",index=""}` | Gauge | `bound`,`index` | End of contiguous range N | + +#### Database Metrics (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------------- | ----- | -------- | --------------------------------- | +| `db_metrics{metric="db_kb_total"}` | Gauge | `metric` | Total database size (KB) | +| `db_metrics{metric="db_kb_ledger"}` | Gauge | `metric` | Ledger database size (KB) | +| `db_metrics{metric="db_kb_transaction"}` | Gauge | `metric` | Transaction database size (KB) | +| `db_metrics{metric="historical_perminute"}` | Gauge | `metric` | Historical ledger fetches per min | + +#### Extended Cache Metrics (additions to existing cache_metrics) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------- | ----- | -------- | ------------------------- | +| `cache_metrics{metric="al_size"}` | Gauge | `metric` | AcceptedLedger cache size | + +#### Extended NodeStore Metrics (additions to existing nodestore_state) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ----- | -------- | ----------------------------------- | +| `nodestore_state{metric="node_reads_duration_us"}` | Gauge | `metric` | Cumulative read time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | Gauge | `metric` | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | + +### New Grafana Dashboards (Phase 9) + +| Dashboard | UID | Data Source | Key Panels | +| ------------------ | ------------ | ----------- | ----------------------------------------------------------------- | +| Fee Market & TxQ | `fee-market` | Prometheus | TxQ depth/capacity, fee levels, load factor breakdown, escalation | +| Job Queue Analysis | `job-queue` | Prometheus | Per-job rates, queue wait times, execution times, queue depth | + +--- + +## 5c. Future: Synthetic Workload Generation & Telemetry Validation (Phase 10) + +> **Plan details**: [06-implementation-phases.md §6.8.3](./06-implementation-phases.md) — motivation, architecture +> **Task breakdown**: [Phase10_taskList.md](./Phase10_taskList.md) — per-task implementation details +> **Tools**: [docker/telemetry/workload/](../docker/telemetry/workload/) — RPC load generator, transaction submitter, validation suite, benchmarks + +Phase 10 builds a 5-node validator docker-compose harness with RPC load generators, transaction submitters, and automated validation scripts that verify all spans, metrics, dashboards, and log-trace correlation work end-to-end. Includes a benchmark suite comparing telemetry-ON vs telemetry-OFF overhead. + +### Running the Validation Suite + +```bash +# Full end-to-end validation (start cluster, generate load, validate): +docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld + +# Validation only (assumes stack and cluster are already running): +python3 docker/telemetry/workload/validate_telemetry.py --report /tmp/report.json + +# Performance benchmark (baseline vs telemetry): +docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300 +``` + +### Validated Telemetry Inventory + +> **Counting note — families vs series.** A _metric family_ is one distinct Prometheus `__name__` +> (histogram `_bucket`/`_count`/`_sum` collapsed to one). A _series_ is a family × its label +> combinations. The legacy overlay-traffic block is the bulk of the count: ~56 message categories × +> 4 (`_bytes_in/_out`, `_messages_in/_out`) ≈ 224 families on its own. The labeled gauges +> (`cache_metrics{metric}`, …) are few families but many series. Validate against the figures +> below as **families currently emitting** (idle nodes under-report — workload-gated metrics such as +> per-RPC/error counters appear only once exercised, which is Phase 10's purpose). + +| Category | Expected Count | Validation Method | Config File | +| ------------------------------ | ------------------------- | -------------------------------- | ----------------------- | +| Trace spans | ~37 (required + optional) | Tempo API query | `expected_spans.json` | +| Span attributes | per-span assertion | Per-span attribute assertion | `expected_spans.json` | +| Legacy beast::insight families | ~270 (≈224 traffic) | Prometheus `__name__` query | `expected_metrics.json` | +| Native MetricsRegistry | 35 instruments | Prometheus query | `expected_metrics.json` | +| SpanMetrics RED | 4 per span | Prometheus query | `expected_metrics.json` | +| Grafana dashboards | 15 | Dashboard API "no data" check | `expected_metrics.json` | +| Log-trace links | Present | Loki query + Tempo reverse check | — | + +### Performance Overhead Targets + +| Metric | Target | Measurement Method | +| ----------------- | ------------ | ----------------------------------- | +| CPU overhead | < 3% | ps avg CPU% baseline vs telemetry | +| Memory overhead | < 5MB | ps peak RSS baseline vs telemetry | +| RPC p99 latency | < 2ms impact | server_info round-trip timing | +| Throughput impact | < 5% | Ledger close rate comparison | +| Consensus impact | < 1% | Consensus round time p95 comparison | + +--- + +## 5d. Future: Third-Party Data Collection Pipelines (Phase 11) + +> **Status**: Planned, not yet implemented. +> **Plan details**: [06-implementation-phases.md §6.8.4](./06-implementation-phases.md) — motivation, architecture, consumer gap analysis +> **Task breakdown**: [Phase11_taskList.md](./Phase11_taskList.md) — per-task implementation details + +Phase 11 builds a custom OTel Collector receiver (Go) that polls xrpld's admin RPCs and exports `xrpl_*` metrics for external consumers. No xrpld code changes. + +### Exported Metrics (via Custom OTel Collector Receiver) + +#### Node Health (from server_info) + +| Prometheus Metric | Type | Description | +| --------------------------------------- | ----- | ----------------------------------------------- | +| `xrpl_server_state` | Gauge | Operating mode (0=disconnected ... 5=proposing) | +| `xrpl_server_state_duration_seconds` | Gauge | Seconds in current state | +| `xrpl_uptime_seconds` | Gauge | Consecutive seconds running | +| `xrpl_io_latency_ms` | Gauge | I/O subsystem latency | +| `xrpl_amendment_blocked` | Gauge | 1 if amendment-blocked, 0 otherwise | +| `xrpl_peers_count` | Gauge | Connected peers | +| `xrpl_validated_ledger_seq` | Gauge | Latest validated ledger sequence | +| `xrpl_validated_ledger_age_seconds` | Gauge | Seconds since last validated close | +| `xrpl_last_close_proposers` | Gauge | Proposers in last consensus round | +| `xrpl_last_close_converge_time_seconds` | Gauge | Last consensus round duration | +| `xrpl_load_factor` | Gauge | Transaction cost multiplier | +| `xrpl_state_duration_seconds` | Gauge | Per-state duration (`state` label) | +| `xrpl_state_transitions_total` | Gauge | Per-state transition count (`state` label) | + +#### Peer Topology (from peers) + +| Prometheus Metric | Type | Description | +| --------------------------- | ----- | ----------------------------------- | +| `xrpl_peers_inbound_count` | Gauge | Inbound peer connections | +| `xrpl_peers_outbound_count` | Gauge | Outbound peer connections | +| `xrpl_peer_latency_p50_ms` | Gauge | Median peer latency | +| `xrpl_peer_latency_p95_ms` | Gauge | p95 peer latency | +| `xrpl_peer_version_count` | Gauge | Peers per version (`version` label) | +| `xrpl_peer_diverged_count` | Gauge | Peers with diverged tracking status | + +#### Validator & Amendment (from validators, feature) + +| Prometheus Metric | Type | Description | +| ------------------------------------- | ----- | --------------------------------------- | +| `xrpl_trusted_validators_count` | Gauge | UNL validator count | +| `xrpl_amendment_enabled_count` | Gauge | Enabled amendments | +| `xrpl_amendment_majority_count` | Gauge | Amendments with majority | +| `xrpl_amendment_unsupported_majority` | Gauge | 1 if unsupported amendment has majority | +| `xrpl_validator_list_active` | Gauge | 1 if validator list is active | + +#### Fee Market (from fee) + +| Prometheus Metric | Type | Description | +| -------------------------------- | ----- | ------------------------------------- | +| `xrpl_fee_open_ledger_fee_drops` | Gauge | Minimum fee for open ledger inclusion | +| `xrpl_fee_median_fee_drops` | Gauge | Median fee level | +| `xrpl_fee_queue_size` | Gauge | Current transaction queue depth | +| `xrpl_fee_current_ledger_size` | Gauge | Transactions in current open ledger | + +#### DEX & AMM (optional, from book_offers, amm_info) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------- | ----- | --------------------- | ---------------------- | +| `xrpl_amm_tvl_drops` | Gauge | `pool=""` | Total value locked | +| `xrpl_amm_trading_fee` | Gauge | `pool=""` | Pool trading fee (bps) | +| `xrpl_orderbook_bid_depth` | Gauge | `pair=""` | Total bid volume | +| `xrpl_orderbook_ask_depth` | Gauge | `pair=""` | Total ask volume | +| `xrpl_orderbook_spread` | Gauge | `pair=""` | Best bid-ask spread | + +### Phase 9: OTel SDK-Exported Metrics (MetricsRegistry) + +Phase 9 introduces the `MetricsRegistry` class (`src/xrpld/telemetry/MetricsRegistry.h/.cpp`) +which registers metrics directly with the OpenTelemetry Metrics SDK. These are exported +via OTLP/HTTP to the OTel Collector and scraped by Prometheus. + +#### NodeStore I/O (Observable Gauge — `nodestore_state`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ----- | -------- | ------------------------------------ | +| `nodestore_state{metric="node_reads_total"}` | Gauge | `metric` | Cumulative NodeStore read operations | +| `nodestore_state{metric="node_reads_hit"}` | Gauge | `metric` | Reads served from cache | +| `nodestore_state{metric="node_writes"}` | Gauge | `metric` | Cumulative write operations | +| `nodestore_state{metric="node_written_bytes"}` | Gauge | `metric` | Cumulative bytes written | +| `nodestore_state{metric="node_read_bytes"}` | Gauge | `metric` | Cumulative bytes read | +| `nodestore_state{metric="write_load"}` | Gauge | `metric` | Current write load score | +| `nodestore_state{metric="read_queue"}` | Gauge | `metric` | Items in read prefetch queue | + +#### Cache Hit Rates & Sizes (Observable Gauge — `cache_metrics`) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ----- | -------- | ----------------------------- | +| `cache_metrics{metric="sle_hit_rate"}` | Gauge | `metric` | SLE cache hit rate (0.0-1.0) | +| `cache_metrics{metric="ledger_hit_rate"}` | Gauge | `metric` | Ledger cache hit rate | +| `cache_metrics{metric="al_hit_rate"}` | Gauge | `metric` | AcceptedLedger cache hit rate | +| `cache_metrics{metric="treenode_cache_size"}` | Gauge | `metric` | SHAMap TreeNode cache entries | +| `cache_metrics{metric="treenode_track_size"}` | Gauge | `metric` | Tracked tree nodes | +| `cache_metrics{metric="fullbelow_size"}` | Gauge | `metric` | FullBelow cache entries | + +#### Transaction Queue (Observable Gauge — `txq_metrics`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------------- | ----- | -------- | -------------------------------- | +| `txq_metrics{metric="txq_count"}` | Gauge | `metric` | Transactions currently in queue | +| `txq_metrics{metric="txq_max_size"}` | Gauge | `metric` | Maximum queue capacity | +| `txq_metrics{metric="txq_in_ledger"}` | Gauge | `metric` | Transactions in open ledger | +| `txq_metrics{metric="txq_per_ledger"}` | Gauge | `metric` | Expected transactions per ledger | +| `txq_metrics{metric="txq_reference_fee_level"}` | Gauge | `metric` | Reference fee level | +| `txq_metrics{metric="txq_min_processing_fee_level"}` | Gauge | `metric` | Minimum fee to get processed | +| `txq_metrics{metric="txq_med_fee_level"}` | Gauge | `metric` | Median fee level in queue | +| `txq_metrics{metric="txq_open_ledger_fee_level"}` | Gauge | `metric` | Open ledger fee escalation level | + +#### Per-RPC Method Metrics (Synchronous Counters/Histogram) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------- | --------- | ----------------- | -------------------------------- | +| `rpc_method_started_total` | Counter | `method=""` | RPC calls started | +| `rpc_method_finished_total` | Counter | `method=""` | RPC calls completed successfully | +| `rpc_method_errored_total` | Counter | `method=""` | RPC calls that errored | +| `rpc_method_us` | Histogram | `method=""` | Execution time distribution (us) | + +#### Per-Job-Type Metrics (Synchronous Counters/Histogram) + +| Prometheus Metric | Type | Labels | Description | +| -------------------- | --------- | ------------------- | --------------------------------- | +| `job_queued_total` | Counter | `job_type=""` | Jobs enqueued | +| `job_started_total` | Counter | `job_type=""` | Jobs started | +| `job_finished_total` | Counter | `job_type=""` | Jobs completed | +| `job_queued_us` | Histogram | `job_type=""` | Queue wait time distribution (us) | +| `job_running_us` | Histogram | `job_type=""` | Execution time distribution (us) | + +#### Counted Object Instances (Observable Gauge — `object_count`) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------- | ----- | --------------- | ------------------------------ | +| `object_count{type="transaction"}` | Gauge | `type=""` | Live Transaction objects | +| `object_count{type="ledger"}` | Gauge | `type=""` | Live Ledger objects | +| `object_count{type="nodeobject"}` | Gauge | `type=""` | Live NodeObject instances | +| `object_count{type="sttx"}` | Gauge | `type=""` | Serialized transaction objects | +| `object_count{type="stledgerentry"}` | Gauge | `type=""` | Serialized ledger entries | +| `object_count{type="inboundledger"}` | Gauge | `type=""` | Ledgers being fetched | +| `object_count{type="pathfinder"}` | Gauge | `type=""` | Active pathfinding operations | +| `object_count{type="pathrequest"}` | Gauge | `type=""` | Active path requests | +| `object_count{type="hashrouterentry"}` | Gauge | `type=""` | Hash router entries | + +#### Load Factor Breakdown (Observable Gauge — `load_factor_metrics`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------------------- | ----- | -------- | --------------------------------------- | +| `load_factor_metrics{metric="load_factor"}` | Gauge | `metric` | Combined transaction cost multiplier | +| `load_factor_metrics{metric="load_factor_server"}` | Gauge | `metric` | Server + cluster + network contribution | +| `load_factor_metrics{metric="load_factor_local"}` | Gauge | `metric` | Local server load only | +| `load_factor_metrics{metric="load_factor_net"}` | Gauge | `metric` | Network-wide load estimate | +| `load_factor_metrics{metric="load_factor_cluster"}` | Gauge | `metric` | Cluster peer load | +| `load_factor_metrics{metric="load_factor_fee_escalation"}` | Gauge | `metric` | Open ledger fee escalation | +| `load_factor_metrics{metric="load_factor_fee_queue"}` | Gauge | `metric` | Queue entry fee level | + +#### Prometheus Query Examples (Phase 9) + +```promql +# NodeStore cache hit ratio +nodestore_state{metric="node_reads_hit"} / nodestore_state{metric="node_reads_total"} + +# RPC error rate for server_info +rate(rpc_method_errored_total{method="server_info"}[5m]) + +# Job queue wait time p95 +histogram_quantile(0.95, sum by (le) (rate(job_queued_us_bucket[5m]))) + +# TxQ utilization percentage +txq_metrics{metric="txq_count"} / txq_metrics{metric="txq_max_size"} + +# High load factor alert candidate +load_factor_metrics{metric="load_factor"} > 5 +``` + +### Phase 7+: External Dashboard Parity Metrics + +> **Source**: [External Dashboard Parity Spec](./06-implementation-phases.md#appendix-external-dashboard-parity) — metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Task breakdown**: Phase 7 Tasks 7.9-7.16 (implementation), Phase 9 Tasks 9.11-9.13 (dashboards) + +These metrics fill gaps identified by comparing xrpld's internal observability with the community external dashboard's 86-metric coverage. All are exported via the OTel Metrics SDK (same `PeriodicMetricReader` as Phase 9 metrics). + +#### Validation Agreement (Observable Gauge — `validation_agreement`) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ------ | -------- | --------------------------------------- | +| `validation_agreement{metric="agreement_pct_1h"}` | Double | `metric` | Rolling 1h agreement percentage (0-100) | +| `validation_agreement{metric="agreement_pct_24h"}` | Double | `metric` | Rolling 24h agreement percentage | +| `validation_agreement{metric="agreements_1h"}` | Int64 | `metric` | Agreed validations in 1h window | +| `validation_agreement{metric="missed_1h"}` | Int64 | `metric` | Missed validations in 1h window | +| `validation_agreement{metric="agreements_24h"}` | Int64 | `metric` | Agreed validations in 24h window | +| `validation_agreement{metric="missed_24h"}` | Int64 | `metric` | Missed validations in 24h window | + +Data source: `ValidationTracker` class with 8s grace period and 5m late repair window. + +#### Validator Health (Observable Gauge — `validator_health`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ------ | -------- | ------------------------------ | +| `validator_health{metric="amendment_blocked"}` | Int64 | `metric` | 1 if amendment-blocked, else 0 | +| `validator_health{metric="unl_blocked"}` | Int64 | `metric` | 1 if UNL-blocked, else 0 | +| `validator_health{metric="unl_expiry_days"}` | Double | `metric` | Days until UNL list expires | +| `validator_health{metric="validation_quorum"}` | Int64 | `metric` | Validation quorum threshold | + +#### Peer Quality (Observable Gauge — `peer_quality`) + +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------------------- | ------ | -------- | ------------------------------------ | +| `peer_quality{metric="peer_latency_p90_ms"}` | Double | `metric` | P90 peer latency in milliseconds | +| `peer_quality{metric="peers_insane_count"}` | Int64 | `metric` | Peers with diverged tracking status | +| `peer_quality{metric="peers_higher_version_pct"}` | Double | `metric` | % of peers on newer xrpld version | +| `peer_quality{metric="upgrade_recommended"}` | Int64 | `metric` | 1 if >60% of peers are newer version | + +#### Ledger Economy (Observable Gauge — `ledger_economy`) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ------ | -------- | ---------------------------------- | +| `ledger_economy{metric="base_fee_xrp"}` | Double | `metric` | Base transaction fee in drops | +| `ledger_economy{metric="reserve_base_xrp"}` | Double | `metric` | Account reserve in drops | +| `ledger_economy{metric="reserve_inc_xrp"}` | Double | `metric` | Owner reserve increment in drops | +| `ledger_economy{metric="ledger_age_seconds"}` | Double | `metric` | Seconds since last validated close | +| `ledger_economy{metric="transaction_rate"}` | Double | `metric` | Smoothed transaction rate (tx/s) | + +#### State Tracking (Observable Gauge — `state_tracking`) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------------- | ------ | -------- | -------------------------------------- | +| `state_tracking{metric="state_value"}` | Int64 | `metric` | Numeric state 0-6 (see encoding below) | +| `state_tracking{metric="time_in_current_state_seconds"}` | Double | `metric` | Duration in current state | + +State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (FULL + validating), 6=proposing (FULL + proposing). + +#### Storage Detail (Observable Gauge — `storage_detail`) + +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------- | ----- | -------- | ---------------------- | +| `storage_detail{metric="nudb_bytes"}` | Int64 | `metric` | NuDB backend file size | + +#### Synchronous Counters (Phase 7+) + +| Prometheus Metric | Type | Description | Increment Site | +| --------------------------- | ------- | ---------------------------- | ---------------- | +| `ledgers_closed_total` | Counter | Ledgers closed by consensus | RCLConsensus.cpp | +| `validations_sent_total` | Counter | Validations sent | RCLConsensus.cpp | +| `validations_checked_total` | Counter | Network validations observed | LedgerMaster.cpp | +| `state_changes_total` | Counter | Operating mode transitions | NetworkOPs.cpp | + +Lifetime tallies exported as monotonic **ObservableCounters** (not synchronous +counters), observed from an existing cumulative source each collection cycle: + +| Prometheus Metric | Type | Description | Source | +| ----------------------------- | ----------------- | ------------------------------------------ | ---------------------------------------------------- | +| `validation_agreements_total` | ObservableCounter | Lifetime validations that initially agreed | ValidationTracker.cpp | +| `validation_missed_total` | ObservableCounter | Lifetime validations that initially missed | ValidationTracker.cpp | +| `jq_trans_overflow_total` | ObservableCounter | Job queue transaction overflows | Overlay::getJqTransOverflow (PeerImp.cpp increments) | + +> **Counting semantics (initial-classification only):** each reconciled ledger increments exactly +> one of these two counters, at first classification. A later late-repair (miss → agreement) does +> **not** move either counter — keeping both strictly monotonic (a Prometheus `_total` must never +> decrease) and additive (`agreements_total + missed_total` = ledgers reconciled). The +> repair-aware, windowed view remains on `validation_agreement{metric="…"}`. + +#### Span Attribute Enrichments (Phases 2-4) + +| Span Name | New Attribute | Type | Source | +| --------------------------- | ------------------------------------ | ------ | ------------------------ | +| `rpc.command.*` | `xrpl.node.amendment_blocked` | bool | Phase 2 — RPCHandler.cpp | +| `rpc.command.*` | `xrpl.node.server_state` | string | Phase 2 — RPCHandler.cpp | +| `tx.receive` | `xrpl.peer.version` | string | Phase 3 — PeerImp.cpp | +| `consensus.validation.send` | `xrpl.validation.ledger_hash` | string | Phase 4 — RCLConsensus | +| `consensus.validation.send` | `xrpl.validation.full` | bool | Phase 4 — RCLConsensus | +| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | string | Phase 4 — PeerImp.cpp | +| `peer.validation.receive` | `xrpl.peer.validation.full` | bool | Phase 4 — PeerImp.cpp | +| `consensus.accept` | `xrpl.consensus.validation_quorum` | int64 | Phase 4 — RCLConsensus | +| `consensus.accept` | `xrpl.consensus.proposers_validated` | int64 | Phase 4 — RCLConsensus | + +### New Grafana Dashboards (Phase 9) + +| Dashboard | UID | Data Source | Key Panels | +| ------------------------------------ | ------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------ | +| Fee Market & TxQ | `fee-market` | Prometheus | TxQ depth/capacity, fee levels, load factor breakdown | +| Job Queue Analysis | `job-queue` | Prometheus | Per-job rates, queue wait times, execution times | +| RPC Performance (per-method section) | `rpc-performance` | Prometheus | Per-method call rates, error rates, latency distributions (added as a section to the existing RPC Performance dashboard) | +| Validator Health | `validator-health` | Prometheus | Agreement %, validation rate, amendment/UNL, state | +| Peer Quality | `peer-quality` | Prometheus | P90 latency, insane peers, version awareness, disconnects | + +### Updated Grafana Dashboards (Phase 9) + +| Dashboard | UID | New Panels Added | +| -------------------- | -------------------------- | -------------------------------------------------------------------- | +| Node Health (StatsD) | `xrpld-statsd-node-health` | NodeStore I/O, cache hit rates, object instance counts | +| System Node Health | `node-health` | Ledger economy row: base fee, reserves, ledger age, transaction rate | + +### New Grafana Dashboards (Phase 11) + +| Dashboard | UID | Data Source | Key Panels | +| ------------------ | --------------------------- | ----------- | ---------------------------------------------------------------------- | +| Validator Health | `validator-health` | Prometheus | Server state timeline, proposer count, converge time, amendment voting | +| Network Topology | `xrpld-network-topology` | Prometheus | Peer count, version distribution, latency distribution, diverged peers | +| Fee Market (Ext) | `xrpld-fee-market-external` | Prometheus | Fee levels, queue depth, load factor breakdown, escalation timeline | +| DEX & AMM Overview | `xrpld-dex-amm` | Prometheus | AMM TVL, order book depth, spread trends, trading fee revenue | + +### Prometheus Alerting Rules (Phase 11) + +| Alert Name | Severity | Condition | For | +| ---------------------------------- | -------- | ----------------------------------------------------------- | --- | +| `XRPLServerNotFull` | Critical | `xrpl_server_state < 4` for 15m | 15m | +| `XRPLAmendmentBlocked` | Critical | `xrpl_amendment_blocked == 1` | 1m | +| `XRPLNoPeers` | Critical | `xrpl_peers_count == 0` | 5m | +| `XRPLLedgerStale` | Critical | `xrpl_validated_ledger_age_seconds > 120` | 2m | +| `XRPLHighIOLatency` | Critical | `xrpl_io_latency_ms > 100` | 5m | +| `XRPLUnsupportedAmendmentMajority` | Critical | `xrpl_amendment_unsupported_majority == 1` | 1m | +| `XRPLLowPeerCount` | Warning | `xrpl_peers_count < 10` | 15m | +| `XRPLHighLoadFactor` | Warning | `xrpl_load_factor > 10` | 10m | +| `XRPLSlowConsensus` | Warning | `xrpl_last_close_converge_time_seconds > 6` | 5m | +| `XRPLValidatorListExpiring` | Warning | `(xrpl_validator_list_expiration_seconds - time()) < 86400` | 1h | +| `XRPLStateFlapping` | Warning | `rate(xrpl_state_transitions_total{state="full"}[1h]) > 2` | 30m | + +--- + +## 6. Known Issues + +| Issue | Impact | Status | +| ------------------------------------------------------------------ | ------------------------------------------------ | -------------------------------------------------------------------- | +| `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | +| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | +| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | +| Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | + +--- + +## 7. Privacy and Data Collection + +The telemetry system is designed with privacy in mind: + +- **No private keys** are ever included in spans or metrics +- **No account balances** or financial data is traced +- **Transaction hashes** are included (public on-ledger data) but not transaction contents +- **Peer IDs** are internal identifiers, not IP addresses +- **All telemetry is opt-in** — disabled by default at build time (`-Dtelemetry=OFF`) +- **Sampling** — head sampling is fixed at 1.0 (sample everything); reduce data volume with collector-side tail sampling +- **Data stays local** — the default stack sends data to `localhost` only + +--- + +## 8. Configuration Quick Reference + +> **Full reference**: [05-configuration-reference.md](./05-configuration-reference.md) §5.1 for all `[telemetry]` options with defaults, the config parser implementation, and collector YAML configurations (dev and production). + +### Minimal Setup (development) + +```ini +[telemetry] +enabled=1 + +[insight] +server=statsd +address=127.0.0.1:8125 +prefix=xrpld +``` + +### Production Setup + +```ini +[telemetry] +enabled=1 +endpoint=http://otel-collector:4318/v1/traces +trace_peer=0 +batch_size=1024 +max_queue_size=4096 + +[insight] +server=statsd +address=otel-collector:8125 +prefix=xrpld +``` + +### Trace Category Toggle + +| Config Key | Default | Controls | +| -------------------- | ------- | ---------------------------- | +| `trace_rpc` | `1` | `rpc.*` spans | +| `trace_transactions` | `1` | `tx.*` spans | +| `trace_consensus` | `1` | `consensus.*` spans | +| `trace_ledger` | `1` | `ledger.*` spans | +| `trace_peer` | `1` | `peer.*` spans (high volume) | diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md new file mode 100644 index 00000000000..b7289d5004d --- /dev/null +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -0,0 +1,226 @@ +# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld + +## Executive Summary + +> **OTLP** = OpenTelemetry Protocol + +This document provides a comprehensive implementation plan for integrating OpenTelemetry distributed tracing into the xrpld XRP Ledger node software. The plan addresses the unique challenges of a decentralized peer-to-peer system where trace context must propagate across network boundaries between independent nodes. + +### Key Benefits + +- **End-to-end transaction visibility**: Track transactions from submission through consensus to ledger inclusion +- **Consensus round analysis**: Understand timing and behavior of consensus phases across validators +- **RPC performance insights**: Identify slow handlers and optimize response times +- **Network topology understanding**: Visualize message propagation patterns between peers +- **Incident debugging**: Correlate events across distributed nodes during issues + +### Estimated Performance Overhead + +| Metric | Overhead | Notes | +| ------------- | ---------- | ------------------------------------------------ | +| CPU | 1-3% | Span creation and attribute setting | +| Memory | <10 MB | SDK statics + batch buffer + worker thread stack | +| Network | 10-50 KB/s | Compressed OTLP export to collector | +| Latency (p99) | <2% | With proper sampling configuration | + +--- + +## Document Structure + +This implementation plan is organized into modular documents for easier navigation: + +
+ +```mermaid +flowchart TB + overview["📋 OpenTelemetryPlan.md
(This Document)"] + + subgraph fundamentals["Fundamentals"] + fund["00-tracing-fundamentals.md"] + end + + subgraph analysis["Analysis & Design"] + arch["01-architecture-analysis.md"] + design["02-design-decisions.md"] + end + + subgraph impl["Implementation"] + strategy["03-implementation-strategy.md"] + config["05-configuration-reference.md"] + end + + subgraph deploy["Deployment & Planning"] + phases["06-implementation-phases.md"] + backends["07-observability-backends.md"] + appendix["08-appendix.md"] + secure["secure-OTel.md"] + dataref["09-data-collection-reference.md"] + end + + overview --> fundamentals + overview --> analysis + overview --> impl + overview --> deploy + + fund --> arch + arch --> design + design --> strategy + strategy --> config + config --> phases + phases --> backends + backends --> appendix + backends --> secure + appendix --> dataref + + style overview fill:#1b5e20,stroke:#0d3d14,color:#fff,stroke-width:2px + style fundamentals fill:#00695c,stroke:#004d40,color:#fff + style fund fill:#00695c,stroke:#004d40,color:#fff + style analysis fill:#0d47a1,stroke:#082f6a,color:#fff + style impl fill:#bf360c,stroke:#8c2809,color:#fff + style deploy fill:#4a148c,stroke:#2e0d57,color:#fff + style arch fill:#0d47a1,stroke:#082f6a,color:#fff + style design fill:#0d47a1,stroke:#082f6a,color:#fff + style strategy fill:#bf360c,stroke:#8c2809,color:#fff + style config fill:#bf360c,stroke:#8c2809,color:#fff + style phases fill:#4a148c,stroke:#2e0d57,color:#fff + style backends fill:#4a148c,stroke:#2e0d57,color:#fff + style appendix fill:#4a148c,stroke:#2e0d57,color:#fff + style secure fill:#4a148c,stroke:#2e0d57,color:#fff + style dataref fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +
+ +--- + +## Table of Contents + +| Section | Document | Description | +| ------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- | +| **0** | [Tracing Fundamentals](./00-tracing-fundamentals.md) | Distributed tracing concepts, span relationships, context propagation | +| **1** | [Architecture Analysis](./01-architecture-analysis.md) | xrpld component analysis, trace points, instrumentation priorities | +| **2** | [Design Decisions](./02-design-decisions.md) | SDK selection, exporters, span naming, attributes, context propagation | +| **3** | [Implementation Strategy](./03-implementation-strategy.md) | Directory structure, key principles, performance optimization | +| **5** | [Configuration Reference](./05-configuration-reference.md) | xrpld config, CMake integration, Collector configurations | +| **6** | [Implementation Phases](./06-implementation-phases.md) | 5-phase timeline, tasks, risks, success metrics | +| **7** | [Observability Backends](./07-observability-backends.md) | Backend selection guide and production architecture | +| **8** | [Appendix](./08-appendix.md) | Glossary, references, version history | +| **9** | [Data Collection Reference](./09-data-collection-reference.md) | Complete inventory of spans, attributes, metrics, and dashboards | +| **Sec** | [Securing the OTel Pipeline](./secure-OTel.md) | Threat model and hardening (mTLS, peer trace-context validation) | + +--- + +## 0. Tracing Fundamentals + +This document introduces distributed tracing concepts for readers unfamiliar with the domain. It covers what traces and spans are, how parent-child and follows-from relationships model causality, how context propagates across service boundaries, and how sampling controls data volume. It also maps these concepts to xrpld-specific scenarios like transaction relay and consensus. + +➡️ **[Read Tracing Fundamentals](./00-tracing-fundamentals.md)** + +--- + +## 1. Architecture Analysis + +> **WS** = WebSocket | **TxQ** = Transaction Queue + +The xrpld node consists of several key components that require instrumentation for comprehensive distributed tracing. The main areas include the RPC server (HTTP/WebSocket), Overlay P2P network, Consensus mechanism (RCLConsensus), JobQueue for async task execution, PathFinding, Transaction Queue (TxQ), fee escalation (LoadManager), ledger acquisition, validator management, and existing observability infrastructure (PerfLog, Insight/StatsD, Journal logging). + +Key trace points span across transaction submission via RPC, peer-to-peer message propagation, consensus round execution, ledger building, path computation, transaction queue behavior, fee escalation, and validator health. The implementation prioritizes high-value, low-risk components first: RPC handlers provide immediate value with minimal risk, while consensus tracing requires careful implementation to avoid timing impacts. + +➡️ **[Read full Architecture Analysis](./01-architecture-analysis.md)** + +--- + +## 2. Design Decisions + +> **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation + +The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). + +Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. + +**Data Collection & Privacy**: Telemetry collects only operational metadata (timing, counts, hashes) — never sensitive content (private keys, balances, amounts, raw payloads). Privacy protection includes account hashing, configurable redaction, sampling, and collector-level filtering. Node operators retain full control over telemetry configuration. + +➡️ **[Read full Design Decisions](./02-design-decisions.md)** + +--- + +## 3. Implementation Strategy + +The telemetry code is organized under `include/xrpl/telemetry/` for headers and `src/libxrpl/telemetry/` for implementation. Key principles include RAII-based span management via `SpanGuard` (with `discard()` for dropping unwanted spans), a `FilteringSpanProcessor` that intercepts `OnEnd()` to prevent discarded spans from entering the export pipeline, conditional compilation with `XRPL_ENABLE_TELEMETRY`, and minimal runtime overhead through batch processing and efficient sampling. + +Performance optimization strategies include head sampling fixed at 100% (intentionally not configurable, so trace keep/drop decisions stay coherent across nodes), tail-based sampling at the collector for errors and slow traces to reduce volume, batch export to reduce network overhead, and conditional instrumentation that compiles to no-ops when disabled. + +➡️ **[Read full Implementation Strategy](./03-implementation-strategy.md)** + +--- + +## 5. Configuration Reference + +> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring + +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. + +OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. + +➡️ **[View full Configuration Reference](./05-configuration-reference.md)** + +--- + +## 6. Implementation Phases + +The implementation spans 13 weeks across 8 phases: + +| Phase | Duration | Focus | Key Deliverables | +| ----- | ----------- | --------------------- | ----------------------------------------------------------- | +| 1 | Weeks 1-2 | Core Infrastructure | SDK integration, Telemetry interface, Configuration | +| 2 | Weeks 3-4 | RPC Tracing | HTTP context extraction, Handler instrumentation | +| 3 | Weeks 5-6 | Transaction Tracing | Protocol Buffer context, Relay propagation | +| 4 | Weeks 7-8 | Consensus Tracing | Round spans, Proposal/validation tracing | +| 5 | Week 9 | Documentation | Runbook, Dashboards, Training | +| 6 | Week 10 | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards | +| 7 | Weeks 11-12 | Native OTel Metrics | OTelCollector impl, OTLP metrics export, StatsD deprecation | +| 8 | Week 13 | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking | + +**Total Effort**: 65.1 developer-days with 2 developers + +➡️ **[View full Implementation Phases](./06-implementation-phases.md)** + +--- + +## 7. Observability Backends + +> **APM** = Application Performance Monitoring | **GCS** = Google Cloud Storage + +Grafana Tempo is recommended for all environments due to its cost-effectiveness and Grafana integration, while Elastic APM is ideal for organizations with existing Elastic infrastructure. + +The recommended production architecture uses a gateway collector pattern with regional collectors performing tail-based sampling, routing traces to multiple backends (Tempo for primary storage, Elastic for log correlation, S3/GCS for long-term archive). + +➡️ **[View Observability Backend Recommendations](./07-observability-backends.md)** + +--- + +## 8. Appendix + +The appendix contains a glossary of OpenTelemetry and xrpld-specific terms, references to external documentation and specifications, version history for this implementation plan, and a complete document index. + +➡️ **[View Appendix](./08-appendix.md)** + +--- + +## 9. Data Collection Reference + +A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld. Covers all 16 OpenTelemetry spans with their 22 attributes, all StatsD metrics (gauges, counters, histograms, overlay traffic), SpanMetrics-derived Prometheus metrics, and all 10 Grafana dashboards. Includes Tempo search guides and Prometheus query examples. + +➡️ **[View Data Collection Reference](./09-data-collection-reference.md)** + +--- + +## Securing the OTel Pipeline + +Threat model and hardening guidance for production deployments where xrpld nodes ship telemetry to a centrally-hosted collector across an untrusted network. Covers the two attack surfaces (collector ingress and peer trace-context spoofing) and the chosen defenses: mTLS as primary collector auth, NetworkPolicy as defense-in-depth, and source-side validation plus per-peer rate limiting for the `protocol::TraceContext` field on peer messages. + +➡️ **[View Securing the OTel Pipeline](./secure-OTel.md)** + +--- + +_This document provides a comprehensive implementation plan for integrating OpenTelemetry distributed tracing into the xrpld XRP Ledger node software. For detailed information on any section, follow the links to the corresponding sub-documents._ diff --git a/OpenTelemetryPlan/Phase10_taskList.md b/OpenTelemetryPlan/Phase10_taskList.md new file mode 100644 index 00000000000..7022652b9ed --- /dev/null +++ b/OpenTelemetryPlan/Phase10_taskList.md @@ -0,0 +1,258 @@ +# Phase 10: Synthetic Workload Generation & Telemetry Validation — Task List + +> **Status**: Future Enhancement +> +> **Goal**: Build tools that generate realistic XRPL traffic to validate the full Phases 1-9 telemetry stack end-to-end — all spans, attributes, metrics, dashboards, and log-trace correlation — under controlled load. +> +> **Scope**: Python/shell test harness + multi-node docker-compose environment + automated validation scripts + performance benchmarks. +> +> **Branch**: `pratik/otel-phase10-workload-validation` (from `pratik/otel-phase9-metric-gap-fill`) +> +> **Depends on**: Phase 9 (internal metric gap fill) — validates the full metric surface + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | --------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 10 plan: motivation, architecture, exit criteria (§6.8.3) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines the full inventory of spans/metrics to validate | +| [Phase9_taskList.md](./Phase9_taskList.md) | Prerequisite — all internal metrics must be emitting | + +### Why This Phase Exists + +Before Phases 1-9 can be considered production-ready, we need proof that: + +1. All required spans fire with correct attributes under real transaction workloads +2. All 255+ StatsD metrics + ~50 Phase 9 metrics appear in Prometheus with non-zero values +3. Log-trace correlation (Phase 8) produces clickable trace_id links in Loki +4. All 10 Grafana dashboards render meaningful data (no empty panels) +5. Performance overhead stays within bounds (< 3% CPU, < 5MB memory) +6. The telemetry stack survives sustained load without data loss or queue backpressure + +--- + +## Task 10.1: Multi-Node Test Harness + +**Objective**: Create a docker-compose environment with 3-5 validator nodes that produces real consensus rounds. + +**What to do**: + +- Create `docker/telemetry/docker-compose.workload.yaml`: + - 5 xrpld validator nodes with UNL configured for each other + - All telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel` + - Full OTel stack: Collector, Tempo, Prometheus, Loki, Grafana + - Shared network with service discovery + +- Each node should: + - Generate validator keys at startup + - Configure all 5 nodes in its UNL + - Enable all trace categories including `trace_peer=1` + - Write logs to a file tailed by the OTel Collector filelog receiver + +- Include a `Makefile` target: `make telemetry-workload-up` / `make telemetry-workload-down` + +**Key files**: + +- New: `docker/telemetry/docker-compose.workload.yaml` +- New: `docker/telemetry/workload/generate-validator-keys.sh` +- New: `docker/telemetry/workload/xrpld-validator.cfg.template` + +--- + +## Task 10.2: RPC Load Generator + +**Objective**: Configurable tool that fires all traced RPC commands at controlled rates. + +**What to do**: + +- Create `docker/telemetry/workload/rpc_load_generator.py`: + - Connects to one or more xrpld WebSocket endpoints + - Fires all RPC commands that have trace spans: `server_info`, `ledger`, `tx`, `account_info`, `account_lines`, `fee`, `submit`, etc. + - Configurable parameters: rate (RPS), duration, command distribution weights + - Injects `traceparent` HTTP headers to test W3C context propagation + - Logs progress and errors to stdout + +- Command distribution should match realistic production ratios: + - 40% `server_info` / `fee` (health checks) + - 30% `account_info` / `account_lines` / `account_objects` (wallet queries) + - 15% `ledger` / `ledger_data` (explorer queries) + - 10% `tx` / `account_tx` (transaction lookups) + - 5% `book_offers` / `amm_info` (DEX queries) + +**Key files**: + +- New: `docker/telemetry/workload/rpc_load_generator.py` +- New: `docker/telemetry/workload/requirements.txt` + +--- + +## Task 10.3: Transaction Submitter + +**Objective**: Generate diverse transaction types to exercise `tx.*` and `ledger.*` spans. + +**What to do**: + +- Create `docker/telemetry/workload/tx_submitter.py`: + - Pre-funds test accounts from genesis account + - Submits a mix of transaction types: + - `Payment` (XRP and issued currencies) — exercises `tx.process`, `tx.apply` + - `OfferCreate` / `OfferCancel` — DEX activity + - `TrustSet` — trust line creation for issued currencies + - `NFTokenMint` / `NFTokenCreateOffer` / `NFTokenAcceptOffer` — NFT activity + - `EscrowCreate` / `EscrowFinish` — escrow lifecycle + - `AMMCreate` / `AMMDeposit` / `AMMWithdraw` — AMM pool operations (if amendment enabled) + - Configurable: TPS target, transaction mix weights, duration + - Monitors submission results and tracks success/failure rates + +- The transaction mix ensures the telemetry captures the full range of ledger activity that third parties care about. + +**Key files**: + +- New: `docker/telemetry/workload/tx_submitter.py` +- New: `docker/telemetry/workload/test_accounts.json` (pre-generated keypairs) + +--- + +## Task 10.4: Telemetry Validation Suite + +**Objective**: Automated scripts that verify all expected telemetry data exists after a workload run. + +**What to do**: + +- Create `docker/telemetry/workload/validate_telemetry.py`: + + **Span validation** (queries Tempo API): + - Assert all required span names appear in traces (conditional spans — `grpc.*`, + `ledger.acquire`, `txq.*`, `consensus.mode_change` — are marked `optional` and + skipped when not exercised by the workload) + - Assert each span has its required attributes (bare/underscore keys per the + 2026-05-13 span-attr naming redesign; dotted `xrpl.*` reserved for resource attrs) + - Assert parent-child relationships are correct (`rpc.ws_message` → `rpc.process` → `rpc.command.*`) + - Assert span durations are reasonable (> 0, < 60s) + + **Metric validation** (queries Prometheus API): + - Assert all SpanMetrics-derived metrics are non-zero: `traces_span_metrics_calls_total`, `traces_span_metrics_duration_milliseconds_bucket` + - Assert all StatsD metrics are non-zero: `xrpld_LedgerMaster_Validated_Ledger_Age`, `xrpld_Peer_Finder_Active_*`, etc. + - Assert all Phase 9 metrics are non-zero: `xrpld_nodestore_*`, `xrpld_cache_*`, `xrpld_txq_*`, `xrpld_rpc_method_*`, `xrpld_object_count`, `xrpld_load_factor*` + - Assert metric label cardinality is within bounds + + **Log-trace correlation validation** (queries Loki API): + - Assert logs contain `trace_id=` and `span_id=` fields + - Pick a random trace_id from Tempo → query Loki for matching logs → assert results exist + - Assert Grafana derived field links are functional + + **Dashboard validation**: + - For each of the 10 Grafana dashboards, query the dashboard API and assert no panels show "No data" + +- Output: JSON report with pass/fail per check, suitable for CI. + +**Key files**: + +- New: `docker/telemetry/workload/validate_telemetry.py` +- New: `docker/telemetry/workload/expected_spans.json` (span inventory for validation) +- New: `docker/telemetry/workload/expected_metrics.json` (metric inventory for validation) + +--- + +## Task 10.5: Performance Benchmark Suite + +**Objective**: Measure CPU/memory/latency overhead of the telemetry stack. + +**What to do**: + +- Create `docker/telemetry/workload/benchmark.sh`: + - **Baseline run**: Start cluster with `[telemetry] enabled=0`, run transaction workload for 5 minutes, record metrics + - **Telemetry run**: Start cluster with full telemetry enabled, run identical workload, record metrics + - **Comparison**: Calculate deltas for: + - CPU usage (per-node average) + - Memory RSS (per-node peak) + - RPC p99 latency + - Transaction throughput (TPS) + - Consensus round time p95 + - Ledger close time p95 + +- Output: Markdown table comparing baseline vs. telemetry, with pass/fail against targets: + - CPU overhead < 3% + - Memory overhead < 5MB + - RPC latency impact < 2ms p99 + - Throughput impact < 5% + - Consensus impact < 1% + +- Store results in `docker/telemetry/workload/benchmark-results/` for historical tracking. + +**Key files**: + +- New: `docker/telemetry/workload/benchmark.sh` +- New: `docker/telemetry/workload/collect_system_metrics.sh` + +--- + +## Task 10.6: CI Integration + +**Objective**: Wire the validation suite into CI for regression detection. + +**What to do**: + +- Create a CI workflow (GitHub Actions or equivalent) that: + 1. Builds xrpld with `-DXRPL_ENABLE_TELEMETRY=ON` + 2. Starts the multi-node workload harness + 3. Runs the RPC load generator + transaction submitter for 2 minutes + 4. Runs the validation suite + 5. Runs the benchmark suite + 6. Fails the build if any validation check fails or benchmark exceeds thresholds + 7. Archives the validation report and benchmark results as artifacts + +- This should be a separate workflow (not part of the main CI), triggered manually or on telemetry-related branch changes. + +**Key files**: + +- New: `.github/workflows/telemetry-validation.yml` +- New: `docker/telemetry/workload/run-full-validation.sh` (orchestrator script) + +--- + +## Task 10.7: Documentation + +**Objective**: Document the workload tools and validation process. + +**What to do**: + +- Create `docker/telemetry/workload/README.md`: + - Quick start guide for running workload harness + - Configuration options for load generator and tx submitter + - How to read validation reports + - How to run benchmarks and interpret results + +- Update `docs/telemetry-runbook.md`: + - Add "Validating Telemetry Stack" section + - Add "Performance Benchmarking" section + +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add "Validation" section with expected metric/span counts + +--- + +## Exit Criteria — Delivered in PR #6519 + +- [x] Multi-node validator cluster starts and reaches consensus +- [x] RPC load generator fires all traced RPC commands at configurable rates +- [x] Transaction submitter generates 6+ transaction types at configurable TPS +- [x] Validation suite confirms all required spans, attributes, and metrics +- [x] Log-trace correlation validated end-to-end (Loki ↔ Tempo) +- [x] Grafana dashboards render data (no empty panels) +- [x] Overhead benchmark (`benchmark.sh`) measures telemetry-off vs telemetry-on deltas +- [x] CI workflow runs validation on telemetry branch changes +- [x] Validation report output is CI-parseable (JSON with exit codes) +- [x] OTel-driven regression gate captures per-span/per-RPC/per-job timings from + Prometheus and compares against a committed baseline + +## Follow-up Work (tracked in separate PRs) + +- [ ] FU-2: Automate baseline persistence across CI runs (artifact uploaded + on merge to `develop`, downloaded on PR runs). Current mechanism + requires a manual baseline-refresh PR. +- [ ] FU-4: Replace the proxy measurements in `benchmark.sh` (wall-clock curl + p99, ledger-cadence-as-TPS, ledger-cadence-as-consensus-p95) with + PromQL quantile queries from the same pipeline the regression gate uses. +- [ ] FU-6: Grafana dashboard plotting historical baseline values keyed by + commit SHA, for triaging noisy regressions. diff --git a/OpenTelemetryPlan/Phase11_taskList.md b/OpenTelemetryPlan/Phase11_taskList.md new file mode 100644 index 00000000000..7429063984a --- /dev/null +++ b/OpenTelemetryPlan/Phase11_taskList.md @@ -0,0 +1,544 @@ +# Phase 11: Third-Party Data Collection Pipelines — Task List + +> **Status**: Future Enhancement +> +> **Goal**: Build a custom OTel Collector receiver that periodically polls xrpld's admin RPCs and exports structured metrics for external consumers — making all XRPL health, validator, peer, fee, and DEX data available as Prometheus/OTLP metrics without xrpld code changes. +> +> **Scope**: Go-based OTel Collector receiver plugin + Grafana dashboards + Prometheus alerting rules. +> +> **Branch**: `pratik/otel-phase11-third-party-collection` (from `pratik/otel-phase10-workload-validation`) +> +> **Depends on**: Phase 10 (validation harness for testing the new receiver) + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | --------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 11 plan: motivation, architecture, exit criteria (§6.8.4) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines full metric inventory including third-party metrics | +| [Phase10_taskList.md](./Phase10_taskList.md) | Prerequisite — validation harness for testing | + +### Third-Party Consumer Gap Analysis + +This phase addresses the cross-cutting gap identified during research: **xrpld has no native Prometheus/OTLP metrics export for data accessible only via RPC**. Every consumer (exchanges, payment processors, analytics providers, validators, researchers, compliance firms, custodians) must build custom JSON-RPC polling and conversion. This receiver centralizes that work. + +| Consumer Category | Data Unlocked by This Phase | +| -------------------------- | ------------------------------------------------------------------ | +| **Exchanges** | Real-time fee estimates, TxQ capacity, server health scores | +| **Payment Processors** | Settlement latency percentiles, corridor health, path availability | +| **Analytics Providers** | Validator metrics, network topology, amendment voting status | +| **DeFi / AMM** | AMM pool TVL, DEX order book depth, trade volumes | +| **Validators / Operators** | Per-peer latency, version distribution, UNL health, alerting | +| **Compliance** | Transaction volume trends, network growth metrics | +| **Academic Researchers** | Consensus performance time-series, decentralization metrics | +| **CBDC / Tokenization** | Token supply tracking, trust line adoption, freeze status | +| **Institutional Custody** | Multi-sig status, escrow tracking, reserve calculations | +| **Wallet Providers** | Server health for node selection, fee prediction data | + +--- + +## Task 11.1: OTel Collector Receiver Scaffold + +**Objective**: Create the Go project structure for a custom OTel Collector receiver that polls xrpld JSON-RPC. + +**What to do**: + +- Create `docker/telemetry/otel-rippled-receiver/`: + - `receiver.go` — implements `receiver.Metrics` interface + - `config.go` — configuration struct (endpoint, poll interval, enabled RPCs) + - `factory.go` — receiver factory registration + - `go.mod` / `go.sum` — Go module with OTel Collector SDK dependency + +- Configuration model: + + ```yaml + xrpld_receiver: + endpoint: "http://localhost:5005" # xrpld admin RPC + poll_interval: 30s # how often to poll + enabled_collectors: + - server_info + - get_counts + - fee + - peers + - validators + - feature + - server_state + amm_pools: [] # optional: AMM pool IDs to track + book_offers_pairs: [] # optional: currency pairs for DEX depth + ``` + +- Build a custom OTel Collector binary that includes this receiver alongside the standard receivers. + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/receiver.go` +- New: `docker/telemetry/otel-rippled-receiver/config.go` +- New: `docker/telemetry/otel-rippled-receiver/factory.go` +- New: `docker/telemetry/otel-rippled-receiver/go.mod` +- New: `docker/telemetry/otel-rippled-receiver/Dockerfile` + +--- + +## Task 11.2: server_info / server_state Collector + +**Objective**: Poll `server_info` and `server_state` and export all fields as OTel metrics. + +**What to do**: + +- Implement `serverInfoCollector` that calls `server_info` (admin) and extracts: + + **Node Health Gauges:** + - `xrpl_server_state` (enum → int: disconnected=0, connected=1, syncing=2, tracking=3, full=4, proposing=5) + - `xrpl_server_state_duration_seconds` + - `xrpl_uptime_seconds` + - `xrpl_io_latency_ms` + - `xrpl_amendment_blocked` (0 or 1) + - `xrpl_peers_count` + - `xrpl_peer_disconnects_total` + - `xrpl_peer_disconnects_resources_total` + - `xrpl_jq_trans_overflow_total` + + **Consensus Gauges:** + - `xrpl_last_close_proposers` + - `xrpl_last_close_converge_time_seconds` + - `xrpl_validation_quorum` + + **Ledger Gauges:** + - `xrpl_validated_ledger_seq` + - `xrpl_validated_ledger_age_seconds` + - `xrpl_validated_ledger_base_fee_drops` + - `xrpl_validated_ledger_reserve_base_drops` + - `xrpl_validated_ledger_reserve_inc_drops` + - `xrpl_close_time_offset_seconds` (0 when absent) + + **Load Factor Gauges:** + - `xrpl_load_factor` + - `xrpl_load_factor_server` + - `xrpl_load_factor_fee_escalation` + - `xrpl_load_factor_fee_queue` + - `xrpl_load_factor_local` + - `xrpl_load_factor_net` + - `xrpl_load_factor_cluster` + + **State Accounting Gauges** (per state: disconnected, connected, syncing, tracking, full): + - `xrpl_state_duration_seconds{state=""}` + - `xrpl_state_transitions_total{state=""}` + + **Validator Info** (when node is a validator): + - `xrpl_validator_list_count` + - `xrpl_validator_list_expiration_seconds` (epoch) + - `xrpl_validator_list_active` (0 or 1) + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/server_info.go` + +--- + +## Task 11.3: get_counts Collector + +**Objective**: Poll `get_counts` and export internal object counts and NodeStore stats. + +**What to do**: + +- Implement `getCountsCollector`: + + **Database Gauges:** + - `xrpl_db_size_kb{db="total"}`, `xrpl_db_size_kb{db="ledger"}`, `xrpl_db_size_kb{db="transaction"}` + + **NodeStore Gauges:** + - `xrpl_nodestore_reads_total`, `xrpl_nodestore_reads_hit`, `xrpl_nodestore_writes_total` + - `xrpl_nodestore_read_bytes`, `xrpl_nodestore_written_bytes` + - `xrpl_nodestore_read_duration_us`, `xrpl_nodestore_write_load` + - `xrpl_nodestore_read_queue`, `xrpl_nodestore_read_threads_running` + + **Cache Gauges:** + - `xrpl_cache_hit_rate{cache="SLE"}`, `xrpl_cache_hit_rate{cache="ledger"}`, `xrpl_cache_hit_rate{cache="accepted_ledger"}` + - `xrpl_cache_size{cache="treenode"}`, `xrpl_cache_size{cache="fullbelow"}`, `xrpl_cache_size{cache="accepted_ledger"}` + + **Object Count Gauges:** + - `xrpl_object_count{type=""}` for each counted object type (Transaction, Ledger, NodeObject, STTx, STLedgerEntry, InboundLedger, Pathfinder, etc.) + + **Rates:** + - `xrpl_historical_fetch_per_minute` + - `xrpl_local_txs` + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/get_counts.go` + +--- + +## Task 11.4: Peer Topology Collector + +**Objective**: Poll `peers` and export per-peer and aggregate network metrics. + +**What to do**: + +- Implement `peersCollector`: + + **Aggregate Gauges:** + - `xrpl_peers_inbound_count` + - `xrpl_peers_outbound_count` + - `xrpl_peers_cluster_count` + + **Per-Peer Gauges** (with labels `peer_key` truncated to 8 chars for cardinality control): + - `xrpl_peer_latency_ms{peer="", version="", inbound=""}` + - `xrpl_peer_uptime_seconds{peer=""}` + - `xrpl_peer_load{peer=""}` + + **Distribution Gauges** (aggregated across all peers): + - `xrpl_peer_latency_p50_ms`, `xrpl_peer_latency_p95_ms`, `xrpl_peer_latency_p99_ms` + - `xrpl_peer_version_count{version=""}` — count of peers per software version + + **Tracking Status:** + - `xrpl_peer_diverged_count` — peers with `track=diverged` + - `xrpl_peer_unknown_count` — peers with `track=unknown` + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/peers.go` + +**Cardinality note**: Per-peer metrics use truncated keys. For large peer sets (50+), the aggregate distribution gauges are preferred over per-peer labels. + +--- + +## Task 11.5: Validator & Amendment Collector + +**Objective**: Poll `validators` and `feature` to export validator health and amendment voting status. + +**What to do**: + +- Implement `validatorCollector`: + + **From `validators` RPC:** + - `xrpl_trusted_validators_count` + - `xrpl_validator_signing` (0 or 1 — whether local validator is signing) + + **From `feature` RPC:** + - `xrpl_amendment_enabled_count` — total enabled amendments + - `xrpl_amendment_majority_count` — amendments with majority but not yet enabled + - `xrpl_amendment_vetoed_count` — locally vetoed amendments + - `xrpl_amendment_unsupported_majority` (0 or 1) — any unsupported amendment has majority (critical alert) + + **Per-amendment with majority** (limited cardinality — only amendments with `majority` set): + - `xrpl_amendment_majority_time{name=""}` — epoch time when majority was gained + - `xrpl_amendment_votes{name=""}` — current vote count + - `xrpl_amendment_threshold{name=""}` — votes needed + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/validators.go` + +--- + +## Task 11.6: Fee & TxQ Collector + +**Objective**: Poll `fee` RPC and export real-time fee market data. + +**What to do**: + +- Implement `feeCollector` that calls the public `fee` RPC: + + **Fee Level Gauges:** + - `xrpl_fee_current_ledger_size` — transactions in current open ledger + - `xrpl_fee_expected_ledger_size` — expected transactions at close + - `xrpl_fee_max_queue_size` — maximum transaction queue size + - `xrpl_fee_open_ledger_fee_drops` — minimum fee for open ledger inclusion + - `xrpl_fee_median_fee_drops` — median fee level + - `xrpl_fee_minimum_fee_drops` — base reference fee + - `xrpl_fee_queue_size` — current queue depth + +- This overlaps with Phase 9's internal TxQ metrics but provides an external-only collection path that doesn't require xrpld code changes. + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/fee.go` + +--- + +## Task 11.7: DEX & AMM Collector (Optional) + +**Objective**: Periodically poll configured AMM pools and order book pairs for DeFi metrics. + +**What to do**: + +- Implement `dexCollector` (enabled only when `amm_pools` or `book_offers_pairs` are configured): + + **AMM Pool Gauges** (per configured pool): + - `xrpl_amm_reserve{pool="", asset=""}` — pool reserve amount + - `xrpl_amm_lp_token_supply{pool=""}` — outstanding LP tokens + - `xrpl_amm_trading_fee{pool=""}` — pool trading fee (basis points) + - `xrpl_amm_tvl_drops{pool=""}` — total value locked (XRP-denominated) + + **Order Book Gauges** (per configured pair): + - `xrpl_orderbook_bid_depth{pair="/"}` — total bid volume + - `xrpl_orderbook_ask_depth{pair="/"}` — total ask volume + - `xrpl_orderbook_spread{pair="/"}` — best bid-ask spread + - `xrpl_orderbook_offer_count{pair="/", side="bid|ask"}` — number of offers + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/dex.go` + +**Note**: This is optional because it requires explicit configuration of which pools/pairs to track. Default configuration tracks no DEX data. + +--- + +## Task 11.8: Prometheus Alerting Rules + +**Objective**: Create production-ready alerting rules for the metrics exported by this receiver. + +**What to do**: + +- Create `docker/telemetry/prometheus/rippled-alerts.yml`: + + **Tier 1 — Critical (page immediately):** + + ```yaml + - alert: XRPLServerNotFull + expr: xrpl_server_state < 4 + for: 15m + + - alert: XRPLAmendmentBlocked + expr: xrpl_amendment_blocked == 1 + for: 1m + + - alert: XRPLNoPeers + expr: xrpl_peers_count == 0 + for: 5m + + - alert: XRPLLedgerStale + expr: xrpl_validated_ledger_age_seconds > 120 + for: 2m + + - alert: XRPLHighIOLatency + expr: xrpl_io_latency_ms > 100 + for: 5m + + - alert: XRPLUnsupportedAmendmentMajority + expr: xrpl_amendment_unsupported_majority == 1 + for: 1m + ``` + + **Tier 2 — Warning (investigate within hours):** + + ```yaml + - alert: XRPLLowPeerCount + expr: xrpl_peers_count < 10 + for: 15m + + - alert: XRPLHighLoadFactor + expr: xrpl_load_factor > 10 + for: 10m + + - alert: XRPLSlowConsensus + expr: xrpl_last_close_converge_time_seconds > 6 + for: 5m + + - alert: XRPLValidatorListExpiring + expr: (xrpl_validator_list_expiration_seconds - time()) < 86400 + for: 1h + + - alert: XRPLClockDrift + expr: xrpl_close_time_offset_seconds > 0 + for: 5m + + - alert: XRPLStateFlapping + expr: rate(xrpl_state_transitions_total{state="full"}[1h]) > 2 + for: 30m + ``` + +**Key files**: + +- New: `docker/telemetry/prometheus/rippled-alerts.yml` +- Update: `docker/telemetry/prometheus/prometheus.yml` (add rule_files reference) + +--- + +## Task 11.9: New Grafana Dashboards + +**Objective**: Create 4 new dashboards for the data exported by the receiver. + +**What to do**: + +- **Validator Health** (`validator-health`): + - Server state timeline, state duration breakdown + - Proposer count trend, converge time trend, validation quorum + - Validator list expiration countdown + - Amendment voting status (majority/enabled/vetoed) + +- **Network Topology** (`xrpld-network-topology`): + - Peer count (inbound/outbound/cluster), peer version distribution + - Peer latency distribution (p50/p95/p99), diverged peer count + - Geographic distribution (if enriched with GeoIP) + - Peer uptime distribution + +- **Fee Market** (`xrpld-fee-market-external`): + - Current fee levels (open ledger, median, minimum), fee escalation timeline + - Queue depth vs. capacity, transactions per ledger + - Load factor breakdown (server/network/cluster/escalation) + +- **DEX & AMM Overview** (`xrpld-dex-amm`) (only populated when DEX collectors are configured): + - AMM pool TVL, reserve ratios, LP token supply + - Order book depth per pair, spread trends + - Trading fee revenue estimates + +**Key files**: + +- New: `docker/telemetry/grafana/dashboards/rippled-validator-health.json` +- New: `docker/telemetry/grafana/dashboards/rippled-network-topology.json` +- New: `docker/telemetry/grafana/dashboards/rippled-fee-market-external.json` +- New: `docker/telemetry/grafana/dashboards/rippled-dex-amm.json` + +--- + +## Task 11.10: Integration with Phase 10 Validation + +**Objective**: Extend the Phase 10 validation suite to verify this receiver's metrics. + +**What to do**: + +- Update `docker/telemetry/workload/validate_telemetry.py`: + - Add assertions for all `xrpl_*` metrics produced by the receiver + - Verify metric labels have expected values + - Verify alerting rules fire correctly (inject a "bad" state and check alert) + +- Update `docker/telemetry/docker-compose.workload.yaml`: + - Add the custom OTel Collector build with the xrpld receiver + - Configure the receiver to poll one of the test nodes + +**Key files**: + +- Update: `docker/telemetry/workload/validate_telemetry.py` +- Update: `docker/telemetry/docker-compose.workload.yaml` +- Update: `docker/telemetry/workload/expected_metrics.json` + +--- + +## Task 11.11: Documentation + +**Objective**: Document the receiver, its metrics, deployment, and alerting. + +**What to do**: + +- Create `docker/telemetry/otel-rippled-receiver/README.md`: + - Architecture overview (how the receiver fits into the OTel Collector) + - Configuration reference (all config options with defaults) + - Metric reference table (all exported metrics with types and labels) + - Deployment guide (building custom collector binary, docker-compose integration) + +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add "Third-Party Metrics (OTel Collector Receiver)" section + - Add new Grafana dashboard reference (4 dashboards) + - Add alerting rules reference + +- Update `docs/telemetry-runbook.md`: + - Add "Third-Party Metrics Receiver" troubleshooting section + - Add alerting playbook (what to do for each Tier 1/Tier 2 alert) + +--- + +## Task 11.12: Alert Rules for External Dashboard Parity Metrics + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — 18 alert rules ported from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics), Phase 9 Tasks 9.11-9.13 (dashboards). +> **Downstream**: None — terminal task in the parity chain. + +**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the `xrpld_*` internal metrics. + +**Critical Group** (8 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------------- | --- | +| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | +| Not Proposing | `xrpld_state_tracking{metric="state_value"} < 6` | 10s | +| Unhealthy State | `xrpld_state_tracking{metric="state_value"} < 4` | 10s | +| Amendment Blocked | `xrpld_validator_health{metric="amendment_blocked"} == 1` | 1m | +| UNL Expiring | `xrpld_validator_health{metric="unl_expiry_days"} < 14` | 1h | +| High IO Latency | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50` | 1m | +| High Load Factor | `xrpld_load_factor_metrics{metric="load_factor"} > 1000` | 1m | +| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5` | 1m | + +**Network Group** (3 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------------- | ----------------------------------------------------------------- | --- | +| Peer Drop >10% | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | +| Peer Drop >30% | Same formula, threshold -30 | 30s | +| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | + +**Performance Group** (7 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------------ | --- | +| CPU High | Per-core CPU > 80% (requires node_exporter) | 2m | +| Memory Critical | Memory usage > 90% (requires node_exporter) | 1m | +| Disk Warning | Disk usage > 85% (requires node_exporter) | 2m | +| Job Queue Overflow | `rate(xrpld_jq_trans_overflow_total[5m]) > 0` | 1m | +| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | +| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | +| Stale Ledger | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | + +**Notification channel templates**: Email/SMTP, Discord, Slack, PagerDuty. + +**Key files**: + +- New/extend: `docker/telemetry/grafana/alerting/alert-rules-parity.yaml` +- New: `docker/telemetry/grafana/alerting/contact-points.yaml` (template configs) +- New: `docker/telemetry/grafana/alerting/notification-policies.yaml` + +**Exit Criteria**: + +- [ ] All 18 rules evaluate without errors in Grafana alerting UI +- [ ] Critical rules fire within expected timeframe when conditions are met +- [ ] Notification channel templates are documented (not hard-coded to any service) + +--- + +## Task 11.13: Dual-Datasource Architecture Documentation + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Document the external dashboard's "fast path" pattern as a future optimization for real-time panels. + +**Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval. + +**Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow for operational dashboards. + +**Decision**: Document as a future option, not implement now. The current 10s interval is acceptable for v1. The external dashboard achieves 2-5s freshness by polling RPC directly, which is what the Phase 11 receiver already does. Adding a separate scrape endpoint to xrpld would only be needed if sub-second metric freshness is required from the internal metrics pipeline. + +**What to document**: + +- Architecture comparison: OTLP pipeline (10-15s) vs. direct scrape (2-5s) vs. push gateway +- When to consider: operator feedback indicating 10s is insufficient for alerting SLOs +- How to implement if needed: add `/metrics` HTTP endpoint to xrpld with Prometheus client library +- Trade-offs: additional port, additional dependency, duplication with OTLP metrics + +**Key files**: + +- Update: `OpenTelemetryPlan/09-data-collection-reference.md` (add "Future: Dual-Datasource Architecture" section) +- Update: `docs/telemetry-runbook.md` (add brief note in performance tuning section) + +**Exit Criteria**: + +- [ ] Architecture comparison documented with clear trade-offs +- [ ] Decision rationale recorded (why deferred, when to revisit) + +--- + +## Exit Criteria + +- [ ] Custom OTel Collector receiver builds and starts without errors +- [ ] All `xrpl_*` metrics from server_info, get_counts, peers, validators, fee appear in Prometheus +- [ ] Metrics update at configured poll interval (default 30s) +- [ ] 4 new Grafana dashboards operational with data +- [ ] Prometheus alerting rules fire correctly for simulated failure conditions +- [ ] DEX/AMM collector works when configured (optional — not required for base exit criteria) +- [ ] Phase 10 validation suite passes with receiver metrics included +- [ ] Receiver handles xrpld restart/unavailability gracefully (no crash, logs warning, retries) +- [ ] Documentation complete: receiver README, metric reference, alerting playbook +- [ ] Go receiver has unit tests with >80% coverage +- [ ] 18 Grafana alert rules for Phase 7+ parity metrics evaluate correctly (Task 11.12) +- [ ] Dual-datasource architecture documented with trade-offs (Task 11.13) diff --git a/OpenTelemetryPlan/Phase2_taskList.md b/OpenTelemetryPlan/Phase2_taskList.md new file mode 100644 index 00000000000..89d9ddd6e03 --- /dev/null +++ b/OpenTelemetryPlan/Phase2_taskList.md @@ -0,0 +1,240 @@ +# Phase 2: RPC Tracing Completion Task List + +> **Goal**: Complete RPC tracing coverage with unit tests, Grafana search filters, PathFind instrumentation, and config hardening. Build on the Phase 1c SpanGuard factory foundation to achieve production-quality RPC observability. +> +> **Scope**: Unit tests for core telemetry, Grafana Tempo search filters, PathFind RPC tracing, config validation (`std::clamp`). +> +> **Branch**: `pratik/otel-phase2-rpc-tracing` (from `pratik/otel-phase1c-rpc-integration`) + +### Related Plan Documents + +| Document | Relevance | +| ------------------------------------------------------------ | ------------------------------------------------------------- | +| [04-code-samples.md](./04-code-samples.md) | TraceContextPropagator (§4.4.2), RPC instrumentation (§4.5.3) | +| [02-design-decisions.md](./02-design-decisions.md) | W3C Trace Context (§2.5), span attributes (§2.4.2) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 2 tasks (§6.3), definition of done (§6.11.2) | + +--- + +## Task 2.1: W3C Trace Context HTTP Header Extraction + +**Status**: DEFERRED → Phase 3 + +**Reason**: W3C context propagation (`traceparent`/`tracestate` headers) requires a consumer — in Phase 2, RPC spans are entirely local to the node. Phase 3 introduces cross-node transaction tracing via protobuf context propagation, which is the first use case for extracted trace context. Implementing it here without a consumer would be dead code. + +**Implemented in**: `pratik/otel-phase3-tx-tracing` — `TraceContextPropagator.h/.cpp` + +--- + +## Task 2.2: Per-Category Span Creation + +**Status**: COMPLETE (superseded by Phase 1c design) + +**Original plan**: Add `XRPL_TRACE_PEER` and `XRPL_TRACE_LEDGER` macros. + +**Actual implementation**: Phase 1c replaced all tracing macros with the `SpanGuard::span(TraceCategory, prefix, name)` factory pattern. The `TraceCategory` enum (`Rpc`, `Transactions`, `Consensus`, `Peer`, `Ledger`) serves the same conditional-creation purpose without macros. No separate task needed — the factory already supports all categories. + +--- + +## Task 2.3: Add shouldTraceLedger() to Telemetry Interface + +**Objective**: The `Setup` struct has a `traceLedger` field but there's no corresponding virtual method. Add it for interface completeness. + +**What to do**: + +- Edit `include/xrpl/telemetry/Telemetry.h`: + - Add `virtual bool shouldTraceLedger() const = 0;` + +- Update all implementations: + - `src/libxrpl/telemetry/Telemetry.cpp` (TelemetryImpl, NullTelemetryOtel) + - `src/libxrpl/telemetry/NullTelemetry.cpp` (NullTelemetry) + +**Key modified files**: + +- `include/xrpl/telemetry/Telemetry.h` +- `src/libxrpl/telemetry/Telemetry.cpp` +- `src/libxrpl/telemetry/NullTelemetry.cpp` + +--- + +## Task 2.4: Unit Tests for Core Telemetry Infrastructure + +**Status**: COMPLETE + +**Objective**: Add unit tests for the core telemetry abstractions to validate correctness and catch regressions. + +**Implemented**: + +- `src/tests/libxrpl/telemetry/TelemetryConfig.cpp`: + - Test Setup defaults (all fields have correct initial values) + - Test `setupTelemetry` config parser (empty section, full section, edge cases) + - Test `samplingRatio` clamping (values outside 0.0-1.0) + +- `src/tests/libxrpl/telemetry/SpanGuardFactory.cpp`: + - Test null guard methods are safe (setAttribute, setOk, setError, addEvent on null) + - Test category span returns null when telemetry disabled + - Test child/linked span null when no parent context + - Test move construction transfers ownership + - Test recordException safe on null guard + - Test discard() safe on null guard + +- `src/tests/libxrpl/telemetry/main.cpp` — GTest runner +- `src/tests/libxrpl/CMakeLists.txt` — test target with optional OTel linking + +--- + +## Task 2.5: Enhance RPC Span Attributes + +**Status**: DEFERRED (low priority) + +**Reason**: The high-value attributes (`command`, `version`, `role`, `status`) are already set by Phase 1c. The remaining HTTP transport-level attributes (`http.method`, `net.peer.ip`, `http.status_code`) provide limited additional insight since: + +- `http.method` is always POST for JSON-RPC +- `net.peer.ip` is debug-level info available in logs +- `duration_ms` is redundant with span duration (OTel captures start/end time natively) + +These can be added later if dashboard queries specifically need them. The node health attributes (Task 2.8) provide far more operational value and were prioritized instead. + +--- + +## Task 2.6: Build Verification and Performance Baseline + +**Objective**: Verify the build succeeds with and without telemetry, and establish a performance baseline. + +**What to do**: + +1. Build with `telemetry=ON` and verify no compilation errors +2. Build with `telemetry=OFF` and verify no regressions +3. Run existing unit tests to verify no breakage +4. Document any build issues in lessons.md + +**Verification Checklist**: + +- [ ] `conan install . --build=missing -o telemetry=True` succeeds +- [ ] `cmake --preset default -Dtelemetry=ON` configures correctly +- [ ] Build succeeds with telemetry ON +- [ ] Build succeeds with telemetry OFF +- [ ] Existing tests pass with telemetry ON +- [ ] Existing tests pass with telemetry OFF + +--- + +## Task 2.8: RPC Span Attribute Enrichment — Node Health Context + +**Status**: DROPPED. + +Node health (`amendment_blocked`, `server_state`) is not part of the telemetry surface. Operators consume the same data via the existing `server_info` / `server_state` RPC commands, so duplicating it on traces adds storage and cardinality cost without new value. The OTel C++ SDK 1.18.0 also does not support runtime updates to the resource, ruling out resource-level emission of these dynamic-by-nature flags. + +--- + +## Task 2.9: PathFind RPC Instrumentation + +**Status**: COMPLETE + +**Objective**: Trace the path_find and ripple_path_find RPC handlers to capture request latency and computation cost. + +**Spans added**: + +- `pathfind.request` — wraps `doPathFind()` and `doRipplePathFind()` RPC handlers +- `pathfind.compute` — wraps `PathRequest::doUpdate()` (`pathfind_fast` attr) +- `pathfind.update_all` — wraps `PathRequestManager::updateAll()` on ledger close (`pathfind_ledger_index`, `pathfind_num_requests` attrs; emitted only when active subscriptions exist) +- `pathfind.discover` — wraps the entire per-source-asset loop in `PathRequest::findPaths()` (`pathfind_search_level`, `pathfind_num_paths` attrs). One span per RPC call instead of N (one per source asset). Trade-off: per-asset breakdown is lost; storage and cardinality bounded. + +**Attribute namespacing**: All pathfind attributes use the `pathfind_*` underscore form per the Phase 1c naming-spec rule 5. + +**New file**: `src/xrpld/rpc/detail/PathFindSpanNames.h` + +**Modified files**: + +- `src/xrpld/rpc/handlers/orderbook/PathFind.cpp` +- `src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp` +- `src/xrpld/rpc/detail/PathRequest.cpp` +- `src/xrpld/rpc/detail/PathRequestManager.cpp` +- `src/xrpld/rpc/detail/Pathfinder.cpp` + +--- + +## Task 2.10: RPC and PathFind Span Attribute Gap Fill + +**Status**: COMPLETE + +**Objective**: Wire up workflow-identifying attributes that enable filtering and grouping traces by request characteristics without drilling into child spans. + +**Attributes added**: + +| Span | Attribute | Type | Source | +| ------------------- | ---------------------------- | ------ | --------------------------------- | +| `rpc.http_request` | `request_payload_size` | int64 | `request.body().size()` | +| `rpc.process` | `is_batch` | bool | `method == "batch"` check | +| `rpc.process` | `batch_size` | int64 | `params.size()` (only when batch) | +| `rpc.ws_message` | `command` | string | `jv[command]` or `jv[method]` | +| `rpc.command.*` | `load_type` | string | `context.loadType.label()` | +| `pathfind.compute` | `pathfind_dest_currency` | string | `to_string(saDstAmount_.asset())` | +| `pathfind.discover` | `pathfind_num_source_assets` | int64 | `sourceAssets.size()` | + +_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._ + +**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`). + +**Modified files**: + +- `src/xrpld/rpc/detail/RpcSpanNames.h` +- `src/xrpld/rpc/detail/PathFindSpanNames.h` +- `src/xrpld/rpc/detail/ServerHandler.cpp` +- `src/xrpld/rpc/detail/RPCHandler.cpp` +- `src/xrpld/rpc/detail/PathRequest.cpp` + +--- + +## Summary + +| Task | Description | Status | Notes | +| ---- | ------------------------------------------- | ------------------- | --------------------------------------------------------- | +| 2.1 | W3C Trace Context header extraction | Deferred → Phase 3 | No consumer in Phase 2; needs cross-node tracing | +| 2.2 | Per-category span creation | Complete (Phase 1c) | Superseded by TraceCategory enum + SpanGuard | +| 2.3 | Add shouldTraceLedger() interface method | Complete (Phase 1c) | Delivered in Phase 1c base branch | +| 2.4 | Unit tests for core telemetry | Complete | TelemetryConfig + SpanGuardFactory tests | +| 2.5 | Enhanced RPC span attributes (HTTP-level) | Deferred | Low value; span duration covers timing natively | +| 2.6 | Build verification and performance baseline | Complete | Verified in CI on Phase 1c | +| 2.7 | Grafana Tempo search filters | Complete | rpc-command, rpc-status, rpc-role filters | +| 2.8 | RPC span attribute enrichment (node health) | Dropped | Available via `server_info`/`server_state` RPC | +| 2.9 | PathFind RPC instrumentation | Complete | request, compute, update_all, discover | +| 2.10 | RPC/PathFind span attribute gap fill | Complete | Batch detection, payload size, load cost, pathfind params | + +**Delivered in this branch**: Tasks 2.4, 2.7, 2.9, 2.10. +**Deferred with rationale**: Tasks 2.1 (→Phase 3), 2.5 (low priority). +**Dropped**: Task 2.8 (node health not duplicated on traces). +**Superseded**: Task 2.2 (Phase 1c SpanGuard factory covers this). + +--- + +## Known Issues / Future Work + +### Thread safety of TelemetryImpl::stop() vs startSpan() + +`TelemetryImpl::stop()` resets `sdkProvider_` (a `std::shared_ptr`) without +synchronization. `getTracer()` reads the same member from RPC handler threads. +This is a data race if any thread calls `startSpan()` concurrently with `stop()`. + +**Current mitigation**: `Application::stop()` shuts down `serverHandler_`, +`overlay_`, and `jobQueue_` before calling `telemetry_->stop()`, so no callers +remain. See comments in `Telemetry.cpp:stop()` and `Application.cpp`. + +**TODO**: Add an `std::atomic stopped_` flag checked in `getTracer()` to +make this robust against future shutdown order changes. + +### Macro incompatibility: XRPL_TRACE_SPAN vs XRPL_TRACE_SET_ATTR + +`XRPL_TRACE_SPAN` and `XRPL_TRACE_SPAN_KIND` declare `_xrpl_guard_` as a bare +`SpanGuard`, but `XRPL_TRACE_SET_ATTR` and `XRPL_TRACE_EXCEPTION` call +`_xrpl_guard_.has_value()` which requires `std::optional`. Using +`XRPL_TRACE_SPAN` followed by `XRPL_TRACE_SET_ATTR` in the same scope would +fail to compile. + +**Current mitigation**: No call site currently uses `XRPL_TRACE_SPAN` — all +production code uses the conditional macros (`XRPL_TRACE_RPC`, `XRPL_TRACE_TX`, +etc.) which correctly wrap the guard in `std::optional`. + +**TODO**: Either make `XRPL_TRACE_SPAN`/`XRPL_TRACE_SPAN_KIND` also wrap in +`std::optional`, or document that `XRPL_TRACE_SET_ATTR` is only compatible with +the conditional macros. diff --git a/OpenTelemetryPlan/Phase3_taskList.md b/OpenTelemetryPlan/Phase3_taskList.md new file mode 100644 index 00000000000..c5d3c95251b --- /dev/null +++ b/OpenTelemetryPlan/Phase3_taskList.md @@ -0,0 +1,555 @@ +# Phase 3: Transaction Tracing Task List + +> **Goal**: Trace the full transaction lifecycle from RPC submission through peer relay, including cross-node context propagation via Protocol Buffer extensions. This is the WALK phase that demonstrates true distributed tracing. +> +> **Scope**: Protocol Buffer `TraceContext` message, context serialization, PeerImp transaction instrumentation, NetworkOPs processing instrumentation, HashRouter visibility, and multi-node relay context propagation. +> +> **Branch**: `pratik/otel-phase3-tx-tracing` (from `pratik/otel-phase2-rpc-tracing`) + +### Related Plan Documents + +| Document | Relevance | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| [04-code-samples.md](./04-code-samples.md) | TraceContext protobuf (§4.4.1), PeerImp instrumentation (§4.5.1), context serialization (§4.4.2) | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | Transaction flow (§1.3), key trace points (§1.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 3 tasks (§6.4), definition of done (§6.11.3) | +| [02-design-decisions.md](./02-design-decisions.md) | Context propagation design (§2.5), attribute schema (§2.4.3) | + +--- + +## Task 3.1: Define TraceContext Protocol Buffer Message + +**Objective**: Add trace context fields to the P2P protocol messages so trace IDs can propagate across nodes. + +**What to do**: + +- Edit `include/xrpl/proto/xrpl.proto` (or `src/xrpld/proto/ripple.proto`, wherever the proto is): + - Add `TraceContext` message definition: + ```protobuf + message TraceContext { + bytes trace_id = 1; // 16-byte trace identifier + bytes span_id = 2; // 8-byte span identifier + uint32 trace_flags = 3; // bit 0 = sampled + string trace_state = 4; // W3C tracestate value + } + ``` + - Add `optional TraceContext trace_context = 1001;` to: + - `TMTransaction` + - `TMProposeSet` (for Phase 4 use) + - `TMValidation` (for Phase 4 use) + - Use high field numbers (1001+) to avoid conflicts with existing fields + +- Regenerate protobuf C++ code + +**Key modified files**: + +- `include/xrpl/proto/xrpl.proto` (or equivalent) + +**Reference**: + +- [04-code-samples.md §4.4.1](./04-code-samples.md) — TraceContext message definition +- [02-design-decisions.md §2.5.2](./02-design-decisions.md) — Protocol buffer context propagation design + +--- + +## Task 3.2: Implement Protobuf Context Serialization + +**Objective**: Create utilities to serialize/deserialize OTel trace context to/from protobuf `TraceContext` messages. + +**What to do**: + +- Create `include/xrpl/telemetry/TraceContextPropagator.h` (extend from Phase 2 if exists, or add protobuf methods): + - Add protobuf-specific methods: + - `static Context extractFromProtobuf(protocol::TraceContext const& proto)` — reconstruct OTel context from protobuf fields + - `static void injectToProtobuf(Context const& ctx, protocol::TraceContext& proto)` — serialize current span context into protobuf fields + - Both methods guard behind `#ifdef XRPL_ENABLE_TELEMETRY` + +- Create/extend `src/libxrpl/telemetry/TraceContextPropagator.cpp`: + - Implement extraction: read trace_id (16 bytes), span_id (8 bytes), trace_flags from protobuf, construct `SpanContext`, wrap in `Context` + - Implement injection: get current span from context, serialize its TraceId, SpanId, and TraceFlags into protobuf fields + +**Key new/modified files**: + +- `include/xrpl/telemetry/TraceContextPropagator.h` +- `src/libxrpl/telemetry/TraceContextPropagator.cpp` + +**Reference**: + +- [04-code-samples.md §4.4.2](./04-code-samples.md) — Full extract/inject implementation + +--- + +## Task 3.3: Instrument PeerImp Transaction Handling + +**Objective**: Add trace spans to the peer-level transaction receive and relay path. + +**What to do**: + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp`: + - In `onMessage(TMTransaction)` / `handleTransaction()`: + - Extract parent trace context from incoming `TMTransaction::trace_context` field (if present) + - Create `tx.receive` span as child of extracted context (or new root if none) + - Set attributes: `tx_hash`, `peer_id`, `tx_status` + - On HashRouter suppression (duplicate): set `suppressed=true`, add `tx.duplicate` event + - Wrap validation call with child span `tx.validate` + - Wrap relay with `tx.relay` span + - When relaying to peers: + - Inject current trace context into outgoing `TMTransaction::trace_context` + - Set `relay_count` attribute + +- Use `SpanGuard::span(TraceCategory::Transactions, "tx", "receive")` factory + (Phase 1c replaced macros with the SpanGuard factory pattern) + +> **Note**: The `tx.receive` guard is `.detached()` before being moved into the +> `RcvCheckTx` job so its Scope is popped on the peer thread, not leaked to the +> worker (else later peer messages would inherit this transaction's trace). + +**Key modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` + +**Reference**: + +- [04-code-samples.md §4.5.1](./04-code-samples.md) — Full PeerImp instrumentation example +- [01-architecture-analysis.md §1.3](./01-architecture-analysis.md) — Transaction flow diagram +- [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.receive trace point + +--- + +## Task 3.4: Instrument NetworkOPs Transaction Processing + +**Objective**: Trace the transaction processing pipeline in NetworkOPs, covering both sync and async paths. + +**What to do**: + +- Edit `src/xrpld/app/misc/NetworkOPs.cpp`: + - In `processTransaction()`: + - Create `tx.process` span + - Set attributes: `tx_hash`, `tx_type`, `local` (whether from RPC or peer) + - Record whether sync or async path is taken + - `.detached()` the guard before storing it in `TransactionStatus::span`, + since it is applied on a batch worker thread — this pops the Scope on the + origin thread and stops later work inheriting this transaction's trace + + - In `doTransactionAsync()`: + - Capture parent context before queuing + - Create `tx.queue` span with queue depth attribute + - Add event when transaction is dequeued for processing + + - In `doTransactionSync()`: + - Create `tx.process_sync` span + - Record result (applied, queued, rejected) + +**Key modified files**: + +- `src/xrpld/app/misc/NetworkOPs.cpp` + +**Reference**: + +- [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.validate and tx.process trace points +- [02-design-decisions.md §2.4.3](./02-design-decisions.md) — Transaction attribute schema + +--- + +## Task 3.5: Instrument HashRouter for Dedup Visibility + +**Objective**: Make transaction deduplication visible in traces by recording HashRouter decisions as span attributes/events. + +**What to do**: + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp` (in handleTransaction): + - After calling `HashRouter::shouldProcess()` or `addSuppressionPeer()`: + - Record `suppressed` attribute (true/false) + - Record `tx_flags` showing current HashRouter state (SAVED, TRUSTED, etc.) + - Add `tx.first_seen` or `tx.duplicate` event + +- This is NOT a modification to HashRouter itself — just recording its decisions as span attributes in the existing PeerImp instrumentation from Task 3.3. + +**Key modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` (same changes as 3.3, logically grouped) + +--- + +## Task 3.6: Context Propagation in Transaction Relay + +**Status**: COMPLETE (transaction relay). Consensus proposal/validation +propagation is deferred to Phase 4 — see "Planned (Phase 4)" below. + +**Objective**: Ensure trace context flows correctly when transactions are relayed between peers, creating linked spans across nodes. + +**What was done**: + +- **TX send side**: `NetworkOPs::apply()` now injects the tx.process span's trace + context into the outgoing `TMTransaction` protobuf before relay, using + `telemetry::injectSpanContext()`. The receiving node's `txReceiveSpan()` (already + wired in PeerImp) extracts the parent span_id and creates the tx.receive span + as a child of the sender's tx.process span. + +- **Edge cases**: Missing trace context (older peers) degrades gracefully to + standalone spans. Invalid/corrupted context is treated as absent. Trace + flags are propagated and respected. + +**New infrastructure**: + +- `SpanGuard::getTraceBytes()` — extracts raw trace_id/span_id/trace_flags + from a span without exposing OTel types. Safe to call from any thread. +- `PropagationHelpers.h` — `injectSpanContext(SpanGuard&, proto)` bridge + between SpanGuard and protobuf TraceContext. +- `TraceContextPropagator.h` — `injectToProtobuf(ctx, proto)` for + same-thread injection via OTel RuntimeContext. + +**Key modified files**: + +- `src/xrpld/app/misc/NetworkOPs.cpp` — tx relay injection +- `include/xrpl/telemetry/SpanGuard.h` — `TraceBytes` struct, `getTraceBytes()` +- `src/libxrpl/telemetry/SpanGuard.cpp` — `getTraceBytes()` implementation +- `src/xrpld/telemetry/PropagationHelpers.h` — inject helpers (new file) + +**Planned (Phase 4 — not in this PR)**: + +The consensus proposal/validation propagation below is Phase 4 scope and is +not implemented on this branch. It is listed here only to record the intended +design. + +- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the + current thread's active span context into the `TMProposeSet` protobuf via + `telemetry::injectToProtobuf()`. PeerImp creates a + `consensus.proposal.receive` span that extracts the sender's trace context + as parent (via `ConsensusReceiveTracing.h`). + +- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects + the current thread's active span context into the `TMValidation` protobuf. + PeerImp creates a `consensus.validation.receive` span that extracts the + sender's trace context as parent. + +- Planned files: `src/xrpld/app/consensus/RCLConsensus.cpp` (send injection), + `src/xrpld/overlay/detail/PeerImp.cpp` (receive spans), + `src/xrpld/telemetry/ConsensusReceiveTracing.h` (receive span helpers, + new file). + +**Reference**: + +- [02-design-decisions.md §2.5](./02-design-decisions.md) — Context propagation design +- [04-code-samples.md §4.5.1](./04-code-samples.md) — Relay context injection pattern + +--- + +## Task 3.7: Build Verification and Testing + +**Objective**: Verify all Phase 3 changes compile and work correctly. + +**What to do**: + +1. Build with `telemetry=ON` — verify no compilation errors +2. Build with `telemetry=OFF` — verify no regressions +3. Run existing unit tests +4. Verify protobuf regeneration produces correct C++ code +5. Document any issues encountered + +**Verification Checklist**: + +- [ ] Protobuf changes generate valid C++ +- [ ] Build succeeds with telemetry ON +- [ ] Build succeeds with telemetry OFF +- [ ] Existing tests pass +- [ ] No undefined symbols from new telemetry calls + +--- + +## Task 3.8: Transaction Span Peer Version Attribute + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds peer version context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 2 (RPC span infrastructure must exist). +> **Downstream**: Phase 10 (validation checks for this attribute). + +**Objective**: Add the relaying peer's xrpld version to `tx.receive` spans so operators can correlate transaction issues with peer version mismatches during network upgrades. + +**What to do**: + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp`: + - In the `tx.receive` span block (after existing `peer_id` setAttribute call): + - Add `peer_version` (string) — from `this->getVersion()` + - Only set if `getVersion()` returns a non-empty string (avoid empty-string attributes) + +**New span attribute**: + +| Attribute | Type | Source | Example | +| -------------- | ------ | -------------------- | --------------- | +| `peer_version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` | + +**Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues. The community dashboard tracks peer versions externally; this brings version awareness into the trace itself. + +**Key modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` + +**Exit Criteria**: + +- [ ] `tx.receive` spans carry `peer_version` attribute with a non-empty version string +- [ ] Attribute is omitted (not set to empty string) when `getVersion()` returns empty +- [ ] Attribute visible in Tempo trace detail view + +--- + +## Task 3.9: Deterministic Transaction Trace ID + +> **Upstream**: Task 3.2 (protobuf serialization), Task 3.3 (PeerImp span exists). +> **Downstream**: Phase 10 (workload validation can query by tx hash directly). +> **Pattern**: Mirrors the consensus deterministic trace ID in Phase 4a +> (`createDeterministicContext` in `RCLConsensus.cpp`), adapted for transactions. + +**Objective**: Derive the trace_id for transaction spans deterministically from the +transaction hash so that all nodes handling the same transaction independently produce +spans under the same trace_id — regardless of whether protobuf context propagation +succeeds. + +**Why**: The current approach creates spans with random trace_ids and relies entirely +on protobuf `TraceContext` propagation to link them. If any hop in the relay chain +drops the context (older peers, message corruption, mixed-version networks), the trace +splits and downstream spans become impossible to find. With deterministic trace_ids, +correlation is guaranteed because every node derives the same trace_id from the same +`txID`. + +**Approach — deterministic trace_id + protobuf span_id propagation**: + +1. Derive `trace_id = txHash[0:16]` (first 16 bytes of the 32-byte transaction hash). +2. Generate a random 8-byte `span_id` per node (each node's span is unique within + the shared trace). +3. Create the span under this deterministic context as parent. +4. **Additionally**, if protobuf `TraceContext` is present in the incoming + `TMTransaction` message, extract the sender's `span_id` and use it as the span's + parent — this preserves parent-child ordering in the trace tree. +5. If protobuf context is absent (older peer, first hop), the span still has the + correct deterministic `trace_id` — it appears as a sibling root in the same trace + rather than being lost. + +This gives the best of both worlds: guaranteed cross-node correlation via deterministic +`trace_id`, plus parent-child relay ordering via protobuf `span_id` when available. + +**What to do**: + +- Create `createDeterministicTxContext(uint256 const& txHash)` utility function: + - Location: shared header or file-local in `PeerImp.cpp` and `NetworkOPs.cpp` + (or a shared telemetry utility if both need it). + - Pattern: identical to `createDeterministicContext(uint256 const& ledgerId)` in + `RCLConsensus.cpp` — take `txHash[0:16]` as trace_id, random span_id via + `default_prng()`, sampled flag set, `remote=false`. + - Guard behind `#ifdef XRPL_ENABLE_TELEMETRY`. + + ```cpp + opentelemetry::context::Context + createDeterministicTxContext(uint256 const& txHash) + { + namespace trace = opentelemetry::trace; + + // First 16 bytes of the 32-byte tx hash as trace ID. + trace::TraceId traceId( + opentelemetry::nostd::span(txHash.data(), 16)); + + // Random span_id so each node's span is unique within the trace. + uint8_t spanIdBytes[8]; + auto const rval = default_prng()(); + std::memcpy(spanIdBytes, &rval, sizeof(spanIdBytes)); + trace::SpanId spanId( + opentelemetry::nostd::span(spanIdBytes, 8)); + + trace::SpanContext syntheticCtx( + traceId, spanId, trace::TraceFlags(1), /* remote = */ false); + + return opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr( + new trace::DefaultSpan(syntheticCtx))); + } + ``` + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp` — restructure `handleTransaction()`: + - **Move span creation after deserialization** (txID must be known first): + 1. Deserialize `STTx` and get `txID` (existing code at line ~1382). + 2. Create deterministic parent context: `auto detCtx = createDeterministicTxContext(txID)`. + 3. If `m->has_trace_context()`: extract protobuf context via `extractFromProtobuf()`, + **combine** with deterministic trace_id — use the protobuf span_id as parent + to preserve relay ordering, but override trace_id with the deterministic one. + 4. If no protobuf context: create span under `detCtx` directly. + 5. Set all existing attributes (`hash`, `peerId`, `peerVersion`, `suppressed`, etc.). + + - **Combining deterministic trace_id with protobuf parent span_id**: + When both are available, construct a synthetic `SpanContext` with: + - `trace_id` = `txHash[0:16]` (deterministic) + - `span_id` = extracted from protobuf (sender's span_id → becomes parent) + - `trace_flags` = from protobuf + - `remote` = true (came from another node) + + ```cpp + // Pseudo-code for the combined context: + auto detTraceId = trace::TraceId(txHash.data(), 16); + auto remoteSpanId = /* from extractFromProtobuf */; + auto remoteFlags = /* from extractFromProtobuf */; + + trace::SpanContext combinedCtx( + detTraceId, remoteSpanId, remoteFlags, /* remote = */ true); + // Use as parent context for the new span. + ``` + +- Edit `src/xrpld/app/misc/NetworkOPs.cpp` — update `processTransaction()`: + - `transaction->getID()` is already available at the top of the function. + - Create deterministic parent context from `txID`. + - Create `tx.process` span under this context. + - No protobuf context to extract here (NetworkOPs is intra-node), so + deterministic context alone is sufficient. + +- Add `trace_strategy` attribute to spans: + - Add `inline constexpr auto traceStrategy = "trace_strategy";` + to `TxSpanNames.h`. + - Set on each tx span: `span.setAttribute(tx_span::attr::traceStrategy, "deterministic")`. + +**Key new/modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` — restructured span creation +- `src/xrpld/app/misc/NetworkOPs.cpp` — deterministic context for tx.process +- `src/xrpld/telemetry/TxSpanNames.h` — new `traceStrategy` attribute constant +- New or shared utility for `createDeterministicTxContext()` (location TBD: could be + a shared header like `include/xrpl/telemetry/DeterministicContext.h`, or file-local + if only used in two places) + +**Interaction with existing tasks**: + +- **Task 3.3 (PeerImp instrumentation)**: The span creation in `handleTransaction()` + must be restructured — the span currently starts before `txID` is known. This task + moves it after deserialization. +- **Task 3.6 (Relay context propagation)**: Protobuf injection at the relay site + remains the same — `injectToProtobuf()` serializes the current span's `span_id`. + The receiver extracts it and combines with the deterministic `trace_id`. +- **Phase 4a (Consensus deterministic trace ID)**: This task follows the same pattern. + Consider extracting a shared utility (e.g., `createDeterministicContext(uint256)`) + that both consensus and transaction tracing use. + +**Exit Criteria**: + +- [ ] `tx.receive` and `tx.process` spans have deterministic trace_id = `txHash[0:16]` +- [ ] All nodes handling the same transaction produce spans under the same trace_id +- [x] Protobuf `span_id` propagation still works when available (parent-child ordering) +- [ ] Missing protobuf context (old peer) degrades gracefully to sibling spans, not lost traces +- [ ] `trace_strategy` attribute set to `"deterministic"` on all tx spans +- [ ] Trace queryable by tx hash (truncate hash → trace_id → direct lookup in Tempo) + +**Deliverables implemented (not in original plan)**: + +- **`SpanGuard::txSpan()` factory method** (`include/xrpl/telemetry/SpanGuard.h`): + Two overloads for creating transaction spans with deterministic trace IDs: + - `txSpan(category, group, name, txHash)` — standalone span (deterministic + trace_id from `txHash[0:16]`, no parent span_id). + - `txSpan(category, group, name, txHash, parentCtx)` — child span (deterministic + trace_id combined with protobuf-extracted parent span_id for relay ordering). + +- **`TxTracing.h` helper functions** (`src/xrpld/telemetry/TxTracing.h`): + File-local helpers that wrap `SpanGuard::txSpan()` for the two main PeerImp call + sites: + - `txReceiveSpan(txHash, parentCtx)` — creates `tx.receive` span with + deterministic trace_id and optional protobuf parent context. + - `txProcessSpan(txHash)` — creates `tx.process` span with deterministic + trace_id only (no protobuf parent, used intra-node). + - **Note**: `TxTracing.h` includes `xrpl.pb.h` unconditionally (outside + `#ifdef XRPL_ENABLE_TELEMETRY`) because `protocol::TMTransaction` appears in + the function signatures regardless of telemetry build mode. + +--- + +## Task 3.10: TxQ Instrumentation + +**Status**: COMPLETE + +**Objective**: Trace the transaction queue lifecycle — enqueue decisions, direct apply, batch clear, ledger-close accept loop, per-tx apply, and cleanup. + +**Spans added**: + +- `txq.enqueue` — wraps `TxQ::apply()` with tx_hash attribute +- `txq.apply_direct` — wraps `TxQ::tryDirectApply()` fast-path +- `txq.batch_clear` — wraps `TxQ::tryClearAccountQueueUpThruTx()` +- `txq.accept` — wraps `TxQ::accept()` ledger-close dequeue with queue_size attr +- `txq.accept_tx` — per-tx span inside accept loop with tx_hash, ter_code, + retries_remaining attributes +- `txq.cleanup` — wraps `TxQ::processClosedLedger()` with ledger_seq attribute + +**New file**: `src/xrpld/app/misc/detail/TxQSpanNames.h` + +**Modified file**: `src/xrpld/app/misc/detail/TxQ.cpp` + +--- + +## Task 3.11: TX and TxQ Span Attribute Gap Fill + +**Status**: COMPLETE + +**Objective**: Add workflow-identifying attributes to transaction spans so operators can filter by transaction type and see outcomes without off-chain correlation. + +**Attributes added**: + +| Span | Attribute | Type | Source | +| ----------------- | -------------------- | ------ | ------------------------------------------------------------------- | +| `tx.process` | `tx_type` | string | `TxFormats::getInstance().findByType(stx->getTxnType())->getName()` | +| `tx.process` | `fee` | int64 | `stx->getFieldAmount(sfFee).xrp().drops()` | +| `tx.process` | `sequence` | int64 | `stx->getSeqProxy().value()` | +| `tx.process` | `ter_result` | string | `transToken(e.result)` (set after batch application) | +| `tx.process` | `applied` | bool | `e.applied` (set after batch application) | +| `tx.receive` | `tx_type` | string | `TxFormats::getInstance().findByType(stx->getTxnType())->getName()` | +| `txq.enqueue` | `tx_type` | string | same pattern as above | +| `txq.enqueue` | `txq_status` | string | `queued` / `applied_direct` / `applied` / `rejected` | +| `txq.enqueue` | `fee_level_paid` | int64 | `getFeeLevelPaid(view, *tx).value()` | +| `txq.enqueue` | `required_fee_level` | int64 | `getRequiredFeeLevel(...).value()` | +| `txq.batch_clear` | `num_cleared` | int64 | queued txs cleared ahead of the applying tx | +| `txq.cleanup` | `expired_count` | int64 | entries dropped for passed `LastLedgerSequence` | +| `txq.accept_tx` | `txq_status` | string | `applied` / `failed` / `retried` | +| `txq.accept` | `ledger_changed` | bool | set at end of accept loop | + +**New attr keys**: `TxSpanNames.h` (`txType`, `fee`, `sequence`, `terResult`, `applied`), `TxQSpanNames.h` (`txType`). + +**Modified files**: + +- `src/xrpld/telemetry/TxSpanNames.h` +- `src/xrpld/app/misc/detail/TxQSpanNames.h` +- `src/xrpld/app/misc/NetworkOPs.cpp` +- `src/xrpld/overlay/detail/PeerImp.cpp` +- `src/xrpld/app/misc/detail/TxQ.cpp` + +--- + +## Summary + +| Task | Description | New Files | Modified Files | Depends On | +| ---- | ----------------------------------- | --------- | -------------- | ---------- | +| 3.1 | TraceContext protobuf message | 0 | 1 | Phase 2 | +| 3.2 | Protobuf context serialization | 1-2 | 0 | 3.1 | +| 3.3 | PeerImp transaction instrumentation | 0 | 1 | 3.2 | +| 3.4 | NetworkOPs transaction processing | 0 | 1 | Phase 2 | +| 3.5 | HashRouter dedup visibility | 0 | 1 | 3.3 | +| 3.6 | Relay context propagation | 0 | 1-2 | 3.3, 3.5 | +| 3.7 | Build verification and testing | 0 | 0 | 3.1-3.6 | +| 3.8 | TX span peer version attribute | 0 | 1 | 3.3 | +| 3.9 | Deterministic transaction trace ID | 0-1 | 3 | 3.2, 3.3 | +| 3.10 | TxQ instrumentation (6 spans) | 1 | 1 | 3.4 | +| 3.11 | TX/TxQ span attribute gap fill | 0 | 5 | 3.3, 3.10 | + +**Parallel work**: Tasks 3.1 and 3.4 can start in parallel. Task 3.2 depends on 3.1. Tasks 3.3 and 3.5 depend on 3.2. Task 3.6 depends on 3.3 and 3.5. Task 3.8 depends on 3.3 (span must exist). Task 3.9 depends on 3.2 and 3.3. Task 3.10 depends on 3.4 (tx.process span must exist). + +**Exit Criteria** (from [06-implementation-phases.md §6.11.3](./06-implementation-phases.md)): + +- [x] Transaction traces span across nodes +- [x] Trace context in Protocol Buffer messages +- [ ] HashRouter deduplication visible in traces +- [ ] <5% overhead on transaction throughput +- [x] Deterministic trace_id: same trace_id for same tx across all nodes +- [x] Protobuf span_id propagation preserves parent-child ordering when available + +--- + +## Known Issues / Future Work + +### Unused trace_state proto field + +The `TraceContext.trace_state` field (field 4) in `xrpl.proto` is reserved for +W3C `tracestate` vendor-specific key-value pairs but is not read or written by +`TraceContextPropagator`. Wire it when cross-vendor trace propagation is needed. +No wire cost since proto `optional` fields are zero-cost when absent. diff --git a/OpenTelemetryPlan/Phase4_taskList.md b/OpenTelemetryPlan/Phase4_taskList.md new file mode 100644 index 00000000000..e83a16262ef --- /dev/null +++ b/OpenTelemetryPlan/Phase4_taskList.md @@ -0,0 +1,946 @@ +# Phase 4: Consensus Tracing Task List + +> **Goal**: Full observability into consensus rounds — track round lifecycle, phase transitions, proposal handling, and validation. This is the RUN phase that completes the distributed tracing story. +> +> **Scope**: RCLConsensus instrumentation for round starts, phase transitions (open/establish/accept), proposal send/receive, validation handling, and correlation with transaction traces from Phase 3. +> +> **Branch**: `pratik/otel-phase4-consensus-tracing` (from `pratik/otel-phase3-tx-tracing`) + +> **Note on attribute names**: the `xrpl..` keys shown below are +> written in the older dotted form for readability — it mirrors how the fully +> qualified attribute reads in a Tempo trace view. The implemented keys follow +> the convention in [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming) +> (underscore form, e.g. `consensus_round`, `consensus_mode`); the +> `*SpanNames.h` constants are the single source of truth. + +### Related Plan Documents + +| Document | Relevance | +| ------------------------------------------------------------ | ----------------------------------------------------------- | +| [04-code-samples.md](./04-code-samples.md) | Consensus instrumentation (§4.5.2), consensus span patterns | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | Consensus round flow (§1.4), key trace points (§1.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 4 tasks (§6.5), definition of done (§6.11.4) | +| [02-design-decisions.md](./02-design-decisions.md) | Consensus attribute schema (§2.4.4) | + +--- + +## Task 4.1: Instrument Consensus Round Start ✅ + +**Objective**: Create a root span for each consensus round that captures the round's key parameters. + +**Status**: DONE (implemented via Task 4a.2 `startRoundTracing()` helper). + +**What was done**: + +- `RCLConsensus::Adaptor::startRoundTracing()` creates `consensus.round` span + via `SpanGuard::hashSpan()` (deterministic) or `SpanGuard::span()` (attribute strategy) +- Attributes set: `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, + `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id` +- Round span stored as `roundSpan_` member in `RCLConsensus::Adaptor` +- `roundSpanContext_` snapshot captured for cross-thread span linking + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/app/consensus/RCLConsensus.h` (span and context members) + +**Reference**: + +- [04-code-samples.md §4.5.2](./04-code-samples.md) — startRound instrumentation example +- [01-architecture-analysis.md §1.4](./01-architecture-analysis.md) — Consensus round flow + +--- + +## Task 4.2: Instrument Phase Transitions ✅ + +**Objective**: Create child spans for each consensus phase (open, establish, accept) to show timing breakdown. + +**Status**: DONE. All consensus phases are now instrumented: + +- `consensus.establish` — created in `Consensus.h::startEstablishTracing()` +- `consensus.ledger_close` — created in `RCLConsensus.cpp::onClose()` +- `consensus.accept` / `consensus.accept.apply` — created in `onAccept()` / `doAccept()` +- `consensus.phase.open` — `openSpan_` member in `Consensus.h`, created in `startRoundInternal()`, ended in `closeLedger()` + +**Design notes**: + +- `phase` attribute — phases are distinguished by span names instead +- `phase.enter` / `phase.exit` events — not added (span start/end serves this purpose) +- `phase_duration_ms` attribute — not set (span duration captures this) + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/consensus/Consensus.h` (template-level establish phase tracking) + +**Reference**: + +- [04-code-samples.md §4.5.2](./04-code-samples.md) — phaseTransition instrumentation + +--- + +## Task 4.3: Instrument Proposal Handling ✅ + +**Objective**: Trace proposal send and receive to show validator coordination. + +**Status**: DONE. Both send and receive paths are instrumented. + +**What was done**: + +- In `Adaptor::propose()`: + - Creates `consensus.proposal.send` span via `SpanGuard::span()` + - Sets `xrpl.consensus.round` attribute + +- In `PeerImp::onMessage(TMProposeSet)`: + - Creates `consensus.proposal.receive` span + - Sets `trusted` attribute (bool) + +**Done here** (cross-node propagation, send + receive): + +- Trace context injection for `TMProposeSet::trace_context` in `propose()` +- Receive-side extraction in `PeerImp::onMessage(TMProposeSet)` via + `telemetry::proposalReceiveSpan()` (parents the receive span on the + sender's context when a valid `trace_context` is present) + +**Not implemented** (deferred to Phase 4b): + +- `consensus.proposal.relay` span in `share(RCLCxPeerPos)` + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +**Reference**: + +- [04-code-samples.md §4.5.2](./04-code-samples.md) — peerProposal instrumentation +- [02-design-decisions.md §2.4.4](./02-design-decisions.md) — Consensus attribute schema + +--- + +## Task 4.4: Instrument Validation Handling ✅ + +**Objective**: Trace validation send and receive to show ledger validation flow. + +**Status**: DONE. Both send and receive paths are instrumented. + +**What was done**: + +- In `Adaptor::validate()` (called from `doAccept()`): + - Creates `consensus.validation.send` span via `Adaptor::createValidationSpan()` + - Uses `SpanGuard::linkedSpan()` to create a follows-from link to the round span + - Thread-safe: uses `roundSpanContext_` snapshot (captured on consensus thread, + read on jtACCEPT thread) + - Sets `xrpl.ledger.seq` and `proposing` attributes + +- In `PeerImp::onMessage(TMValidation)`: + - Creates `consensus.validation.receive` span + - Sets `trusted` attribute (bool) + - Sets `xrpl.ledger.seq` attribute + +**Not implemented** (deferred to Phase 4b — cross-node propagation): + +- Validated ledger hash, signing time attributes on send span (see Task 4.8) + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Task 4.5: Add Consensus-Specific Attributes ✅ + +**Objective**: Enrich consensus spans with detailed attributes for debugging and analysis. + +**Status**: DONE. All core attributes are set across various spans, including the previously missing `tx_count` and `disputes_count`. + +**Implemented attributes** (across various spans): + +- `xrpl.ledger.seq` — on `consensus.round`, `consensus.accept.apply` +- `xrpl.consensus.round` — on `consensus.proposal.send` +- `xrpl.consensus.mode` — on `consensus.round`, `consensus.ledger_close` +- `proposers` — on `consensus.accept`, `consensus.establish`, `consensus.update_positions` +- `converge_percent` — on `consensus.establish`, `consensus.update_positions`, `consensus.check` +- `tx_count` — on `consensus.accept.apply` span (in `doAccept()`) +- `disputes_count` — on `consensus.update_positions` span (in `updateOurPositions()`) + +**Design notes**: + +- `phase` — phases distinguished by span names instead +- `phase_duration_ms` — span duration captures this + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/consensus/Consensus.h` + +--- + +## Task 4.6: Correlate Transaction and Consensus Traces ✅ + +**Objective**: Link transaction traces from Phase 3 with consensus traces so you can follow a transaction from submission through consensus into the ledger. + +**Status**: DONE. Transaction-consensus correlation implemented via `tx.included` events in `doAccept()`. + +**What was done**: + +- In `doAccept()` (RCLConsensus.cpp): + - Records `tx.included` events on the `consensus.accept.apply` span for each transaction in the accepted set + - Each event includes `xrpl.tx.id` attribute with the transaction hash + - This links consensus traces to individual transactions + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Task 4.7: Build Verification and Testing ✅ + +**Objective**: Verify all Phase 4 changes compile and don't affect consensus timing. + +**What to do**: + +1. Build with `telemetry=ON` — verify no compilation errors +2. Build with `telemetry=OFF` — verify no regressions (critical for consensus code) +3. Run existing consensus-related unit tests +4. Verify that `SpanGuard` factory methods compile to no-ops when disabled +5. Check that no consensus-critical code paths are affected by instrumentation overhead + +**Verification Checklist**: + +- [x] Build succeeds with telemetry ON +- [x] Build succeeds with telemetry OFF +- [x] Existing consensus tests pass +- [x] `SpanGuard` no-op implementation prevents overhead when telemetry is OFF +- [x] Phase timing instrumentation doesn't use blocking operations + +--- + +## Task 4.8: Consensus Validation Span Enrichment — NOT DONE + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds validation agreement context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 4 tasks 4.1-4.4 (span creation must exist). +> **Downstream**: Phase 7 (ValidationTracker reads these attributes), Phase 10 (validation checks). + +**Objective**: Add ledger hash, validation type, and quorum data to consensus validation spans on both send and receive paths. This enables trace-level validation agreement analysis — filter by ledger hash to see which validators agreed for a given ledger. + +**Status**: Not implemented. None of the enrichment attributes are set. The `consensus.validation.send` span only has `ledger.seq` and `proposing`. The `consensus.accept` span has `quorum` set to `result.proposers` (not the actual validator quorum from `app_.validators().quorum()`). No `PeerImp.cpp` changes were made. + +**What to do**: + +- Edit `src/xrpld/app/consensus/RCLConsensus.cpp`: + - On the `consensus.validation.send` span (in `validate()` / `doAccept()`): + - Add `xrpl.validation.ledger_hash` (string) — the ledger hash being validated + - Add `xrpl.validation.full` (bool) — whether this is a full validation (not partial) + - On the `consensus.accept` span (in `onAccept()`): + - Add `validation_quorum` (int64) — from `app_.validators().quorum()` + - Add `proposers_validated` (int64) — from `result.proposers` + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp`: + - On the `peer.validation.receive` span: + - Add `xrpl.peer.validation.ledger_hash` (string) — from deserialized `STValidation` object + - Add `xrpl.peer.validation.full` (bool) — from `STValidation` flags + +**New span attributes**: + +| Span | Attribute | Type | Source | +| --------------------------- | ---------------------------------- | ------ | --------------------------------- | +| `consensus.validation.send` | `xrpl.validation.ledger_hash` | string | Ledger hash from validate() args | +| `consensus.validation.send` | `xrpl.validation.full` | bool | Full vs partial validation | +| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | string | From STValidation deserialization | +| `peer.validation.receive` | `xrpl.peer.validation.full` | bool | From STValidation flags | +| `consensus.accept` | `validation_quorum` | int64 | `app_.validators().quorum()` | +| `consensus.accept` | `proposers_validated` | int64 | `result.proposers` | + +**Rationale**: The external dashboard's most valuable feature is validation agreement tracking. By recording the ledger hash on both outgoing and incoming validation spans, we create the raw data for agreement analysis at the trace level. Example Tempo query: + +``` +{name="consensus.validation.send"} | xrpl.validation.ledger_hash = "A1B2C3..." +``` + +Phase 7's `ValidationTracker` builds metric-level aggregation (1h/24h agreement %) on top of this data. + +**Key modified files (not yet modified)**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/overlay/detail/PeerImp.cpp` + +**Exit Criteria**: + +- [x] `consensus.validation.send` spans carry `ledger_hash` and `full_validation` +- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full` +- [ ] `consensus.accept` spans carry `validation_quorum` and `proposers_validated` +- [x] Ledger hash attributes match between send and receive for the same ledger +- [ ] No impact on consensus performance + +--- + +## Task 4.9: Consensus Span Attribute Gap Fill + +**Status**: COMPLETE + +**Objective**: Add workflow-critical attributes to consensus spans that enable operators to understand consensus outcomes, identify bow-out proposals, and correlate validations to specific ledgers. + +**Attributes added**: + +| Span | Attribute | Type | Source | +| --------------------------- | ----------------- | ------ | ------------------------------------- | +| `consensus.proposal.send` | `is_bow_out` | bool | `proposal.isBowOut()` | +| `consensus.accept` | `consensus_state` | string | `result.state` (yes/moved_on/expired) | +| `consensus.accept` | `disputes_count` | int64 | `result.disputes.size()` | +| `consensus.validation.send` | `ledger_hash` | string | `ledger.ledger->header().hash` | + +**New attr keys**: `ConsensusSpanNames.h` (`isBowOut`, `ledgerHash`). + +**Modified files**: + +- `src/xrpld/consensus/ConsensusSpanNames.h` +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Summary + +| Task | Description | Status | New Files | Modified Files | Depends On | +| ---- | ------------------------------------------- | ----------- | --------- | -------------- | ------------- | +| 4.1 | Consensus round start instrumentation | ✅ Done | 0 | 2 | Phase 3 | +| 4.2 | Phase transition instrumentation | ✅ Done | 0 | 1-2 | 4.1 | +| 4.3 | Proposal handling instrumentation | ✅ Done | 0 | 2 | 4.1 | +| 4.4 | Validation handling instrumentation | ✅ Done | 0 | 2 | 4.1 | +| 4.5 | Consensus-specific attributes | ✅ Done | 0 | 2 | 4.2, 4.3, 4.4 | +| 4.6 | Transaction-consensus correlation | ✅ Done | 0 | 1 | 4.2, Phase 3 | +| 4.7 | Build verification and testing | ✅ Done | 0 | 0 | 4.1-4.6 | +| 4.8 | Validation span enrichment (ext. dashboard) | ❌ Not done | 0 | 2 | 4.4 | +| 4.9 | Consensus span attribute gap fill | ✅ Done | 0 | 2 | 4.1-4.5 | + +**Parallel work**: Tasks 4.2, 4.3, and 4.4 can run in parallel after 4.1 is complete. Task 4.5 depends on all three. Task 4.6 depends on 4.2 and Phase 3. Task 4.8 depends on 4.4 (validation spans must exist). + +### Implemented Spans + +| Span Name | Method | Key Attributes | +| --------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.proposal.send` | `Adaptor::propose` | `xrpl.consensus.round`, `is_bow_out` | +| `consensus.ledger_close` | `Adaptor::onClose` | `xrpl.ledger.seq`, `xrpl.consensus.mode` | +| `consensus.accept` | `Adaptor::onAccept` | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | +| `consensus.accept.apply` | `Adaptor::doAccept` | `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `xrpl.ledger.seq`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction` | +| `consensus.validation.send` | `Adaptor::onAccept` (via validate) | `proposing`, `ledger_hash`, `ledger_seq`, `full_validation`, `validation_sign_time` | + +#### Close Time Attributes (consensus.accept.apply) + +The `consensus.accept.apply` span captures ledger close time agreement details +driven by `avCT_CONSENSUS_PCT` (75% validator agreement threshold): + +- **`close_time`** — Agreed-upon ledger close time (epoch seconds). When validators disagree (`consensusCloseTime == epoch`), this is synthetically set to `prevCloseTime + 1s`. +- **`close_time_correct`** — `true` if validators reached agreement, `false` if they "agreed to disagree" (close time forced to prev+1s). +- **`close_resolution_ms`** — Rounding granularity for close time (starts at 30s, decreases as ledger interval stabilizes). +- **`consensus_state`** — `"finished"` (normal) or `"moved_on"` (consensus failed, adopted best available). +- **`proposing`** — Whether this node was proposing. +- **`round_time_ms`** — Total consensus round duration. +- **`parent_close_time`** — Previous ledger's close time (epoch seconds). Enables computing close-time deltas across consecutive rounds without correlating separate spans. +- **`close_time_self`** — This node's own proposed close time before consensus voting. +- **`close_time_vote_bins`** — Number of distinct close-time vote bins from peer proposals. Higher values indicate less agreement among validators. +- **`resolution_direction`** — Whether close-time resolution `"increased"` (coarser), `"decreased"` (finer), or stayed `"unchanged"` relative to the previous ledger. + +**Exit Criteria** (from [06-implementation-phases.md §6.11.4](./06-implementation-phases.md)): + +- [x] Complete consensus round traces +- [x] Phase transitions visible (open, establish, close, accept) +- [x] Proposals and validations traced — send and receive; relay deferred to Phase 4b +- [x] Close time agreement tracked (per `avCT_CONSENSUS_PCT`) +- [x] No impact on consensus timing +- [x] Transaction-consensus correlation (Task 4.6) — `tx.included` events in doAccept +- [ ] Validation span enrichment (Task 4.8) — not implemented + +--- + +# Phase 4a: Establish-Phase Gap Fill & Cross-Node Correlation + +> **Goal**: Fill tracing gaps in the consensus establish phase (disputes, convergence, +> threshold escalation, mode changes) and establish cross-node correlation using a +> deterministic shared trace ID derived from `previousLedger.id()`. +> +> **Approach**: Direct instrumentation in `Consensus.h` and `RCLConsensus.cpp`. +> All spans use `SpanGuard` factory methods (`span()`, `hashSpan()`, `linkedSpan()`) +> with `TraceCategory::Consensus` gating. Long-lived spans (round, establish) are +> stored as `std::optional` class members. Short-lived scoped spans +> (update_positions, check) are local variables. No macros are used — all tracing +> is via direct `SpanGuard` API calls. `SpanGuard` compiles to no-ops when +> telemetry is disabled. +> +> **Branch**: `pratik/otel-phase4-consensus-tracing` + +## Design: Switchable Correlation Strategy + +Two strategies for cross-node trace correlation, switchable via config: + +### Strategy A — Deterministic Trace ID (Default) + +Derive `trace_id = SHA256(previousLedger.id())[0:16]` so all nodes in the same +consensus round share the same trace_id without P2P context propagation. + +- **Pros**: All nodes appear in the same trace in Tempo automatically. + No collector-side post-processing needed. +- **Cons**: Overrides OTel's random trace_id generation; requires custom + `IdGenerator` or manual span context construction. + +### Strategy B — Attribute-Based Correlation + +Use normal random trace_id but attach `xrpl.consensus.ledger_id` as an attribute +on every consensus span. Correlation happens at query time via Tempo/Grafana +`by attribute` queries. + +- **Pros**: Standard OTel trace_id semantics; no SDK customization. +- **Cons**: Cross-node correlation requires query-time joins, not automatic. + +### Config + +```ini +[telemetry] +# "deterministic" (default) or "attribute" +consensus_trace_strategy=deterministic +``` + +The C++ API to query this at runtime is `Telemetry::getConsensusTraceStrategy()`, +which returns a `std::string const&` (`"deterministic"` or `"attribute"`). + +### Implementation + +In `RCLConsensus::Adaptor::startRound()`: + +- If `deterministic`: + 1. Compute `trace_id_bytes = SHA256(prevLedgerID)[0:16]` + 2. Construct `opentelemetry::trace::TraceId(trace_id_bytes)` + 3. Create a synthetic `SpanContext` with this trace_id and a random span_id: + ```cpp + auto traceId = opentelemetry::trace::TraceId(trace_id_bytes); + auto spanId = opentelemetry::trace::SpanId(random_8_bytes); + auto syntheticCtx = opentelemetry::trace::SpanContext( + traceId, spanId, opentelemetry::trace::TraceFlags(1), false); + ``` + 4. Wrap in `opentelemetry::context::Context` via + `opentelemetry::trace::SetSpan(context, syntheticSpan)` + 5. Call `startSpan("consensus.round", parentContext)` so the new span + inherits the deterministic trace_id. +- If `attribute`: start a normal `consensus.round` span, set + `xrpl.consensus.ledger_id = previousLedger.id()` as attribute. + +Both strategies always set `xrpl.consensus.round_id` (round number) and +`xrpl.consensus.ledger_id` (previous ledger hash) as attributes. + +--- + +## Design: Span Hierarchy + +``` +consensus.round (root — created in RCLConsensus::startRound, closed at accept) +│ link → previous round's SpanContext (follows-from) +│ +├── consensus.establish (phaseEstablish → acceptance, in Consensus.h) +│ ├── consensus.update_positions (each updateOurPositions call) +│ │ └── consensus.dispute.resolve (per-tx dispute resolution event) +│ ├── consensus.check (each haveConsensus call) +│ └── consensus.mode_change (short-lived span in adaptor on mode transition) +│ +├── consensus.accept (existing onAccept span — reparented under round) +│ +└── consensus.validation.send (existing — reparented, follows-from link to round) +``` + +### Span Links (follows-from relationships) + +| Link Source | Link Target | Rationale | +| ----------------------------------------- | -------------------------- | ------------------------------------------------------------------------------ | +| `consensus.round` (N+1) | `consensus.round` (N) | Causal chain: round N+1 exists because round N accepted | +| `consensus.validation.send` | `consensus.round` | Validation follows from the round that produced it; may outlive the round span | +| _(Phase 4b)_ Received proposal processing | Sender's `consensus.round` | Cross-node causal link via P2P context propagation | + +--- + +## Task 4a.0: Prerequisites — Extend SpanGuard and Telemetry APIs ✅ + +**Objective**: Add missing API surface needed by later tasks. + +**Status**: Done, but implemented differently than originally planned. The macro-based +approach (`XRPL_TRACE_CONSENSUS`, `XRPL_TRACE_ADD_EVENT`, `XRPL_TRACE_SET_ATTR`) was +**not used**. Instead, all consensus tracing uses `SpanGuard` factory methods and +direct method calls, which is cleaner and avoids macro control-flow issues. + +**What was done**: + +1. **`SpanGuard::addEvent()` with attributes** — implemented as planned: + + ```cpp + using EventAttribute = std::pair; + + void addEvent(std::string_view name, + std::initializer_list attrs); + ``` + + Callers pass plain `string_view` pairs; the implementation converts internally. + + ```cpp + // Actual usage in Consensus.h::updateOurPositions(): + span.addEvent( + "dispute.resolve", + {{consensus::span::attr::txId, to_string(txId)}, + {consensus::span::attr::disputeOurVote, dispute.getOurVote() ? "yes" : "no"}}); + ``` + +2. **Span link support** — implemented via `SpanGuard::linkedSpan()` static factory + instead of a `Telemetry::startSpan()` overload: + + ```cpp + static SpanGuard linkedSpan( + std::string_view name, SpanContext const& linkTarget); + ``` + +3. **No macros added** — `TracingInstrumentation.h` was not created. The `XRPL_TRACE_CONSENSUS`, + `XRPL_TRACE_ADD_EVENT`, and `XRPL_TRACE_SET_ATTR` macros from the original plan were + not implemented. All consensus tracing uses direct `SpanGuard` API: + - `SpanGuard::span()` — create scoped spans + - `SpanGuard::hashSpan()` — create spans with deterministic trace IDs + - `SpanGuard::linkedSpan()` — create spans with follows-from links + - `span.setAttribute()` — set attributes directly + - `span.addEvent()` — add events directly + +**Key modified files**: + +- `include/xrpl/telemetry/SpanGuard.h` — `addEvent()` overload, `EventAttribute` type alias +- `src/libxrpl/telemetry/SpanGuard.cpp` — `addEvent()` implementation + +--- + +## Task 4a.1: Adaptor `getTelemetry()` Method — NOT DONE (Not Needed) + +**Objective**: Give `Consensus.h` access to the telemetry subsystem without +coupling the generic template to OTel headers. + +**Status**: Not implemented as specified. The `getTelemetry()` adaptor method was +not needed because `SpanGuard::span()` is a static factory method that internally +checks telemetry state via the global `Telemetry` singleton. `Consensus.h` creates +spans by calling `SpanGuard::span(TraceCategory::Consensus, ...)` directly, without +needing adaptor access. Only `RCLConsensus::Adaptor` uses `app_.getTelemetry()` +directly (for `getConsensusTraceStrategy()` in `startRoundTracing()`). + +**Key insight**: The `XRPL_TRACE_*` macro approach would have required +`adaptor_.getTelemetry()`. Since macros were not used, this task became unnecessary. + +--- + +## Task 4a.2: Switchable Round Span with Deterministic Trace ID ✅ + +**Objective**: Create a `consensus.round` root span in `startRound()` that uses +the switchable correlation strategy. Store span context as a member for child +spans in `Consensus.h`. + +**Status**: Done. Implemented in `Adaptor::startRoundTracing()`. + +**What was done**: + +- `RCLConsensus::Adaptor::startRoundTracing()` helper: + - Reads `consensus_trace_strategy` via `app_.getTelemetry().getConsensusTraceStrategy()` + - **Deterministic**: uses `SpanGuard::hashSpan()` with `prevLgr.id()` data + - **Attribute**: uses `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "round")` + - Sets attributes: `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id` + - Captures `roundSpanContext_` snapshot for cross-thread span linking + - Saves `prevRoundContext_` from previous round for follows-from links + +- **`SpanGuard::hashSpan()` factory**: encapsulates deterministic trace ID logic: + + ```cpp + static SpanGuard hashSpan( + TraceCategory cat, std::string_view name, + std::uint8_t const* hashData, std::size_t hashSize); + ``` + + Derives `trace_id = hashData[0:16]` so all nodes in the same round share + the same trace_id. Compiles to no-op when telemetry is disabled. + +- `consensus_trace_strategy` config parsed in `TelemetryConfig.cpp`, + stored in `Telemetry::Setup`, accessible via `Telemetry::getConsensusTraceStrategy()` + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` — `startRoundTracing()` implementation +- `src/xrpld/app/consensus/ConsensusSpanNames.h` — **(new)** compile-time span name and attribute key constants +- `include/xrpl/telemetry/Telemetry.h` — `consensusTraceStrategy` in Setup, `getConsensusTraceStrategy()` +- `src/libxrpl/telemetry/TelemetryConfig.cpp` — parse new config option + +--- + +## Task 4a.3: Span Members in `Consensus.h` ✅ + +**Objective**: Add span storage to the `Consensus` class so that spans created +in `startRound()` (adaptor) are accessible from `phaseEstablish()`, +`updateOurPositions()`, and `haveConsensus()` (template methods). + +**Status**: Done with documented plan deviation. + +**What was done**: + +- `establishSpan_` added to `Consensus` private members (as planned): + + ```cpp + std::optional establishSpan_; + ``` + +- **Plan deviation**: `roundSpan_`, `prevRoundContext_`, and `roundSpanContext_` + are stored in `RCLConsensus::Adaptor` (not `Consensus.h`) because the adaptor + has access to telemetry config for the deterministic trace ID strategy. + +- **No `#ifdef XRPL_ENABLE_TELEMETRY` guards**: Members use `std::optional` + and `SpanContext` which have no-op implementations when telemetry is disabled, + so `#ifdef` guards are unnecessary. The members are always present in the class + layout but incur negligible overhead. + +- Includes added unconditionally to `Consensus.h`: + ```cpp + #include + #include + ``` + No `TracingInstrumentation.h` include (file doesn't exist; macros not used). + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` +- `src/xrpld/app/consensus/RCLConsensus.h` (round span and context members) + +--- + +## Task 4a.4: Instrument `phaseEstablish()` ✅ + +**Objective**: Create `consensus.establish` span wrapping the establish phase, +with attributes for convergence progress. + +**Status**: Done. Implemented via three private helpers in `Consensus.h`. + +**What was done**: + +- `startEstablishTracing()` — creates `consensus.establish` span via + `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "establish")`. + Called once at start of establish phase. No `#ifdef` guards needed — + `SpanGuard::span()` returns a no-op guard when telemetry is disabled. + +- `updateEstablishTracing()` — sets attributes on each `phaseEstablish()` call: + - `converge_percent` — `convergePercent_` + - `establish_count` — `establishCounter_` + - `proposers` — `currPeerPositions_.size()` + +- `endEstablishTracing()` — calls `establishSpan_.reset()` on phase exit. + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` — `phaseEstablish()` method + 3 helper methods + +--- + +## Task 4a.5: Instrument `updateOurPositions()` ✅ + +**Objective**: Trace each position update cycle including dispute resolution +details. + +**Status**: DONE. Span, dispute events with yays/nays, and disputes_count attribute are all implemented. + +**What was done**: + +- Creates `consensus.update_positions` scoped span via + `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "update_positions")`: + + ```cpp + auto span = SpanGuard::span(TraceCategory::Consensus, seg::consensus, "update_positions"); + ``` + +- Attributes set: + - `converge_percent` — current convergence + - `proposers` — `currPeerPositions_.size()` + - `have_close_time_consensus` — close time consensus state + - `close_time_threshold` — `avCT_CONSENSUS_PCT` + - `disputes_count` — number of active disputes + +- Dispute events recorded via direct `span.addEvent()` call with yays/nays: + ```cpp + span.addEvent( + "dispute.resolve", + {{consensus::span::attr::txId, to_string(txId)}, + {consensus::span::attr::disputeOurVote, dispute.getOurVote() ? "yes" : "no"}, + {consensus::span::attr::disputeYays, std::to_string(dispute.getYays())}, + {consensus::span::attr::disputeNays, std::to_string(dispute.getNays())}}); + ``` + +**Not implemented**: + +- `proposers_agreed` / `proposers_total` attributes — not set + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` — `updateOurPositions()` method +- `src/xrpld/consensus/DisputedTx.h` — added `getYays()` / `getNays()` (currently unused) + +--- + +## Task 4a.6: Instrument `haveConsensus()` (Threshold & Convergence) ✅ + +**Objective**: Trace consensus checking including threshold escalation. + +**Status**: DONE. The `consensus.check` span is created with all planned attributes +including the avalanche threshold. + +**What was done**: + +- Creates `consensus.check` scoped span via + `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "check")`: + + ```cpp + auto span = SpanGuard::span(TraceCategory::Consensus, seg::consensus, "check"); + ``` + +- Attributes set: + - `agree_count` — peers that agree with our position + - `disagree_count` — peers that disagree + - `converge_percent` — convergence percentage + - `have_close_time_consensus` — close time consensus state + - `threshold_percent` — set to `avCT_CONSENSUS_PCT` (75%) + - `consensus_result` — "yes", "no", or "moved_on" + - `avalanche_threshold` — the escalated weight from `getNeededWeight()` on the `consensus.update_positions` span + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` — `haveConsensus()` method + +--- + +## Task 4a.7: Instrument Mode Changes ✅ + +**Objective**: Trace consensus mode transitions (proposing ↔ observing, +wrongLedger, switchedLedger). + +**Status**: Done. + +**What was done**: + +- In `RCLConsensus::Adaptor::onModeChange()`, creates a scoped span via direct + `SpanGuard::span()` call: + + ```cpp + auto span = telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, "mode_change"); + span.setAttribute(consensus::span::attr::modeOld, to_string(before).c_str()); // "mode_old" + span.setAttribute(consensus::span::attr::modeNew, to_string(after).c_str()); // "mode_new" + ``` + +- `MonitoredMode::set()` in `Consensus.h` calls `adaptor_.onModeChange(before, after)`. + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` — `onModeChange()` + +--- + +## Task 4a.8: Reparent Existing Spans Under Round ✅ + +**Objective**: Make existing consensus spans (`consensus.accept`, +`consensus.accept.apply`, `consensus.validation.send`) children of the +`consensus.round` root span instead of being standalone. + +**Status**: DONE. All three spans are now parented under the round span. + +**What was done**: + +- `consensus.validation.send` uses `SpanGuard::linkedSpan()` to create a + follows-from link to `roundSpanContext_`. This is thread-safe because + `roundSpanContext_` is a lightweight `SpanContext` snapshot captured on the + consensus thread and read on the jtACCEPT worker thread. + +- `consensus.accept` and `consensus.accept.apply` now use + `SpanGuard::childSpan(name, roundSpanContext_)` instead of `SpanGuard::span()` + to explicitly parent under the round span context. This solves the cross-thread + parenting problem: + - `doAccept()` runs on the jtACCEPT worker thread (not the consensus thread) + - `childSpan()` explicitly passes the parent context, bypassing OTel's + thread-local context propagation + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Task 4a.9: Build Verification and Testing ✅ + +**Objective**: Verify all Phase 4a changes compile cleanly with telemetry ON +and OFF, and don't affect consensus timing. + +**What to do**: + +1. Build with `telemetry=ON` — verify no compilation errors +2. Build with `telemetry=OFF` — verify `SpanGuard` compiles to no-ops +3. Run existing consensus unit tests +4. Verify `SpanGuard` / `SpanContext` members have negligible overhead when disabled +5. Run `pccl` pre-commit checks + +**Verification Checklist**: + +- [x] Build succeeds with telemetry ON +- [x] Build succeeds with telemetry OFF +- [x] Existing consensus tests pass +- [x] `SpanGuard` no-op path verified (no `#ifdef` needed — disabled at runtime) +- [x] No new virtual calls in hot consensus paths +- [x] `pccl` passes + +--- + +## Phase 4a Summary + +| Task | Description | Status | New Files | Modified Files | Depends On | +| ---- | ------------------------------------------------ | ------------------------ | --------- | -------------- | ---------- | +| 4a.0 | Prerequisites: extend SpanGuard & Telemetry APIs | ✅ Done (no macros) | 0 | 2 | Phase 4 | +| 4a.1 | Adaptor `getTelemetry()` method | ⏭️ Skipped (not needed) | 0 | 0 | Phase 4 | +| 4a.2 | Switchable round span with deterministic traceID | ✅ Done | 1 | 3 | 4a.0 | +| 4a.3 | Span members in `Consensus.h` | ✅ Done (with deviation) | 0 | 2 | — | +| 4a.4 | Instrument `phaseEstablish()` | ✅ Done | 0 | 1 | 4a.3 | +| 4a.5 | Instrument `updateOurPositions()` | ✅ Done | 0 | 2 | 4a.0, 4a.3 | +| 4a.6 | Instrument `haveConsensus()` (thresholds) | ✅ Done | 0 | 1 | 4a.3 | +| 4a.7 | Instrument mode changes | ✅ Done | 0 | 1 | — | +| 4a.8 | Reparent existing spans under round | ✅ Done | 0 | 1 | 4a.0, 4a.2 | +| 4a.9 | Build verification and testing | ✅ Done | 0 | 0 | 4a.0-4a.8 | + +**Parallel work**: Tasks 4a.0 and 4a.1 can run in parallel. Tasks 4a.4, 4a.5, 4a.6, and 4a.7 can run in parallel after 4a.3 (and 4a.0 for 4a.5). + +### New Spans (Phase 4a) + +| Span Name | Location | Key Attributes (actually set) | +| ---------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | `RCLConsensus.cpp` | `xrpl.consensus.round_id`, `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy` | +| `consensus.establish` | `Consensus.h` | `converge_percent`, `establish_count`, `proposers` | +| `consensus.update_positions` | `Consensus.h` | `converge_percent`, `proposers`, `have_close_time_consensus`, `close_time_threshold`, `disputes_count`, `avalanche_threshold` | +| `consensus.check` | `Consensus.h` | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | +| `consensus.mode_change` | `RCLConsensus.cpp` | `mode_old`, `mode_new` | + +### New Events (Phase 4a) + +| Event Name | Parent Span | Attributes (actually set) | +| ----------------- | ---------------------------- | ---------------------------------------------------------------- | +| `dispute.resolve` | `consensus.update_positions` | `xrpl.tx.id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | +| `tx.included` | `consensus.accept.apply` | `xrpl.tx.id` | + +### New Attributes (Phase 4a) + +```cpp +// Round-level (on consensus.round) — ALL IMPLEMENTED +"xrpl.consensus.round_id" = int64 // Consensus round number +"xrpl.consensus.ledger_id" = string // previousLedger.id() hash +"trace_strategy" = string // "deterministic" or "attribute" + +// Establish-level — IMPLEMENTED +"converge_percent" = int64 // Convergence % (0-100+) +"establish_count" = int64 // Number of establish iterations +"agree_count" = int64 // Peers that agree (haveConsensus) +"disagree_count" = int64 // Peers that disagree +"threshold_percent" = int64 // Current threshold (avCT_CONSENSUS_PCT = 75%) +"consensus_result" = string // "yes", "no", "moved_on" +"have_close_time_consensus" = bool // Close time consensus reached +"close_time_threshold" = int64 // Close time voting threshold + +// Establish-level — IMPLEMENTED +"disputes_count" = int64 // Active disputes (on update_positions) +"avalanche_threshold" = int64 // Escalated weight (on update_positions) + +// Establish-level — NOT IMPLEMENTED +// "proposers_agreed" = int64 // Peers agreeing with us — not set +// "proposers_total" = int64 // Total peer positions — not set (not defined) + +// Mode change — ALL IMPLEMENTED +"mode_old" = string // Previous mode +"mode_new" = string // New mode +``` + +### Implementation Notes + +- **No macros**: The planned `XRPL_TRACE_CONSENSUS`, `XRPL_TRACE_ADD_EVENT`, and + `XRPL_TRACE_SET_ATTR` macros were not implemented. All consensus tracing uses + `SpanGuard` factory methods (`span()`, `hashSpan()`, `linkedSpan()`) and direct + method calls (`setAttribute()`, `addEvent()`). This avoids macro control-flow + issues and is cleaner than the planned approach. +- **Separation of concerns**: All non-trivial telemetry code extracted to private + helpers (`startRoundTracing`, `createValidationSpan`, `startEstablishTracing`, + `updateEstablishTracing`, `endEstablishTracing`). Business logic methods contain + single-line calls to these helpers. +- **Thread safety**: `createValidationSpan()` runs on the jtACCEPT worker thread. + Instead of accessing `roundSpan_` across threads, a `roundSpanContext_` snapshot + (lightweight `SpanContext` value type) is captured on the consensus thread in + `startRoundTracing()` and read by `createValidationSpan()`. The job queue + provides the happens-before guarantee. +- **No `#ifdef` guards**: Span members use `std::optional` and `SpanContext` + which have no-op implementations when telemetry is disabled. No `#ifdef XRPL_ENABLE_TELEMETRY` + guards needed around members or includes. +- **No `getTelemetry()` adaptor method**: `SpanGuard::span()` is a static factory that + internally checks telemetry state, so `Consensus.h` doesn't need adaptor access + for span creation. Only `RCLConsensus::Adaptor` accesses `app_.getTelemetry()` directly. +- **Config validation**: `consensus_trace_strategy` is validated to be either + `"deterministic"` or `"attribute"`, falling back to `"deterministic"` for + unrecognised values. +- **Plan deviation**: `roundSpan_` is stored in `RCLConsensus::Adaptor` (not + `Consensus.h`) because the adaptor has access to telemetry config and can + implement the deterministic trace ID strategy. `establishSpan_` is correctly + in `Consensus.h` as planned. + +--- + +# Phase 4b: Cross-Node Propagation (Future — Documentation Only) + +> **Goal**: Wire `TraceContextPropagator` for P2P messages so that proposals +> and validations carry trace context between nodes. This enables true +> distributed tracing where a proposal sent by Node A creates a child span +> on Node B. +> +> **Status**: NOT IMPLEMENTED. The protobuf fields and propagator class exist +> but are not wired. This section documents the design for future work. + +## Architecture + +``` +Node A (proposing) Node B (receiving) +───────────────── ────────────────── +consensus.round consensus.round +├── propose() ├── peerProposal() +│ └── TraceContextPropagator │ └── TraceContextPropagator +│ ::injectToProtobuf( │ ::extractFromProtobuf( +│ TMProposeSet.trace_context) │ TMProposeSet.trace_context) +│ │ └── span link → Node A's context +└── validate() └── onValidation() + └── inject into TMValidation └── extract from TMValidation +``` + +## Wiring Points + +| Message | Inject Location | Extract Location | Protobuf Field | +| --------------- | ---------------------------------- | ----------------------------------- | -------------------------- | +| `TMProposeSet` | `Adaptor::propose()` | `PeerImp::onMessage(TMProposeSet)` | field 1001: `TraceContext` | +| `TMValidation` | `Adaptor::validate()` | `PeerImp::onMessage(TMValidation)` | field 1001: `TraceContext` | +| `TMTransaction` | `NetworkOPs::processTransaction()` | `PeerImp::onMessage(TMTransaction)` | field 1001: `TraceContext` | + +## Span Link Semantics + +Received messages use **span links** (follows-from), NOT parent-child: + +- The receiver's processing span links to the sender's context +- This preserves each node's independent trace tree +- Cross-node correlation visible via linked traces in Tempo + +## Interaction with Deterministic Trace ID (Strategy A) + +When using deterministic trace_id (Phase 4a default), cross-node spans already +share the same trace_id. P2P propagation adds **span-level** linking: + +- Without propagation: spans from different nodes appear in the same trace + (same trace_id) but without parent-child or follows-from relationships. +- With propagation: spans have explicit links showing which proposal/validation + from Node A caused processing on Node B. + +## Prerequisites + +- Phase 4a (this task list) — establish phase tracing must be in place +- `TraceContextPropagator` free functions (already exist in + `include/xrpl/telemetry/TraceContextPropagator.h`) +- Protobuf `TraceContext` message (already exists, field 1001) diff --git a/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md new file mode 100644 index 00000000000..2bf33a390cb --- /dev/null +++ b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md @@ -0,0 +1,221 @@ +# Phase 5: Integration Test Task List + +> **Goal**: End-to-end verification of the complete telemetry pipeline using a +> 6-node consensus network. Proves that RPC, transaction, and consensus spans +> flow through the observability stack (otel-collector, Tempo, Prometheus, +> Grafana) under realistic conditions. +> +> **Scope**: Integration test script, manual testing plan, 6-node local network +> setup, Tempo/Prometheus/Grafana verification. +> +> **Branch**: `pratik/otel-phase5-docs-deployment` + +### Related Plan Documents + +| Document | Relevance | +| ---------------------------------------------------------------- | ------------------------------------------ | +| [07-observability-backends.md](./07-observability-backends.md) | Tempo, Grafana, Prometheus setup | +| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config, Docker Compose | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 5 tasks, definition of done | +| [Phase5_taskList.md](./Phase5_taskList.md) | Phase 5 main task list (5.6 = integration) | + +--- + +## Task IT.1: Create Integration Test Script + +**Objective**: Automated bash script that stands up a 6-node xrpld network +with telemetry, exercises all span categories, and verifies data in +Tempo/Prometheus. + +**What to do**: + +- Create `docker/telemetry/integration-test.sh`: + - Prerequisites check (docker, xrpld binary, curl, jq) + - Start observability stack via `docker compose` + - Generate 6 validator key pairs via temp standalone xrpld + - Generate 6 node configs + shared `validators.txt` + - Start 6 xrpld nodes in consensus mode (`--start`, no `-a`) + - Wait for all nodes to reach `"proposing"` state (120s timeout) + +**Key new file**: `docker/telemetry/integration-test.sh` + +**Verification**: + +- [ ] Script starts without errors +- [ ] All 6 nodes reach "proposing" state +- [ ] Observability stack is healthy (otel-collector, Tempo, Prometheus, Grafana) + +--- + +## Task IT.2: RPC Span Verification (Phase 2) + +**Objective**: Verify RPC spans flow through the telemetry pipeline. + +**What to do**: + +- Send `server_info`, `server_state`, `ledger` RPCs to node1 (port 5005) +- Wait for batch export (5s) +- Query Tempo API for: + - `rpc.request` spans (ServerHandler::onRequest) + - `rpc.process` spans (ServerHandler::processRequest) + - `rpc.command.server_info` spans (callMethod) + - `rpc.command.server_state` spans (callMethod) + - `rpc.command.ledger` spans (callMethod) +- Verify `command` attribute present on `rpc.command.*` spans + +**Verification**: + +- [ ] Tempo shows `rpc.request` traces +- [ ] Tempo shows `rpc.process` traces +- [ ] Tempo shows `rpc.command.*` traces with correct attributes + +--- + +## Task IT.3: Transaction Span Verification (Phase 3) + +**Objective**: Verify transaction spans flow through the telemetry pipeline. + +**What to do**: + +- Get genesis account sequence via `account_info` RPC +- Submit Payment transaction using genesis seed (`snoPBrXtMeMyMHUVTgbuqAfg1SUTb`) +- Wait for consensus inclusion (10s) +- Query Tempo API for: + - `tx.process` spans (NetworkOPsImp::processTransaction) on submitting node + - `tx.receive` spans (PeerImp::handleTransaction) on peer nodes +- Verify `xrpl.tx.hash` attribute on `tx.process` spans +- Verify `xrpl.peer.id` attribute on `tx.receive` spans + +**Verification**: + +- [ ] Tempo shows `tx.process` traces with `xrpl.tx.hash` +- [ ] Tempo shows `tx.receive` traces with `xrpl.peer.id` + +--- + +## Task IT.4: Consensus Span Verification (Phase 4) + +**Objective**: Verify consensus spans flow through the telemetry pipeline. + +**What to do**: + +- Consensus runs automatically in 6-node network +- Query Tempo API for: + - `consensus.proposal.send` (Adaptor::propose) + - `consensus.ledger_close` (Adaptor::onClose) + - `consensus.accept` (Adaptor::onAccept) + - `consensus.validation.send` (Adaptor::validate) +- Verify attributes: + - `xrpl.consensus.mode` on `consensus.ledger_close` + - `proposers` on `consensus.accept` + - `xrpl.consensus.ledger.seq` on `consensus.validation.send` + +**Verification**: + +- [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode` +- [ ] Tempo shows `consensus.accept` traces with `proposers` +- [ ] Tempo shows `consensus.proposal.send` traces +- [ ] Tempo shows `consensus.validation.send` traces + +--- + +## Task IT.5: Spanmetrics Verification (Phase 5) + +**Objective**: Verify spanmetrics connector derives RED metrics from spans. + +**What to do**: + +- Query Prometheus for `traces_span_metrics_calls_total` +- Query Prometheus for `traces_span_metrics_duration_milliseconds_count` +- Verify Grafana loads at `http://localhost:3000` + +**Verification**: + +- [ ] Prometheus returns non-empty results for `traces_span_metrics_calls_total` +- [ ] Prometheus returns non-empty results for duration histogram +- [ ] Grafana UI accessible with dashboards visible + +--- + +## Task IT.6: Manual Testing Plan + +**Objective**: Document how to run tests manually for future reference. + +**What to do**: + +- Create `docker/telemetry/TESTING.md` with: + - Prerequisites section + - Single-node standalone test (quick verification) + - 6-node consensus test (full verification) + - Expected span catalog (all 11 span names with attributes) + - Verification queries (Tempo API, Prometheus API) + - Troubleshooting guide + +**Key new file**: `docker/telemetry/TESTING.md` + +**Verification**: + +- [ ] Document covers both single-node and multi-node testing +- [ ] All 11 span names documented with source file and attributes +- [ ] Troubleshooting section covers common failure modes + +--- + +## Task IT.7: Run and Verify + +**Objective**: Execute the integration test and validate results. + +**What to do**: + +- Run `docker/telemetry/integration-test.sh` locally +- Debug any failures +- Leave stack running for manual verification +- Share URLs: + - Tempo: `http://localhost:3200` + - Grafana: `http://localhost:3000` + - Prometheus: `http://localhost:9090` + +**Verification**: + +- [ ] Script completes with all checks passing +- [ ] Tempo UI shows xrpld service with all expected span names +- [ ] Grafana dashboards load and show data + +--- + +## Task IT.8: Commit + +**Objective**: Commit all new files to Phase 5 branch. + +**What to do**: + +- Run `pcc` (pre-commit checks) +- Commit 3 new files to `pratik/otel-phase5-docs-deployment` + +**Verification**: + +- [ ] `pcc` passes +- [ ] Commit created on Phase 5 branch + +--- + +## Summary + +| Task | Description | New Files | Depends On | +| ---- | ----------------------------- | --------- | ---------- | +| IT.1 | Integration test script | 1 | Phase 5 | +| IT.2 | RPC span verification | 0 | IT.1 | +| IT.3 | Transaction span verification | 0 | IT.1 | +| IT.4 | Consensus span verification | 0 | IT.1 | +| IT.5 | Spanmetrics verification | 0 | IT.1 | +| IT.6 | Manual testing plan | 1 | -- | +| IT.7 | Run and verify | 0 | IT.1-IT.6 | +| IT.8 | Commit | 0 | IT.7 | + +**Exit Criteria**: + +- [ ] All 6 xrpld nodes reach "proposing" state +- [ ] All 11 expected span names visible in Tempo +- [ ] Spanmetrics available in Prometheus +- [ ] Grafana dashboards show data +- [ ] Manual testing plan document complete diff --git a/OpenTelemetryPlan/Phase5_taskList.md b/OpenTelemetryPlan/Phase5_taskList.md new file mode 100644 index 00000000000..ed5dacc3055 --- /dev/null +++ b/OpenTelemetryPlan/Phase5_taskList.md @@ -0,0 +1,250 @@ +# Phase 5: Documentation & Deployment Task List + +> **Goal**: Production readiness — Grafana dashboards, spanmetrics pipeline, operator runbook, alert definitions, and final integration testing. This phase ensures the telemetry system is useful and maintainable in production. +> +> **Scope**: Grafana dashboard definitions, OTel Collector spanmetrics connector, Prometheus integration, alert rules, operator documentation, and production-ready Docker Compose stack. +> +> **Branch**: `pratik/otel-phase5-docs-deployment` (from `pratik/otel-phase4-consensus-tracing`) + +> **Note on attribute names**: the `xrpl..` keys shown below +> (including the collector spanmetrics dimension examples) are written in the +> older dotted form for readability — it mirrors how the fully qualified +> attribute reads in a Tempo trace view. The implemented keys follow the +> convention in [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming) +> (underscore form, e.g. `command`, `rpc_status`); the `*SpanNames.h` constants +> are the single source of truth, and the real collector dimensions must use +> those exact underscore keys (the CI naming check enforces this). + +### Related Plan Documents + +| Document | Relevance | +| ---------------------------------------------------------------- | -------------------------------------------------------------------------- | +| [07-observability-backends.md](./07-observability-backends.md) | Tempo setup (§7.1), Grafana dashboards (§7.6), alerts (§7.6.3) | +| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config (§5.5), production config (§5.5.2), Docker Compose (§5.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 5 tasks (§6.6), definition of done (§6.11.5) | + +--- + +## Task 5.1: Add Spanmetrics Connector to OTel Collector + +**Objective**: Derive RED metrics (Rate, Errors, Duration) from trace spans automatically, enabling Grafana time-series dashboards. + +**What to do**: + +- Edit `docker/telemetry/otel-collector-config.yaml`: + - Add `spanmetrics` connector: + ```yaml + connectors: + spanmetrics: + histogram: + explicit: + buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s] + dimensions: + - name: command + - name: rpc_status + - name: consensus_phase + - name: tx_type + ``` + - Add `prometheus` exporter: + ```yaml + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + ``` + - Wire the pipeline: + ```yaml + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/tempo, spanmetrics] + metrics: + receivers: [spanmetrics] + exporters: [prometheus] + ``` + +- Edit `docker/telemetry/docker-compose.yml`: + - Expose port `8889` on the collector for Prometheus scraping + - Add Prometheus service + - Add Prometheus as Grafana datasource + +**Key modified files**: + +- `docker/telemetry/otel-collector-config.yaml` +- `docker/telemetry/docker-compose.yml` + +**Key new files**: + +- `docker/telemetry/prometheus.yml` (Prometheus scrape config) +- `docker/telemetry/grafana/provisioning/datasources/prometheus.yaml` + +**Reference**: + +- [POC_taskList.md §Next Steps](./POC_taskList.md) — Metrics pipeline for Grafana dashboards + +--- + +## Task 5.2: Create Grafana Dashboards + +**Objective**: Provide pre-built Grafana dashboards for RPC performance, transaction lifecycle, and consensus health. + +**What to do**: + +- Create `docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml` (provisioning config) +- Create dashboard JSON files: + 1. **RPC Performance Dashboard** (`rpc-performance.json`): + - RPC request latency (p50/p95/p99) by command — histogram panel + - RPC throughput (requests/sec) by command — time series + - RPC error rate by command — bar gauge + - Top slowest RPC commands — table + + 2. **Transaction Overview Dashboard** (`transaction-overview.json`): + - Transaction processing rate — time series + - Transaction latency distribution — histogram + - Suppression rate (duplicates) — stat panel + - Transaction processing path (sync vs async) — pie chart + + 3. **Consensus Health Dashboard** (`consensus-health.json`): + - Consensus round duration — time series + - Phase duration breakdown (open/establish/accept) — stacked bar + - Proposals sent/received per round — stat panel + - Consensus mode distribution (proposing/observing) — pie chart + +- Store dashboards in `docker/telemetry/grafana/dashboards/` + +**Key new files**: + +- `docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml` +- `docker/telemetry/grafana/dashboards/rpc-performance.json` +- `docker/telemetry/grafana/dashboards/transaction-overview.json` +- `docker/telemetry/grafana/dashboards/consensus-health.json` + +**Reference**: + +- [07-observability-backends.md §7.6](./07-observability-backends.md) — Grafana dashboard specifications +- [01-architecture-analysis.md §1.8.3](./01-architecture-analysis.md) — Dashboard panel examples + +--- + +## Task 5.3: Define Alert Rules + +**Objective**: Create alert definitions for key telemetry anomalies. + +**What to do**: + +- Create `docker/telemetry/grafana/provisioning/alerting/alerts.yaml`: + - **RPC Latency Alert**: p99 latency > 1s for any command over 5 minutes + - **RPC Error Rate Alert**: Error rate > 5% for any command over 5 minutes + - **Consensus Duration Alert**: Round duration > 10s (warn), > 30s (critical) + - **Transaction Processing Alert**: Processing rate drops below threshold + - **Telemetry Pipeline Health**: No spans received for > 2 minutes + +**Key new files**: + +- `docker/telemetry/grafana/provisioning/alerting/alerts.yaml` + +**Reference**: + +- [07-observability-backends.md §7.6.3](./07-observability-backends.md) — Alert rule definitions + +--- + +## Task 5.4: Production Collector Configuration + +**Objective**: Create a production-ready OTel Collector configuration with tail-based sampling and resource limits. + +**What to do**: + +- Create `docker/telemetry/otel-collector-config-production.yaml`: + - Tail-based sampling policy: + - Always sample errors and slow traces + - 10% base sampling rate for normal traces + - Always sample first trace for each unique RPC command + - Resource limits: + - Memory limiter processor (80% of available memory) + - Queued retry for export failures + - TLS configuration for production endpoints + - Health check endpoint + +**Key new files**: + +- `docker/telemetry/otel-collector-config-production.yaml` + +**Reference**: + +- [05-configuration-reference.md §5.5.2](./05-configuration-reference.md) — Production collector config + +--- + +## Task 5.5: Operator Runbook + +**Objective**: Create operator documentation for managing the telemetry system in production. + +**What to do**: + +- Create `docs/telemetry-runbook.md`: + - **Setup**: How to enable telemetry in xrpld + - **Configuration**: All config options with descriptions + - **Collector Deployment**: Docker Compose vs. Kubernetes vs. bare metal + - **Troubleshooting**: Common issues and resolutions + - No traces appearing + - High memory usage from telemetry + - Collector connection failures + - Sampling configuration tuning + - **Performance Tuning**: Batch size, queue size, sampling ratio guidelines + - **Upgrading**: How to upgrade OTel SDK and Collector versions + +**Key new files**: + +- `docs/telemetry-runbook.md` + +--- + +## Task 5.6: Final Integration Testing + +**Objective**: Validate the complete telemetry stack end-to-end. + +**What to do**: + +1. Start full Docker stack (Collector, Tempo, Grafana, Prometheus) +2. Build xrpld with `telemetry=ON` +3. Run in standalone mode with telemetry enabled +4. Generate RPC traffic and verify traces in Tempo +5. Verify dashboards populate in Grafana +6. Verify alerts trigger correctly +7. Test telemetry OFF path (no regressions) +8. Run full test suite + +**Verification Checklist**: + +- [ ] Docker stack starts without errors +- [ ] Traces appear in Tempo with correct hierarchy +- [ ] Grafana dashboards show metrics derived from spans +- [ ] Prometheus scrapes spanmetrics successfully +- [ ] Alerts can be triggered by simulated conditions +- [ ] Build succeeds with telemetry ON and OFF +- [ ] Full test suite passes + +--- + +## Summary + +| Task | Description | New Files | Modified Files | Depends On | +| ---- | ---------------------------------- | --------- | -------------- | ---------- | +| 5.1 | Spanmetrics connector + Prometheus | 2 | 2 | Phase 4 | +| 5.2 | Grafana dashboards | 4 | 0 | 5.1 | +| 5.3 | Alert definitions | 1 | 0 | 5.1 | +| 5.4 | Production collector config | 1 | 0 | Phase 4 | +| 5.5 | Operator runbook | 1 | 0 | Phase 4 | +| 5.6 | Final integration testing | 0 | 0 | 5.1-5.5 | + +**Parallel work**: Tasks 5.1, 5.4, and 5.5 can run in parallel. Tasks 5.2 and 5.3 depend on 5.1. Task 5.6 depends on all others. + +**Exit Criteria** (from [06-implementation-phases.md §6.11.5](./06-implementation-phases.md)): + +- [ ] Dashboards deployed and showing data +- [ ] Alerts configured and tested +- [ ] Operator documentation complete +- [ ] Production collector config ready +- [ ] Full test suite passes diff --git a/OpenTelemetryPlan/Phase7_taskList.md b/OpenTelemetryPlan/Phase7_taskList.md new file mode 100644 index 00000000000..971372fdb0e --- /dev/null +++ b/OpenTelemetryPlan/Phase7_taskList.md @@ -0,0 +1,587 @@ +# Phase 7: Native OTel Metrics Migration — Task List + +> **Goal**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency. +> +> **Scope**: New `OTelCollectorImpl` class, `CollectorManager` config change, OTel Collector pipeline update, Grafana dashboard metric name migration, integration tests. +> +> **Branch**: `pratik/otel-phase7-native-metrics` (from `pratik/otel-phase6-statsd`) + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | --------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 7 plan: motivation, architecture, exit criteria (§6.8) | +| [02-design-decisions.md](./02-design-decisions.md) | Collector interface design, beast::insight coexistence strategy | +| [05-configuration-reference.md](./05-configuration-reference.md) | `[insight]` and `[telemetry]` config sections | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Complete metric inventory that must be preserved | + +--- + +## Task 7.1: Add OTel Metrics SDK to Build Dependencies + +**Objective**: Enable the OTel C++ Metrics SDK components in the build system. + +**What to do**: + +- Edit `conanfile.py`: + - Add OTel metrics SDK components to the dependency list when `telemetry=True` + - Components needed: `opentelemetry-cpp::metrics`, `opentelemetry-cpp::otlp_http_metric_exporter` + +- Edit `CMakeLists.txt` (telemetry section): + - Link `opentelemetry::metrics` and `opentelemetry::otlp_http_metric_exporter` targets + +**Key modified files**: + +- `conanfile.py` +- `CMakeLists.txt` (or the relevant telemetry cmake target) + +**Reference**: [05-configuration-reference.md §5.3](./05-configuration-reference.md) — CMake integration + +--- + +## Task 7.2: Implement OTelCollector Class + +**Objective**: Create the core `OTelCollector` implementation that maps beast::insight instruments to OTel Metrics SDK instruments. + +**What to do**: + +- Create `include/xrpl/beast/insight/OTelCollector.h`: + - Public factory: `static std::shared_ptr New(std::string const& endpoint, std::string const& prefix, beast::Journal journal)` + - Derives from `StatsDCollector` (or directly from `Collector` — TBD based on shared code) + +- Create `src/libxrpl/beast/insight/OTelCollector.cpp` (~400-500 lines): + - **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter`. `increment(amount)` calls `counter->Add(amount)`. + - **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge` with an async callback. `set(value)` stores value atomically; callback reads it during collection. + - **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned. + - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram`. `notify(duration)` calls `histogram->Record(duration.count())`. Uses explicit bucket boundaries matching SpanMetrics: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms. + - **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader). + - **OTelCollectorImp**: Main class. + - Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval) + - Creates `OtlpHttpMetricExporter` pointing to `[telemetry]` endpoint + - Sets resource attributes (service.name, service.instance.id) matching trace exporter + - Implements all `make_*()` factory methods + - Prefixes metric names with `[insight] prefix=` value + +- Guard all OTel SDK includes with `#ifdef XRPL_ENABLE_TELEMETRY` to compile to `NullCollector` equivalents when telemetry disabled. + +**Key new files**: + +- `include/xrpl/beast/insight/OTelCollector.h` +- `src/libxrpl/beast/insight/OTelCollector.cpp` + +**Key patterns to follow**: + +- Match `StatsDCollector.cpp` structure: private impl classes, intrusive list for metrics, strand-based thread safety +- Match existing telemetry code style from `src/libxrpl/telemetry/Telemetry.cpp` +- Use RAII for MeterProvider lifecycle (shutdown on destructor) + +**Reference**: [04-code-samples.md](./04-code-samples.md) — code style and patterns + +--- + +## Task 7.3: Update CollectorManager + +**Objective**: Add `server=otel` config option to route metric creation to the new OTel backend. + +**What to do**: + +- Edit `src/xrpld/app/main/CollectorManager.cpp`: + - In the constructor, add a third branch after `server == "statsd"`: + ```cpp + else if (server == "otel") + { + // Read endpoint from [telemetry] section + auto const endpoint = get(telemetryParams, "endpoint", + "http://localhost:4318/v1/metrics"); + std::string const& prefix(get(params, "prefix")); + collector_ = beast::insight::OTelCollector::New( + endpoint, prefix, journal); + } + ``` + - This requires access to the `[telemetry]` config section — may need to pass it as a parameter or read from Application config. + +- Edit `src/xrpld/app/main/CollectorManager.h`: + - Add `#include ` + +**Key modified files**: + +- `src/xrpld/app/main/CollectorManager.cpp` +- `src/xrpld/app/main/CollectorManager.h` + +--- + +## Task 7.4: Update OTel Collector Configuration + +**Objective**: Add a metrics pipeline to the OTLP receiver and remove the StatsD receiver dependency. + +**What to do**: + +- Edit `docker/telemetry/otel-collector-config.yaml`: + - Remove `statsd` receiver (no longer needed when `server=otel`) + - Add metrics pipeline under `service.pipelines`: + ```yaml + metrics: + receivers: [otlp, spanmetrics] + processors: [batch] + exporters: [prometheus] + ``` + - The OTLP receiver already listens on :4318 — it just needs to be added to the metrics pipeline receivers. + - Keep `spanmetrics` connector in the metrics pipeline so span-derived RED metrics continue working. + +- Edit `docker/telemetry/docker-compose.yml`: + - Remove UDP :8125 port mapping from otel-collector service + - Update xrpld service config: change `[insight] server=statsd` to `server=otel` + +**Key modified files**: + +- `docker/telemetry/otel-collector-config.yaml` +- `docker/telemetry/docker-compose.yml` + +**Note**: Keep a commented-out `statsd` receiver block for operators who need backward compatibility. + +--- + +## Task 7.5: Preserve Metric Names in Prometheus + +**Objective**: Ensure existing Grafana dashboards continue working with identical metric names. + +**What to do**: + +- In `OTelCollector.cpp`, construct OTel instrument names to match existing Prometheus metric names: + - beast::insight `make_gauge("LedgerMaster", "Validated_Ledger_Age")` → OTel instrument name: `xrpld_LedgerMaster_Validated_Ledger_Age` + - The prefix + group + name concatenation must produce the same string as `StatsDCollector`'s format + - Use underscores as separators (matching StatsD convention) + +- Verify in integration test that key Prometheus queries still return data: + - `xrpld_LedgerMaster_Validated_Ledger_Age` + - `xrpld_Peer_Finder_Active_Inbound_Peers` + - `xrpld_rpc_requests` + +**Key consideration**: OTel Prometheus exporter may normalize metric names differently than StatsD receiver. Test this early (Task 7.2) and adjust naming strategy if needed. The OTel SDK's Prometheus exporter adds `_total` suffix to counters and converts dots to underscores — match existing conventions. + +--- + +## Task 7.6: Update Grafana Dashboards + +**Objective**: Update the 3 StatsD dashboards if any metric names change due to OTLP export format differences. + +**What to do**: + +- If Task 7.5 confirms metric names are preserved exactly, no dashboard changes needed. +- If OTLP export produces different names (e.g., `_total` suffix on counters), update: + - `docker/telemetry/grafana/dashboards/statsd-node-health.json` + - `docker/telemetry/grafana/dashboards/statsd-network-traffic.json` + - `docker/telemetry/grafana/dashboards/statsd-rpc-pathfinding.json` +- Rename dashboard titles from "StatsD" to "System Metrics" or similar (since they're no longer StatsD-sourced). + +**Key modified files**: + +- `docker/telemetry/grafana/dashboards/statsd-*.json` (3 files, conditionally) + +--- + +## Task 7.7: Update Integration Tests + +**Objective**: Verify the full OTLP metrics pipeline end-to-end. + +**What to do**: + +- Edit `docker/telemetry/integration-test.sh`: + - Update test config to use `[insight] server=otel` + - Verify metrics arrive in Prometheus via OTLP (not StatsD) + - Add check that StatsD receiver is no longer required + - Preserve all existing metric presence checks + +**Key modified files**: + +- `docker/telemetry/integration-test.sh` + +--- + +## Task 7.8: Update Documentation + +**Objective**: Update all plan docs, runbook, and reference docs to reflect the migration. + +**What to do**: + +- Edit `docs/telemetry-runbook.md`: + - Update `[insight]` config examples to show `server=otel` + - Update troubleshooting section (no more StatsD UDP debugging) + +- Edit `OpenTelemetryPlan/09-data-collection-reference.md`: + - Update Data Flow Overview diagram (remove StatsD receiver) + - Update Section 2 header from "StatsD Metrics" to "System Metrics (OTel native)" + - Update config examples + +- Edit `OpenTelemetryPlan/05-configuration-reference.md`: + - Add `server=otel` option to `[insight]` section docs + +- Edit `docker/telemetry/TESTING.md`: + - Update setup instructions to use `server=otel` + +**Key modified files**: + +- `docs/telemetry-runbook.md` +- `OpenTelemetryPlan/09-data-collection-reference.md` +- `OpenTelemetryPlan/05-configuration-reference.md` +- `docker/telemetry/TESTING.md` + +--- + +## Task 7.9: ValidationTracker — Validation Agreement Computation + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — the most valuable metric from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 4 Task 4.8 (validation span attributes provide ledger hash context). +> **Downstream**: Phase 9 (Validator Health dashboard), Phase 10 (validation checks), Phase 11 (agreement alert rules). + +**Objective**: Implement a stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows with an 8-second grace period and 5-minute late repair window. + +**Architecture**: + +``` +consensus.validation.send ────> ValidationTracker ────> MetricsRegistry +(records our validation (reconciles after (exports agreement + for ledger X) 8s grace period) gauges every 10s) + +ledger.validate ──────────────> ValidationTracker +(records which ledger (marks ledger X as + network validated) agreed or missed) +``` + +**What to do**: + +- Create `src/xrpld/telemetry/ValidationTracker.h`: + - `recordOurValidation(ledgerHash, ledgerSeq)` — called when we send a validation + - `recordNetworkValidation(ledgerHash, seq)` — called when a ledger is fully validated + - `reconcile()` — called periodically; reconciles pending ledger events after 8s grace period + - Getters: `agreementPct1h()`, `agreementPct24h()`, `agreements1h()`, `missed1h()`, `agreements24h()`, `missed24h()`, `totalAgreements()`, `totalMissed()`, `totalValidationsSent()`, `totalValidationsChecked()` + - Thread-safety: atomics for counters, mutex for window deques + +- Create `src/xrpld/telemetry/detail/ValidationTracker.cpp`: + - Reconciliation logic: after 8s grace period, check if `weValidated && networkValidated && sameHash` → agreement; else missed + - Late repair: if a late validation arrives within 5 minutes, correct a false-positive miss + - Sliding window: `std::deque` evicts entries older than 1h/24h on each reconciliation pass + - Ring buffer of 1000 `LedgerEvent` structs for pending reconciliation + +- Add recording hooks (modifying Phase 4 code from Phase 7 branch): + - `RCLConsensus.cpp` `validate()`: call `tracker.recordOurValidation()` + - `LedgerMaster.cpp` fully-validated path: call `tracker.recordNetworkValidation()` + +**Key data structures**: + +```cpp +struct LedgerEvent { + uint256 ledgerHash; + LedgerIndex seq; + TimePoint closeTime; + bool weValidated = false; + bool networkValidated = false; + bool reconciled = false; + bool agreed = false; +}; + +struct WindowEvent { + TimePoint time; + bool agreed; +}; +``` + +**Key new files**: + +- `src/xrpld/telemetry/ValidationTracker.h` +- `src/xrpld/telemetry/detail/ValidationTracker.cpp` + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker) +- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks) +- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook) + +**Exit Criteria**: + +- [ ] ValidationTracker correctly tracks agreement with 8s grace period +- [ ] 5-minute late repair corrects false-positive misses +- [ ] Thread-safe (atomics + mutex for window deques) +- [ ] Rolling windows correctly evict stale entries +- [ ] Unit tests: normal agreement, missed validation, late repair, window eviction + +--- + +## Task 7.10: Validator Health Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export amendment blocked, UNL health, and quorum data as a native OTel observable gauge. + +**What to do**: + +- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add: + +```cpp +validatorHealthGauge_ = meter_->CreateDoubleObservableGauge( + "xrpld_validator_health", "Validator health indicators"); +``` + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| ------------------- | ------ | ------------------------------------------------- | +| `amendment_blocked` | int64 | `app_.getOPs().isAmendmentBlocked()` → 0/1 | +| `unl_blocked` | int64 | `app_.getOPs().isUNLBlocked()` → 0/1 | +| `unl_expiry_days` | double | `app_.validators().expires()` → days until expiry | +| `validation_quorum` | int64 | `app_.validators().quorum()` | + +### Sub-task 7.10a: Per-Validator Validation Count (Flag Ledger Window) + +**Objective**: Track how many ledgers each UNL validator has validated over +the last 256 consecutive ledgers (one flag ledger window). This is the key +UNL participation metric — validators consistently below threshold may be +candidates for removal from the UNL. + +**What to do**: + +- Add a new observable gauge: + +```cpp +validatorParticipationGauge_ = meter_->CreateInt64ObservableGauge( + "xrpld_validator_participation", + "Per-validator validation count over the last 256 ledgers"); +``` + +- The callback queries `app_.getValidations()` to get the trusted + validation set for each of the last 256 ledger hashes (from + `LedgerMaster::getValidatedLedger()` walking backwards). For each + validator public key in the UNL, count how many of those 256 ledgers + have a matching validation. + +- **Label dimensions**: + - `validator` — base58-encoded validator master public key + - `service_instance_id` — this node's identity (standard) + +- **Emission**: every flag ledger (256 ledgers, ~15 minutes) or on a + 10-second async gauge callback with cached results (recompute only + at flag ledger boundaries). + +- **Data source**: `RCLValidations::getTrustedForLedger(hash, seq)` returns + `std::vector>` with `getSignerPublic()` + for each. The UNL list is from `app_.getValidators().getTrustedMasterKeys()`. + +- **Dashboard panel**: Add a table panel to the Validator Health dashboard + showing `xrpld_validator_participation` grouped by `validator` label, + with a threshold color (green >= 240, yellow >= 200, red < 200). + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] Gauge emits one time series per UNL validator +- [ ] Values range 0-256 and update at flag ledger boundaries +- [ ] Grafana table panel shows per-validator participation +- [ ] Validators below 75% participation are highlighted in red + +--- + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] All 4 base label values emitted every 10s +- [ ] `unl_expiry_days` is negative when expired, positive when active +- [ ] Per-validator participation gauge emits at flag ledger boundaries +- [ ] Values visible in Prometheus + +--- + +## Task 7.11: Peer Quality Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export peer health aggregates (latency P90, insane peers, version awareness) as a native OTel observable gauge. + +**What to do**: + +- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add a callback that iterates `app_.overlay().foreach(...)` to: + - Collect per-peer latency values, sort, compute P90 + - Count peers with `tracking_ == diverged` (insane) + - Compare peer `getVersion()` to own version for upgrade awareness + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| -------------------------- | ------ | ------------------------------------- | +| `peer_latency_p90_ms` | double | P90 from sorted peer latencies | +| `peers_insane_count` | int64 | Peers with diverged tracking status | +| `peers_higher_version_pct` | double | % of peers on newer xrpld version | +| `upgrade_recommended` | int64 | 1 if `peers_higher_version_pct > 60%` | + +**Implementation note**: The callback runs every 10s on the metrics reader thread. Iterating ~50-200 peers is acceptable overhead. + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] P90 latency computed correctly +- [ ] Insane count matches `peers` RPC output +- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1") + +--- + +## Task 7.12: Ledger Economy Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export fee, reserve, ledger age, and transaction rate as a native OTel observable gauge. + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| -------------------- | ------ | --------------------------------------------------- | +| `base_fee_xrp` | double | Base fee from validated ledger fee settings (drops) | +| `reserve_base_xrp` | double | Account reserve from validated ledger (drops) | +| `reserve_inc_xrp` | double | Owner reserve increment (drops) | +| `ledger_age_seconds` | double | `now - lastValidatedCloseTime` | +| `transaction_rate` | double | Derived: tx count delta / time delta (smoothed) | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] Fee values match `server_info` RPC output +- [ ] `ledger_age_seconds` increases monotonically between ledger closes +- [ ] `transaction_rate` is smoothed (rolling average) + +--- + +## Task 7.13: State Tracking Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export extended state value (0-6 encoding combining OperatingMode + ConsensusMode) and time-in-current-state. + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| ------------------------------- | ------ | ----------------------------------------------- | +| `state_value` | int64 | 0-6 encoding (see spec for mapping) | +| `time_in_current_state_seconds` | double | `now - lastModeChangeTime` from StateAccounting | + +**State value encoding**: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (full + validating), 6=proposing (full + proposing). + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] `state_value` correctly combines OperatingMode and ConsensusMode +- [ ] `time_in_current_state_seconds` resets on mode change + +--- + +## Task 7.14: Storage Detail and Sync Info Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export NuDB-specific storage size and initial sync duration. + +**Gauge label values**: + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | ------------------------------- | ------ | ----------------------------- | +| `xrpld_storage_detail` | `nudb_bytes` | int64 | NuDB backend file size | +| `xrpld_sync_info` | `initial_sync_duration_seconds` | double | Time from start to first FULL | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] NuDB file size reported in bytes (0 if NuDB not configured) +- [ ] Sync duration captured once and remains stable after reaching FULL + +--- + +## Task 7.15: New Synchronous Counters + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Add 7 new event counters incremented at their respective instrumentation sites. + +| Counter Name | Increment Site | Source File | +| ----------------------------------- | -------------------------------- | --------------------- | +| `xrpld_ledgers_closed_total` | `onAccept()` in consensus | RCLConsensus.cpp | +| `xrpld_validations_sent_total` | `validate()` in consensus | RCLConsensus.cpp | +| `xrpld_validations_checked_total` | Network validation received | LedgerMaster.cpp | +| `xrpld_validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `xrpld_validation_missed_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `xrpld_state_changes_total` | `setMode()` in NetworkOPs | NetworkOPs.cpp | +| `xrpld_jq_trans_overflow_total` | Job queue overflow path | JobQueue.cpp | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (declarations), plus recording sites in RCLConsensus.cpp, LedgerMaster.cpp, NetworkOPs.cpp, JobQueue.cpp + +**Exit Criteria**: + +- [ ] All 7 counters monotonically increase during normal operation +- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s) + +--- + +## Task 7.16: Validation Agreement Observable Gauge + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export rolling window agreement stats from `ValidationTracker` (Task 7.9). + +**Gauge label values**: + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------------- | ------------------- | ------ | --------------------------- | +| `xrpld_validation_agreement` | `agreement_pct_1h` | double | `tracker.agreementPct1h()` | +| | `agreements_1h` | int64 | `tracker.agreements1h()` | +| | `missed_1h` | int64 | `tracker.missed1h()` | +| | `agreement_pct_24h` | double | `tracker.agreementPct24h()` | +| | `agreements_24h` | int64 | `tracker.agreements24h()` | +| | `missed_24h` | int64 | `tracker.missed24h()` | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] Agreement percentages in range [0.0, 100.0] +- [ ] Window stats stabilize after 1h/24h of operation + +--- + +## Summary Table + +| Task | Description | New Files | Modified Files | Depends On | +| ---- | -------------------------------------- | --------- | -------------- | ---------- | +| 7.1 | Add OTel Metrics SDK to build deps | 0 | 2 | — | +| 7.2 | Implement OTelCollector class | 2 | 0 | 7.1 | +| 7.3 | Update CollectorManager config routing | 0 | 2 | 7.2 | +| 7.4 | Update OTel Collector YAML and Docker | 0 | 2 | 7.3 | +| 7.5 | Preserve metric names in Prometheus | 0 | 1 | 7.2 | +| 7.6 | Update Grafana dashboards (if needed) | 0 | 3 | 7.5 | +| 7.7 | Update integration tests | 0 | 1 | 7.4 | +| 7.8 | Update documentation | 0 | 4 | 7.6 | +| 7.9 | ValidationTracker (agreement tracking) | 2 | 4 | 7.2, P4.8 | +| 7.10 | Validator health observable gauges | 0 | 2 | 7.2 | +| 7.11 | Peer quality observable gauges | 0 | 2 | 7.2 | +| 7.12 | Ledger economy observable gauges | 0 | 2 | 7.2 | +| 7.13 | State tracking observable gauges | 0 | 2 | 7.2 | +| 7.14 | Storage detail and sync info gauges | 0 | 2 | 7.2 | +| 7.15 | New synchronous counters | 0 | 6 | 7.2 | +| 7.16 | Validation agreement observable gauge | 0 | 1 | 7.9 | + +**Parallel work**: Tasks 7.4 and 7.5 can run in parallel after 7.2/7.3 complete. Task 7.6 depends on 7.5's findings. Tasks 7.7 and 7.8 can run in parallel after 7.6. Tasks 7.10-7.14 can all run in parallel after 7.2. Task 7.15 depends on 7.2. Task 7.16 depends on 7.9. Task 7.9 depends on 7.2 and Phase 4 Task 4.8. + +**Exit Criteria** (from [06-implementation-phases.md §6.8](./06-implementation-phases.md)): + +- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) +- [ ] `server=otel` is the default in development docker-compose +- [ ] `server=statsd` still works as a fallback +- [ ] Existing Grafana dashboards display data correctly +- [ ] Integration test passes with OTLP-only metrics pipeline +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) +- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant — Meter mapped to OTel Counter +- [ ] ValidationTracker agreement % stabilizes after 1h under normal consensus +- [ ] All new gauges and counters visible in Prometheus with non-zero values diff --git a/OpenTelemetryPlan/Phase8_taskList.md b/OpenTelemetryPlan/Phase8_taskList.md new file mode 100644 index 00000000000..78499908582 --- /dev/null +++ b/OpenTelemetryPlan/Phase8_taskList.md @@ -0,0 +1,239 @@ +# Phase 8: Log-Trace Correlation and Centralized Log Ingestion — Task List + +> **Goal**: Inject trace context (trace_id, span_id) into xrpld's Journal log output for log-trace correlation, and add OTel Collector filelog receiver to ingest logs into Grafana Loki for unified observability. +> +> **Scope**: Two independent sub-phases — 8a (code change: trace_id in logs) and 8b (infra only: filelog receiver to Loki). No changes to the `beast::Journal` public API. +> +> **Branch**: `pratik/otel-phase8-log-correlation` (from `pratik/otel-phase7-native-metrics`) + +### Related Plan Documents + +| Document | Relevance | +| ---------------------------------------------------------------- | -------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 8 plan: motivation, architecture, exit criteria (§6.8.1) | +| [07-observability-backends.md](./07-observability-backends.md) | Loki backend recommendation, Grafana data source provisioning | +| [Phase7_taskList.md](./Phase7_taskList.md) | Prerequisite — native OTel metrics pipeline must be working | +| [05-configuration-reference.md](./05-configuration-reference.md) | `[telemetry]` config (trace_id injection toggle) | + +--- + +## Task 8.1: Inject trace_id into Logs::format() + +**Objective**: Add OTel trace context to every log line that is emitted within an active span. + +**What to do**: + +- Edit `src/libxrpl/basics/Log.cpp`: + - In `Logs::format()` (around line 346), after severity is appended, check for active OTel span. The implementation checks the context value directly to avoid the heap allocation that `GetSpan()` performs on the no-span path: + ```cpp + #ifdef XRPL_ENABLE_TELEMETRY + { + auto context = opentelemetry::context::RuntimeContext::GetCurrent(); + auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey); + if (opentelemetry::nostd::holds_alternative< + opentelemetry::nostd::shared_ptr>(spanValue)) + { + auto span = opentelemetry::nostd::get< + opentelemetry::nostd::shared_ptr>(spanValue); + auto spanCtx = span->GetContext(); + if (spanCtx.IsValid()) + { + char traceId[32], spanId[16]; + spanCtx.trace_id().ToLowerBase16( + opentelemetry::nostd::span{traceId}); + spanCtx.span_id().ToLowerBase16( + opentelemetry::nostd::span{spanId}); + output += "trace_id="; + output.append(traceId, 32); + output += " span_id="; + output.append(spanId, 16); + output += ' '; + } + } + } + #endif + ``` + - Add `#include` for OTel context headers, guarded by `#ifdef XRPL_ENABLE_TELEMETRY` + +- Edit `include/xrpl/basics/Log.h`: + - No changes needed — format() signature unchanged + +**Key modified files**: + +- `src/libxrpl/basics/Log.cpp` + +**Performance note**: The implementation checks the thread-local context value directly (avoiding the heap allocation that `GetSpan()` performs on the no-span path). On threads without an active span (~99% of log lines), the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, an additional shared_ptr copy + `GetContext()` + `IsValid()` adds ~50ns total. Overhead is negligible at typical logging rates. + +--- + +## Task 8.2: Add Loki to Docker Compose Stack + +**Objective**: Add Grafana Loki as a log storage backend in the development observability stack. + +**What to do**: + +- Edit `docker/telemetry/docker-compose.yml`: + - Add Loki service: + ```yaml + loki: + image: grafana/loki:3.4.2 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + ``` + - Add Loki as a Grafana data source in provisioning + +- Create `docker/telemetry/grafana/provisioning/datasources/loki.yaml`: + - Configure Loki data source with derived fields linking `trace_id` to Tempo + +**Key new files**: + +- `docker/telemetry/grafana/provisioning/datasources/loki.yaml` + +**Key modified files**: + +- `docker/telemetry/docker-compose.yml` + +--- + +## Task 8.3: Add Filelog Receiver to OTel Collector + +**Objective**: Configure the OTel Collector to tail xrpld's log file and export to Loki. + +**What to do**: + +- Edit `docker/telemetry/otel-collector-config.yaml`: + - Add `filelog` receiver: + ```yaml + receivers: + filelog: + include: [/var/log/xrpld/*/debug.log] + operators: + - type: regex_parser + regex: '^(?P\S+)\s+(?P\S+):(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' + timestamp: + parse_from: attributes.timestamp + layout: "%Y-%m-%dT%H:%M:%S.%fZ" + ``` + - Add logs pipeline: + ```yaml + service: + pipelines: + logs: + receivers: [filelog] + processors: [batch] + exporters: [otlp/loki] + ``` + - Add Loki exporter: + ```yaml + exporters: + otlphttp/loki: + endpoint: http://loki:3100/otlp + ``` + +- Mount xrpld's log directory into the collector container via docker-compose volume + +**Key modified files**: + +- `docker/telemetry/otel-collector-config.yaml` +- `docker/telemetry/docker-compose.yml` + +--- + +## Task 8.4: Configure Grafana Trace-to-Log Correlation + +**Objective**: Enable one-click navigation from Tempo traces to Loki logs in Grafana. + +**What to do**: + +- Edit Grafana Tempo data source provisioning to add `tracesToLogs` configuration: + + ```yaml + tracesToLogs: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + tags: ["partition", "severity"] + ``` + +- Edit Grafana Loki data source provisioning to add `derivedFields` linking trace_id back to Tempo: + ```yaml + derivedFields: + - datasourceUid: tempo + matcherRegex: "trace_id=(\\w+)" + name: TraceID + url: "$${__value.raw}" + ``` + +**Key modified files**: + +- `docker/telemetry/grafana/provisioning/datasources/loki.yaml` +- `docker/telemetry/grafana/provisioning/datasources/` (Tempo data source file) + +--- + +## Task 8.5: Update Integration Tests + +**Objective**: Verify trace_id appears in logs and Loki correlation works. + +**What to do**: + +- Edit `docker/telemetry/integration-test.sh`: + - After sending RPC requests (which create spans), grep xrpld's log output for `trace_id=` + - Verify trace_id matches a trace visible in Tempo + - Optionally: query Loki via API to confirm log ingestion + +**Key modified files**: + +- `docker/telemetry/integration-test.sh` + +--- + +## Task 8.6: Update Documentation + +**Objective**: Document the log correlation feature in runbook and reference docs. + +**What to do**: + +- Edit `docs/telemetry-runbook.md`: + - Add "Log-Trace Correlation" section explaining how to use Grafana Tempo -> Loki linking + - Add LogQL query examples for filtering by trace_id + +- Edit `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add new section "3. Log Correlation" between SpanMetrics and StatsD sections + - Document the log format with trace_id injection + - Document Loki as a new backend + +- Edit `docker/telemetry/TESTING.md`: + - Add log correlation verification steps + +**Key modified files**: + +- `docs/telemetry-runbook.md` +- `OpenTelemetryPlan/09-data-collection-reference.md` +- `docker/telemetry/TESTING.md` + +--- + +## Summary Table + +| Task | Description | Sub-Phase | New Files | Modified Files | Depends On | +| ---- | ------------------------------------------ | --------- | --------- | -------------- | ---------- | +| 8.1 | Inject trace_id into Logs::format() | 8a | 0 | 1 | Phase 7 | +| 8.2 | Add Loki to Docker Compose stack | 8b | 1 | 1 | -- | +| 8.3 | Add filelog receiver to OTel Collector | 8b | 0 | 2 | 8.1, 8.2 | +| 8.4 | Configure Grafana trace-to-log correlation | 8b | 0 | 2 | 8.3 | +| 8.5 | Update integration tests | 8a + 8b | 0 | 1 | 8.4 | +| 8.6 | Update documentation | 8a + 8b | 0 | 3 | 8.5 | + +**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3-8.6 are sequential. + +**Exit Criteria** (from [06-implementation-phases.md §6.8.1](./06-implementation-phases.md)): + +- [ ] Log lines within active spans contain `trace_id= span_id=` +- [ ] Log lines outside spans have no trace context (no empty fields) +- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver +- [ ] Grafana Tempo -> Loki one-click correlation works +- [ ] Grafana Loki -> Tempo reverse lookup works via derived field +- [ ] Integration test verifies trace_id presence in logs +- [ ] No performance regression from trace_id injection (< 0.1% overhead) diff --git a/OpenTelemetryPlan/Phase9_taskList.md b/OpenTelemetryPlan/Phase9_taskList.md new file mode 100644 index 00000000000..74babaef8f6 --- /dev/null +++ b/OpenTelemetryPlan/Phase9_taskList.md @@ -0,0 +1,549 @@ +# Phase 9: Internal Metric Instrumentation Gap Fill — Task List + +> **Status**: Future Enhancement +> +> **Goal**: Instrument xrpld to emit ~50+ metrics that exist in `get_counts`/`server_info`/TxQ/PerfLog but currently lack time-series export via the OTel or beast::insight pipelines. +> +> **Scope**: Hybrid approach — extend `beast::insight` for metrics near existing registrations, use OTel Metrics SDK `ObservableGauge` callbacks for new categories (TxQ, PerfLog, CountedObjects). +> +> **Branch**: `pratik/otel-phase9-metric-gap-fill` (from `pratik/otel-phase8-log-correlation`) +> +> **Depends on**: Phase 7 (native OTel metrics pipeline) and Phase 8 (log-trace correlation) + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | -------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 9 plan: motivation, architecture, exit criteria (§6.8.2) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Current metric inventory + future metrics section | +| [Phase7_taskList.md](./Phase7_taskList.md) | Prerequisite — OTel Metrics SDK and `OTelCollector` class | +| [Phase8_taskList.md](./Phase8_taskList.md) | Prerequisite — log-trace correlation | + +### Third-Party Consumer Context + +These metrics serve multiple external consumer categories identified during research: + +| Consumer Category | Key Metrics They Need | +| ------------------------- | --------------------------------------------------------------- | +| **Exchanges** | Fee escalation levels, TxQ depth, settlement latency | +| **Payment Processors** | Load factors, io_latency, transaction throughput | +| **Analytics Providers** | NodeStore I/O, cache hit rates, counted objects | +| **Validators/Operators** | Per-job execution times, PerfLog RPC counters, consensus timing | +| **Academic Researchers** | Consensus performance time-series, fee market dynamics | +| **Institutional Custody** | Server health scores, reserve calculations, node availability | + +--- + +## Task 9.1: NodeStore I/O Metrics + +**Objective**: Export node store read/write performance as time-series metrics. + +**What to do**: + +- In `src/libxrpl/nodestore/Database.cpp`, extend existing `beast::insight` registrations to add: + - Gauge: `node_reads_total` (cumulative read operations) + - Gauge: `node_reads_hit` (cache-served reads) + - Gauge: `node_writes` (cumulative write operations) + - Gauge: `node_written_bytes` (cumulative bytes written) + - Gauge: `node_read_bytes` (cumulative bytes read) + - Gauge: `node_reads_duration_us` (cumulative read time in microseconds) + - Gauge: `write_load` (current write load score) + - Gauge: `read_queue` (items in read queue) + +- These values are already computed in `Database::getCountsJson()` (line ~236). Wire the same counters to `beast::insight` hooks. + +**Key modified files**: + +- `src/libxrpl/nodestore/Database.cpp` +- `src/libxrpl/nodestore/Database.h` (add insight members) + +**Derived Prometheus metrics**: `xrpld_nodestore_reads_total`, `xrpld_nodestore_reads_hit`, `xrpld_nodestore_write_load`, etc. + +**Grafana dashboard**: Add "NodeStore I/O" panel group to _Node Health_ dashboard. + +--- + +## Task 9.2: Cache Hit Rate Metrics + +**Objective**: Export SHAMap and ledger cache performance as time-series gauges. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks (via Phase 7's `OTelCollector`) for: + - `SLE_hit_rate` — SLE cache hit rate (0.0–1.0) + - `ledger_hit_rate` — Ledger object cache hit rate + - `AL_hit_rate` — AcceptedLedger cache hit rate + - `treenode_cache_size` — SHAMap TreeNode cache size (entries) + - `treenode_track_size` — Tracked tree nodes + - `fullbelow_size` — FullBelow cache size + +- The callback should read from the same sources as `GetCounts.cpp` handler (line ~43). + +- Create a centralized `MetricsRegistry` class that holds all OTel async gauge registrations, polled at 10-second intervals by the `PeriodicMetricReader`. + +**Key modified files**: + +- New: `src/xrpld/telemetry/MetricsRegistry.h` / `.cpp` +- `src/xrpld/rpc/handlers/GetCounts.cpp` (extract shared access methods) +- `src/xrpld/app/main/Application.cpp` (register MetricsRegistry at startup) + +**Derived Prometheus metrics**: `xrpld_cache_SLE_hit_rate`, `xrpld_cache_ledger_hit_rate`, `xrpld_cache_treenode_size`, etc. + +--- + +## Task 9.3: Transaction Queue (TxQ) Metrics + +**Objective**: Export TxQ depth, capacity, and fee escalation levels as time-series. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks for TxQ state (from `TxQ.h` line ~143): + - `txq_count` — Current transactions in queue + - `txq_max_size` — Maximum queue capacity + - `txq_in_ledger` — Transactions in current open ledger + - `txq_per_ledger` — Expected transactions per ledger + - `txq_reference_fee_level` — Reference fee level + - `txq_min_processing_fee_level` — Minimum fee to get processed + - `txq_med_fee_level` — Median fee level in queue + - `txq_open_ledger_fee_level` — Open ledger fee escalation level + +- Add to the `MetricsRegistry` (Task 9.2). + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add TxQ callbacks) +- `src/xrpld/app/tx/detail/TxQ.h` (expose metrics accessor if needed) + +**Derived Prometheus metrics**: `xrpld_txq_count`, `xrpld_txq_max_size`, `xrpld_txq_open_ledger_fee_level`, etc. + +**Grafana dashboard**: New _Fee Market & TxQ_ dashboard (`fee-market`). + +--- + +## Task 9.4: PerfLog Per-RPC Method Metrics + +**Objective**: Export per-RPC-method call counts and latency as OTel metrics. + +**What to do**: + +- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp` line ~63): + - Counter: `xrpld_rpc_method_started_total{method=""}` — calls started + - Counter: `xrpld_rpc_method_finished_total{method=""}` — calls completed + - Counter: `xrpld_rpc_method_errored_total{method=""}` — calls errored + - Histogram: `xrpld_rpc_method_duration_us{method=""}` — execution time distribution + +- Use OTel `Counter` and `Histogram` instruments with `method` attribute label. + +- Hook into the existing PerfLog callback mechanism rather than adding new instrumentation points. + +**Key modified files**: + +- `src/xrpld/perflog/detail/PerfLogImp.cpp` (add OTel instrument updates alongside existing JSON counters) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (register instruments) + +**Derived Prometheus metrics**: `xrpld_rpc_method_started_total{method="server_info"}`, `xrpld_rpc_method_duration_us_bucket{method="ledger"}`, etc. + +**Grafana dashboard**: Add "Per-Method RPC Breakdown" panel group to _RPC Performance_ dashboard. + +--- + +## Task 9.5: PerfLog Per-Job-Type Metrics + +**Objective**: Export per-job-type queue and execution metrics. + +**What to do**: + +- Register OTel instruments for PerfLog job counters: + - Counter: `xrpld_job_queued_total{job_type=""}` — jobs queued + - Counter: `xrpld_job_started_total{job_type=""}` — jobs started + - Counter: `xrpld_job_finished_total{job_type=""}` — jobs completed + - Histogram: `xrpld_job_queued_duration_us{job_type=""}` — time spent waiting in queue + - Histogram: `xrpld_job_running_duration_us{job_type=""}` — execution time distribution + +- Hook into PerfLog's existing job tracking alongside Task 9.4. + +**Key modified files**: + +- `src/xrpld/perflog/detail/PerfLogImp.cpp` +- `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Derived Prometheus metrics**: `xrpld_job_queued_total{job_type="ledgerData"}`, `xrpld_job_running_duration_us_bucket{job_type="transaction"}`, etc. + +**Grafana dashboard**: New _Job Queue Analysis_ dashboard (`job-queue`). + +--- + +## Task 9.6: Counted Object Instance Metrics + +**Objective**: Export live instance counts for key internal object types. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks for `CountedObject` instance counts: + - `xrpld_object_count{type="Transaction"}` — live Transaction objects + - `xrpld_object_count{type="Ledger"}` — live Ledger objects + - `xrpld_object_count{type="NodeObject"}` — live NodeObject instances + - `xrpld_object_count{type="STTx"}` — serialized transaction objects + - `xrpld_object_count{type="STLedgerEntry"}` — serialized ledger entries + - `xrpld_object_count{type="InboundLedger"}` — ledgers being fetched + - `xrpld_object_count{type="Pathfinder"}` — active pathfinding computations + - `xrpld_object_count{type="PathRequest"}` — active path requests + - `xrpld_object_count{type="HashRouterEntry"}` — hash router entries + +- The `CountedObject` template already tracks these via atomic counters. The callback just reads the current counts. + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add counted object callbacks) +- `include/xrpl/basics/CountedObject.h` (may need static accessor for iteration) + +**Derived Prometheus metrics**: `xrpld_object_count{type="Transaction"}`, `xrpld_object_count{type="NodeObject"}`, etc. + +**Grafana dashboard**: Add "Object Instance Counts" panel to _Node Health_ dashboard. + +--- + +## Task 9.7: Fee Escalation & Load Factor Metrics + +**Objective**: Export the full load factor breakdown as time-series. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks for load factors (from `NetworkOPs.cpp` line ~2694): + - `load_factor` — combined transaction cost multiplier + - `load_factor_server` — server + cluster + network contribution + - `load_factor_local` — local server load only + - `load_factor_net` — network-wide load estimate + - `load_factor_cluster` — cluster peer load + - `load_factor_fee_escalation` — open ledger fee escalation + - `load_factor_fee_queue` — queue entry fee level + +- These overlap with some existing StatsD metrics but provide finer granularity (individual factor breakdown vs. combined value). + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.cpp` +- `src/xrpld/app/misc/NetworkOPs.cpp` (expose load factor accessors if needed) + +**Derived Prometheus metrics**: `xrpld_load_factor`, `xrpld_load_factor_fee_escalation`, etc. + +**Grafana dashboard**: Add "Load Factor Breakdown" panel to _Fee Market & TxQ_ dashboard. + +--- + +## Task 9.7a: push_metrics.py Parity — Missing Observable Gauges + +**Objective**: Fill the remaining metric gaps between the external `push_metrics.py` script (in `ripplex-ansible`) and the internal OTel `MetricsRegistry` observable gauges. After this task, all metrics collected by `push_metrics.py` that CAN be collected internally are covered. + +**What was done**: + +- Extended existing `cacheHitRateGauge_` callback with `AL_size` (AcceptedLedger cache size) +- Extended existing `nodeStoreGauge_` callback with 4 new metrics from `getCountsJson()`: + - `node_reads_duration_us` (JSON string — uses `std::stoll(asString())`) + - `read_request_bundle` (native JSON int) + - `read_threads_running` (native JSON int) + - `read_threads_total` (native JSON int) +- Added new `xrpld_server_info` Int64ObservableGauge with 8 metrics: + - `server_state` — operating mode as int (0=DISCONNECTED .. 4=FULL) + - `uptime` — seconds since server start + - `peers` — total peer count + - `validated_ledger_seq` — validated ledger sequence (atomic read) + - `ledger_current_index` — current open ledger sequence + - `peer_disconnects_resources` — cumulative resource-related disconnects + - `last_close_proposers` — from `getConsensusInfo()["previous_proposers"]` + - `last_close_converge_time_ms` — from `getConsensusInfo()["previous_mseconds"]` +- Added new `xrpld_build_info` Int64ObservableGauge (info-style, value=1 with `version` label) +- Added new `xrpld_complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs +- Added new `xrpld_db_metrics` Int64ObservableGauge with 4 metrics: + - `db_kb_total`, `db_kb_ledger`, `db_kb_transaction` (SQLite stat queries) + - `historical_perminute` (historical ledger fetch rate) + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h` (4 new gauge members, updated ASCII diagram) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (4 new callback registrations, 2 callback extensions) + +**Not implementable inside xrpld**: + +- `connection_count_51233/51234` — OS-level port connection counts from external shell script (`get_connection.sh`) + +**Derived Prometheus metrics**: `xrpld_server_info{metric="server_state"}`, `xrpld_build_info{version="2.4.0"}`, `xrpld_complete_ledgers{bound="start",index="0"}`, `xrpld_db_metrics{metric="db_kb_total"}`, etc. + +**Grafana dashboard**: New panels added to _Node Health_ dashboard (`node-health.json`). + +--- + +## Task 9.8: New Grafana Dashboards + +**Objective**: Create Grafana dashboards for the new metric categories. + +**What to do**: + +- Create 2 new dashboards: + 1. **Fee Market & TxQ** (`fee-market`) — TxQ depth/capacity, fee levels, load factor breakdown, fee escalation timeline + 2. **Job Queue Analysis** (`job-queue`) — Per-job-type rates, queue wait times, execution times, job queue depth + +- Update 2 existing dashboards: + 1. **Node Health** (`xrpld-statsd-node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts + 2. **RPC Performance** (`rpc-performance`) — Add per-method RPC breakdown panels + +**Key modified files**: + +- New: `docker/telemetry/grafana/dashboards/rippled-fee-market.json` +- New: `docker/telemetry/grafana/dashboards/rippled-job-queue.json` +- `docker/telemetry/grafana/dashboards/rippled-statsd-node-health.json` +- `docker/telemetry/grafana/dashboards/rippled-rpc-perf.json` + +--- + +## Task 9.9: Update Documentation + +**Objective**: Update telemetry reference docs with all new metrics. + +**What to do**: + +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors) + - Update Grafana dashboard reference table (add 2 new dashboards) + - Add Prometheus query examples for new metrics + +- Update `docs/telemetry-runbook.md`: + - Add an Alerting section covering the provisioned rules and how to wire a receiver + - Add troubleshooting entries for new metric categories + +- Provision Grafana alert rules (`docker/telemetry/grafana/provisioning/alerting/`): + - 6 rules in 3 groups — consensus/ledger (`LedgerHistoryMismatch`, `LedgerCloseStalled`), validator (`ValidationsMissed`, `ValidationsNotChecked`), job queue (`JobQueueTxOverflow`, `JobQueueLatencyHigh`) + - `xrpld-default` webhook contact point + flat notification policy; auto-loaded via the existing `provisioning/` mount (no docker-compose change) + - Alerting operator docs (per-alert meaning, tuning, receiver wiring) now live in the Alerting section of `docs/telemetry-runbook.md` + +**Key modified files**: + +- `OpenTelemetryPlan/09-data-collection-reference.md` +- `docs/telemetry-runbook.md` +- `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml` (new) +- `docs/telemetry-runbook.md` (Alerting section added) + +--- + +## Task 9.10: Integration Tests + +**Objective**: Verify all new metrics appear in Prometheus after a test workload. + +**What to do**: + +- Extend the existing telemetry integration test: + - Start xrpld with `[telemetry] enabled=1` and `[insight] server=otel` + - Submit a batch of RPC calls and transactions + - Query Prometheus for each new metric family + - Assert non-zero values for: NodeStore reads, cache hit rates, TxQ count, PerfLog RPC counters, object counts, load factors + +- Add unit tests for the `MetricsRegistry` class: + - Verify callback registration and deregistration + - Verify metric values match `get_counts` JSON output + - Verify graceful behavior when telemetry is disabled + +**Key modified files**: + +- `src/test/telemetry/MetricsRegistry_test.cpp` (new) +- Existing integration test script (extend assertions) + +--- + +## Task 9.11: Validator Health Dashboard (External Dashboard Parity) + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — dashboards for Phase 7 metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics must be emitting). +> **Downstream**: Phase 10 (dashboard load checks), Phase 11 (alert rules reference these panels). + +**Objective**: Create a Grafana dashboard for validation agreement, amendment/UNL health, and state tracking. + +**Dashboard**: `validator-health.json` + +| Panel | Type | PromQL | +| -------------------------- | ---------- | -------------------------------------------------------------- | +| Agreement % (1h) | stat | `xrpld_validation_agreement{metric="agreement_pct_1h"}` | +| Agreement % (24h) | stat | `xrpld_validation_agreement{metric="agreement_pct_24h"}` | +| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | +| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | +| Validation Rate | stat | `rate(xrpld_validations_sent_total[5m]) * 60` | +| Validations Checked Rate | stat | `rate(xrpld_validations_checked_total[5m]) * 60` | +| Amendment Blocked | stat | `xrpld_validator_health{metric="amendment_blocked"}` | +| UNL Expiry (days) | stat | `xrpld_validator_health{metric="unl_expiry_days"}` | +| Validation Quorum | stat | `xrpld_validator_health{metric="validation_quorum"}` | +| State Value Timeline | timeseries | `xrpld_state_tracking{metric="state_value"}` | +| Time in Current State | stat | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` | +| State Changes Rate | stat | `rate(xrpld_state_changes_total[1h])` | +| Ledgers Closed Rate | stat | `rate(xrpld_ledgers_closed_total[5m]) * 60` | + +**Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes. + +**Key new files**: `docker/telemetry/grafana/dashboards/rippled-validator-health.json` + +**Exit Criteria**: + +- [ ] All 13 panels render with non-zero data during normal operation +- [ ] `$node` filter works correctly for multi-node deployments +- [ ] Amendment blocked and UNL expiry panels use color thresholds (red=blocked/expiring) + +--- + +## Task 9.12: Peer Quality Dashboard (External Dashboard Parity) + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Create a Grafana dashboard for peer health aggregates. + +**Dashboard**: `peer-quality.json` + +| Panel | Type | PromQL | +| ---------------------- | ---------- | -------------------------------------------------------------- | +| P90 Peer Latency | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}` | +| Insane/Diverged Peers | stat | `xrpld_peer_quality{metric="peers_insane_count"}` | +| Higher Version Peers % | stat | `xrpld_peer_quality{metric="peers_higher_version_pct"}` | +| Upgrade Recommended | stat | `xrpld_peer_quality{metric="upgrade_recommended"}` | +| Resource Disconnects | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges` | +| Inbound vs Outbound | bargauge | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` | + +**Key new files**: `docker/telemetry/grafana/dashboards/rippled-peer-quality.json` + +**Exit Criteria**: + +- [ ] All 6 panels render correctly +- [ ] P90 latency panel shows trend over time +- [ ] Upgrade recommended panel uses color threshold (red=1, green=0) + +--- + +## Task 9.13: Ledger Economy Dashboard Panels (External Dashboard Parity) + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Add "Ledger Economy" row to the existing `node-health.json` dashboard. + +| Panel | Type | PromQL | +| -------------------- | ---------- | --------------------------------------------------- | +| Base Fee (drops) | stat | `xrpld_ledger_economy{metric="base_fee_xrp"}` | +| Reserve Base (drops) | stat | `xrpld_ledger_economy{metric="reserve_base_xrp"}` | +| Reserve Inc (drops) | stat | `xrpld_ledger_economy{metric="reserve_inc_xrp"}` | +| Ledger Age | stat | `xrpld_ledger_economy{metric="ledger_age_seconds"}` | +| Transaction Rate | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}` | + +**Key modified files**: `docker/telemetry/grafana/dashboards/node-health.json` + +**Exit Criteria**: + +- [ ] 5 new panels render correctly in existing dashboard +- [ ] Fee values match `server_info` RPC output +- [ ] Transaction rate shows smooth trend (not spiky) + +--- + +## Exit Criteria + +- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline +- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK +- [ ] Async gauge callbacks execute at 10s intervals without performance impact +- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue) +- [ ] 2 existing dashboards updated with new panel groups +- [ ] Integration test validates all new metric families are non-zero +- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) +- [ ] Documentation updated with full new metric inventory +- [ ] Validator Health dashboard renders all 13 panels +- [ ] Peer Quality dashboard renders all 6 panels +- [ ] Ledger Economy panels added to node-health dashboard + +--- + +## Appendix: Alerting Design + +> Design for the provisioned Grafana alert rules (Task 9.9a). Previously a standalone spec; merged here so the phase plan is self-contained. + +**Date:** 2026-07-06 +**Branch:** `pratik/otel-phase9-metric-gap-fill` (PR #6513, Jira RIPD-5187) +**Status:** Approved + +### Purpose + +Phase 9 exports ~68 internal xrpld metrics and ships Grafana dashboards for +them. This adds the missing operator-facing piece: **provisioned Grafana alert +rules** that fire on the health-critical metrics phase 9 introduces. The +phase-9 task list (line 311) and Jira story RIPD-5187 both already list +"alerting rules" as a phase-9 deliverable, so this closes that gap. + +Scope is deliberately narrow — the three subsystems whose failure is +node-fatal: **consensus/ledger health, validator health, job queue**. RPC/API +health is explicitly out of scope. + +### Why phase 9 (not phase 11) + +Every metric these alerts fire on is _born_ in phase 9 +(`xrpld_ledger_history_mismatch_total`, `xrpld_ledgers_closed_total`, +`xrpld_validation_missed_total`, `xrpld_validations_checked_total`, +`xrpld_jq_trans_overflow_total`, `xrpld_job_queued_duration_us_bucket`). Alerts +belong with the metrics they watch, and this is where the dependency lives. + +### Delivery + +Provisioned YAML, version-controlled — matching the existing datasource / +dashboard provisioning pattern. No docker-compose change: the Grafana service +already mounts `./grafana/provisioning:/etc/grafana/provisioning:ro`, and +Grafana auto-loads `provisioning/alerting/*.yaml`. + +New files under `docker/telemetry/grafana/provisioning/alerting/`: + +| File | Purpose | +| -------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `contactpoints.yaml` | One contact point `xrpld-default` (webhook to a documented placeholder; comments show how to swap for Slack/email). | +| `policies.yaml` | Default notification policy: route all alerts → `xrpld-default`, grouped by `alertname` + `service_instance_id`. | +| `rules.yaml` | 6 alert rules across 3 groups (below). | + +Plus the Alerting section of `docs/telemetry-runbook.md` — operator runbook: +what each alert means, likely causes, and how to point the contact point at a +real receiver. + +### Alert rules + +All rules target Prometheus datasource `uid: prometheus`. Each rule uses the +Grafana rule shape: query (A) → reduce (B, last value) → threshold (C). All +`rate()`/`histogram_quantile()` expressions aggregate with +`sum by (service_instance_id)` (or `+ le`) so **each node alerts independently**. +Alert rules run headless, so they cannot use the dashboards' `$node` template +variables — they match all series and group by `service_instance_id` instead. + +| Group | Alert | Expression (5m window) | Fires | `for` | severity | +| --------- | --------------------- | ----------------------------------------------------------------------------------------------------------- | --------------------- | ----- | -------- | +| Consensus | LedgerHistoryMismatch | `sum by (service_instance_id)(rate(xrpld_ledger_history_mismatch_total[5m]))` | `> 0` | 5m | critical | +| Consensus | LedgerCloseStalled | `sum by (service_instance_id)(rate(xrpld_ledgers_closed_total[5m]))` | `< 0.001` (≈0) | 3m | critical | +| Validator | ValidationsMissed | `sum by (service_instance_id)(rate(xrpld_validation_missed_total[5m]))` | `> 0` | 5m | warning | +| Validator | ValidationsNotChecked | `sum by (service_instance_id)(rate(xrpld_validations_checked_total[5m]))` | `< 0.001` (≈0) | 5m | warning | +| Job queue | JobQueueTxOverflow | `sum by (service_instance_id)(rate(xrpld_jq_trans_overflow_total[5m]))` | `> 0` | 5m | warning | +| Job queue | JobQueueLatencyHigh | `histogram_quantile(0.99, sum by (le, service_instance_id)(rate(xrpld_job_queued_duration_us_bucket[5m])))` | `> 1000000` (µs = 1s) | 5m | warning | + +Each rule carries labels `severity` and `category` (consensus/validator/jobqueue) +and annotations `summary` + `description` (with `{{ $labels.service_instance_id }}` +and `{{ $values.B.Value }}` interpolation). + +#### Threshold rationale + +- **LedgerCloseStalled `< 0.001` for 3m**: healthy nodes close a ledger every + ~3-5s; a 5m rate decaying to ~0 means the node is stuck. The epsilon (not + exact `0`) avoids float rate-noise suppressing the alert. +- **JobQueueLatencyHigh 1s p99**: a default starting point, easy to tune — jobs + queued >1s at p99 indicate the node is saturated. +- Others are `> 0` on error/miss counters: any sustained nonzero rate is + actionable. + +### Non-goals / YAGNI + +- No per-alert silencing schedules, no mute timings. +- No RPC/API, overlay, or fee-market alerts (dashboards cover those visually). +- Single contact point — multi-receiver routing is left to the operator. + +### Verification + +1. `yamllint` (or `python -c yaml.safe_load`) on all three YAML files. +2. `docker compose -f docker/telemetry/docker-compose.yml config -q` still parses. +3. Optional live check: start stack, `GET /api/v1/provisioning/alert-rules` + returns the 6 rules; Grafana logs show no provisioning errors. +4. Code-review pass (subagent) against phase conventions before commit. diff --git a/OpenTelemetryPlan/secure-OTel.md b/OpenTelemetryPlan/secure-OTel.md new file mode 100644 index 00000000000..3d22de5452f --- /dev/null +++ b/OpenTelemetryPlan/secure-OTel.md @@ -0,0 +1,240 @@ +# Securing OpenTelemetry Against Trace Context Spoofing + +> **Part of**: [OpenTelemetry Implementation Plan](./OpenTelemetryPlan.md) — see also [Design Decisions § Privacy](./02-design-decisions.md#244-privacy--sensitive-data-policy) (what we don't collect) and [Configuration Reference § 5.5](./05-configuration-reference.md#55-opentelemetry-collector-configuration) (collector base config). + +Trace context spoofing (or poisoning) occurs when untrusted actors inject tampered or stale trace IDs into your system. If these requests are processed, the spans are appended to historical trace buckets, stretching trace durations, ruining p99 latency metrics, and breaking Grafana dashboards. + +This guide outlines two categories of defense: mitigating tampered contexts and locking down the OpenTelemetry (OTel) Collector to trusted clients only. + +--- + +## Part 1: Mitigating Tampered Trace Contexts + +### 1. Perimeter Defense: Strip Headers at the API Gateway + +The most effective way to prevent spoofing from external sources is to treat your API Gateway (Envoy, NGINX, AWS ALB) as a hard boundary. Strip incoming W3C tracing headers (`traceparent`, `tracestate`) from public traffic so the gateway is forced to generate a fresh, legitimate `trace_id`. + +**NGINX Example (Stripping Headers):** + +```nginx +server { + listen 80; + + location / { + # Clear out untrusted incoming trace headers + proxy_set_header traceparent ""; + proxy_set_header tracestate ""; + + proxy_pass http://backend_service; + } +} +``` + +### **2. Timestamp-Anchored Trace IDs and OTTL Filtering** + +If you use a custom trace ID generator that embeds a timestamp in the first few bytes (like AWS X-Ray or UUIDv7), you can use the OTel Collector's OpenTelemetry Transform Language (OTTL) to detect anomalies. +**Collector Configuration (Conceptual OTTL Filter):** + +```yaml +processors: + filter/stale_traces: + error_mode: ignore + traces: + span: + # Example: Drop spans where the start time is significantly different + # from an expected parameter or embedded timestamp logic. + # Note: Standard W3C trace IDs do not contain timestamps by default. + - 'Keep out-of-bounds spans: time.sub(start_time, now()) > duration("1h")' +``` + +## **Part 2: Restricting Access to the OTel Collector** + +Locking down the Collector ensures that only authenticated, trusted clients can submit telemetry data. + +### **Approach A: Network Layer Security (Kubernetes Network Policies)** + +Ensure your Collector is not exposed to the public internet. If running in Kubernetes, use a NetworkPolicy to restrict ingress traffic to specific namespaces. +**Kubernetes NetworkPolicy Example:** + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-internal-otel + namespace: observability +spec: + podSelector: + matchLabels: + app: opentelemetry-collector + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + environment: production + ports: + - protocol: TCP + port: 4317 # gRPC + - protocol: TCP + port: 4318 # HTTP +``` + +### **Approach B: Transport Layer Security (Mutual TLS / mTLS)** + +Require clients to present a valid cryptographic certificate to connect to the Collector. +**Collector Configuration (mTLS):** + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + # Setting client_ca_file makes the collector require and verify a + # client cert, rejecting connections without a trusted one. + client_ca_file: /certs/client_ca.pem # CA that signs trusted client certs + cert_file: /certs/collector.pem + key_file: /certs/collector.key +``` + +### **Approach C: Application Layer Authentication (Basic Auth Extension)** + +Use the Collector's extension system to require an API key or Basic Auth credentials. +**Collector Configuration (Basic Auth):** + +```yaml +extensions: + basicauth/collector: + htpasswd: + inline: | + # username:trusted-client, password:SecurePassword123 + trusted-client:$apr1$4v8p76o6$DMTX5Wv6uOmrFAZp2X1N1. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + auth: + authenticator: basicauth/collector + +processors: + batch: + +exporters: + otlp: + endpoint: my-backend-storage:4317 + +service: + extensions: [basicauth/collector] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] +``` + +**Client Setup (Environment Variables):** +Developers must pass the authentication header using the standard OTel SDK environment variables: + +```bash +# Base64 encoded "trusted-client:SecurePassword123" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic dHJ1c3RlZC1jbGllbnQ6U2VjdXJlUGFzc3dvcmQxMjM=" +``` + +--- + +Available routes to build on top of: https://github.com/XRPLF/rippled/pull/6425#discussion_r3234751995 + +--- + +# Analysis: Applying the Guide to xrpld + +The guide above is written for HTTP-fronted web services. xrpld is a P2P node daemon, so the threat model and the applicable defenses differ. This section captures how each approach maps to xrpld and the chosen direction. + +## Threat Model + +xrpld has **two distinct attack surfaces**, not one. The original guide conflates them under "trace context spoofing"; for xrpld they need separate defenses. + +| Surface | Attacker | Vector | Defense | +| ----------------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------- | +| **Collector ingress** (xrpld → collector) | Anyone who can reach `4317`/`4318` on the collector host | Forged OTLP traffic, telemetry exfiltration, DoS on collector | mTLS + network policy | +| **Peer trace context** (peer → xrpld) | Malicious peer in the XRPL overlay | Crafted `protocol::TraceContext` field inside peer protobuf messages (TMTransaction, consensus, etc.) — used to forge `trace_id`/`span_id`, pollute p99, attach spans to historical traces | Validate + rate-limit at the receive boundary | + +**Deployment context:** Across-network. xrpld nodes (potentially run by external operators or in different DCs) ship telemetry to a centrally-hosted collector across an untrusted network. The collector is NOT on the same host or private VPC as every node. + +``` + ┌── peer (untrusted) ── TMTransaction{trace_context} ──▶ xrpld + │ │ + │ [validate + rate-limit] + │ │ + │ ▼ + │ SpanGuard (clean) + │ │ + │ │ OTLP/gRPC + │ │ + mTLS + │ ▼ + └───────────────────────────────────────── [client_ca_file: verify client cert] + OTel Collector + (in private subnet, NetPol) +``` + +## Part 1 Applicability — Peer Trace-Context Validation + +The guide's NGINX header stripping and OTTL stale-span filtering target HTTP gateways and post-hoc cleanup. Neither fits xrpld directly: + +- **NGINX header stripping** — N/A. There is no HTTP gateway between peers and xrpld; trace context arrives inside protobuf peer messages (`protocol::TraceContext`), not as W3C `traceparent` headers. See [src/xrpld/telemetry/PropagationHelpers.h](../src/xrpld/telemetry/PropagationHelpers.h). +- **OTTL stale-span filtering** — Weak fit. Post-hoc cleanup at the collector loses peer identity (you can't tell _which_ peer poisoned the trace). Validation at the receive site is stronger. + +**xrpld-specific Part 1 mitigations:** + +1. **Validate extracted context at the boundary** in [src/xrpld/telemetry/ConsensusReceiveTracing.h](../src/xrpld/telemetry/ConsensusReceiveTracing.h) and any other peer-message receive site. Reject if `trace_id` is all-zero, wrong length, or fails W3C format checks. Treat invalid context as "no propagated context" — start a fresh span — rather than dropping the message. +2. **Per-peer sample rate limiting** so a hostile peer cannot flood the collector with spans bearing a fabricated `trace_id`. Use probabilistic sampling on the receive path keyed by peer identity. + +## Part 2 — Comparison of Collector Hardening Approaches + +Evaluated for the across-network deployment shape: + +| Approach | Across-network fit | Cost | Verdict | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | ---------------------------------- | +| **A. NetworkPolicy / firewall** | Necessary baseline (don't expose `4317`/`4318` to the internet), but insufficient on its own when traffic genuinely crosses networks — you cannot NetworkPolicy the public internet. | Cheap. | **Defense-in-depth, not primary.** | +| **B. mTLS** | Strongest fit. Every xrpld node holds a client cert; the collector verifies it via `client_ca_file` in the receiver's `tls` block. Encrypts in transit (raw OTLP over the internet leaks transaction patterns and validator identity). Compromised node = revoke one cert, no shared secret to rotate everywhere. | Cert issuance + rotation pipeline. | **Primary.** | +| **C. Basic Auth** | Worst shape for this topology. Single shared password across all xrpld nodes — one leaked node config compromises the whole fleet. Doesn't encrypt; you'd need TLS underneath anyway, at which point you're 80% of the way to mTLS. | Cheap to set up, expensive to operate (rotation across N operators). | **Skip.** | + +## Decision + +**Primary defense:** mTLS (Approach B) on the collector's OTLP receivers. The collector requires and verifies each client certificate when `client_ca_file` is set in the receiver's `tls` block (there is no `auth_type` field — setting `client_ca_file` is what enforces client-cert verification). + +**Defense-in-depth:** NetworkPolicy / firewall rules (Approach A) so `4317`/`4318` are never reachable from outside the expected operator subnets even if mTLS were misconfigured. + +**Skipped:** Basic Auth (Approach C) — wrong shape for an across-network, multi-operator topology. + +**Plus xrpld-specific Part 1 work:** trace-context validation and per-peer rate limiting at peer-message receive sites. + +## Decisions Made + +| Decision | Choice | Rationale | +| -------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cert source for mTLS | **Reuse XRPL node identity key** | One identity per node, no separate PKI to operate. Fits XRPL's existing trust model; requires small CA tooling step to derive/sign the OTel client cert from the node key. | +| Part 1 scope | **Include in this spec** | Collector hardening and peer trace-context validation share one threat model. Coherent design doc; can still be split into multiple PRs at implementation. | +| Dev impact | **Production-only** | Local `docker/telemetry/docker-compose.yml` keeps `insecure: true` and no auth for fast iteration. Only production deployment manifests gain mTLS. Accepted risk: minor dev/prod drift, mitigated by integration tests against a TLS-enabled collector in CI. | + +## Out of Scope + +- NGINX/Envoy header stripping (no HTTP gateway in front of xrpld-to-collector traffic). +- OTTL stale-span filtering at the collector (weaker than source validation; loses peer identity). +- Local development docker-compose hardening. +- Telemetry backend (Tempo) hardening — separate concern, downstream of the collector. + +## Next Step + +Write this up as a design doc with full sections covering: + +1. Threat model & architecture (this section, expanded) +2. Collector hardening — mTLS config, NetworkPolicy +3. Cert pipeline — deriving OTel client cert from XRPL node key +4. Peer trace-context validation — receive-site checks in `ConsensusReceiveTracing.h` +5. Per-peer span rate limiting +6. Testing & rollout diff --git a/cfg/xrpld-example.cfg b/cfg/xrpld-example.cfg index 9e334e6f4f3..54d3936c423 100644 --- a/cfg/xrpld-example.cfg +++ b/cfg/xrpld-example.cfg @@ -1621,3 +1621,108 @@ validators.txt # set to ssl_verify to 0. [ssl_verify] 1 +#------------------------------------------------------------------------------- +# +# 11. Telemetry (OpenTelemetry Tracing) +# +#------------------------------------------------------------------------------- +# +# Enables distributed tracing via OpenTelemetry. Requires building with +# -DXRPL_ENABLE_TELEMETRY=ON (telemetry Conan option). +# +# [telemetry] +# +# enabled=0 +# +# Enable or disable telemetry at runtime. Default: 0 (disabled). +# +# service_name=xrpld +# +# OTel resource attribute `service.name`. Default: xrpld. +# The node's network ID (from [network_id]) is automatically added +# as the `xrpl.network.id` and `xrpl.network.type` resource attributes. +# +# service_instance_id= +# +# OTel resource attribute `service.instance.id`. Uniquely identifies +# this node. Default: the node's public key (auto-detected). +# +# endpoint=http://localhost:4318/v1/traces +# +# The OTLP/HTTP exporter endpoint. The server sends trace data as +# protobuf-encoded HTTP POST requests to this URL. +# Default: http://localhost:4318/v1/traces. +# +# --- TLS settings for the OTLP exporter connection --- +# +# use_tls=0 +# +# Enable TLS for the OTLP/HTTP exporter connection. Default: 0 (off). +# +# tls_ca_cert= +# +# Path to a PEM-encoded CA certificate bundle for TLS verification. +# Only used when use_tls=1. Default: empty (system CA store). +# +# tls_client_cert= +# +# Path to this node's PEM-encoded client certificate, presented to the +# collector for mutual TLS (mTLS). Only used when use_tls=1. Leave empty +# for one-way (server-only) TLS. Default: empty. +# +# tls_client_key= +# +# Path to the PEM-encoded private key for tls_client_cert. Required +# whenever tls_client_cert is set. Only used when use_tls=1. +# Default: empty. +# +# Head sampling is intentionally fixed at 1.0 (sample everything) and is +# not configurable. A per-node sampling ratio would let nodes make +# divergent keep/drop decisions for the same distributed trace, producing +# broken/partial traces. A ParentBasedSampler ensures spans inheriting a +# remote parent honor the upstream decision. Reduce volume at the collector +# via tail sampling instead; for node-local post-hoc dropping use +# SpanGuard::discard() in code. +# +# trace_rpc=1 +# +# Enable tracing for JSON-RPC and WebSocket API request handling — +# command parsing, execution, and response serialization. Default: 1. +# +# trace_transactions=1 +# +# Enable tracing for the transaction lifecycle — submission, validation, +# application to ledgers, and final disposition. Default: 1. +# +# trace_consensus=1 +# +# Enable tracing for the consensus round lifecycle — proposals, +# validations, mode changes, and ledger acceptance. Default: 1. +# +# trace_peer=1 +# +# Enable tracing for peer-to-peer protocol messages — overlay message +# send/receive, peer handshakes, and routing. High volume; enabled +# by default. Default: 1. +# +# trace_ledger=1 +# +# Enable tracing for ledger close and accept operations — ledger +# building, state hashing, and write-back to the node store. Default: 1. +# +# --- Batch processor tuning --- +# +# batch_size=512 +# +# Maximum number of spans exported in a single batch. Default: 512. +# +# batch_delay_ms=5000 +# +# Maximum delay (milliseconds) before a partial batch is flushed. +# Default: 5000 (5 seconds). +# +# max_queue_size=2048 +# +# Maximum number of spans queued in memory before drops occur. +# Default: 2048. +# diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index 3e492677152..40e7c0f8542 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -78,6 +78,19 @@ include(target_link_modules) # Level 01 add_module(xrpl beast) target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main) +# OTelCollector in beast/insight uses the OTel Metrics SDK when telemetry is +# enabled. Link the Conan-provided umbrella target rather than individual +# component targets: the OTel package's per-component dependency graph is +# under-declared (e.g. the OTLP client references sdk::common symbols without +# declaring the edge), so naming components directly reorders the static-link +# line into an unresolvable state. The umbrella carries the full, internally +# consistent graph the package authors validated. +if(telemetry) + target_link_libraries( + xrpl.libxrpl.beast + PUBLIC opentelemetry-cpp::opentelemetry-cpp + ) +endif() include(GitInfo) add_module(xrpl git) @@ -198,8 +211,36 @@ target_link_libraries( xrpl.libxrpl.conditions ) +# Telemetry module — OpenTelemetry distributed tracing support. +# Sources: include/xrpl/telemetry/ (headers), src/libxrpl/telemetry/ (impl). +# When telemetry=ON, links the Conan-provided umbrella target +# opentelemetry-cpp::opentelemetry-cpp (individual component targets like +# ::api, ::sdk are not available in the Conan package). +# +# Links xrpl.libxrpl.protocol PRIVATELY for sha512Half (digest.h) +add_module(xrpl telemetry) +target_link_libraries( + xrpl.libxrpl.telemetry + PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.beast xrpl.libxrpl.config + PRIVATE xrpl.libxrpl.protocol +) +if(telemetry) + # Telemetry owns both the trace and (as of the direct-metrics API) the + # metrics pipeline. Link the umbrella target: it supplies the trace and + # metrics SDK components with the correct static-link ordering, which + # naming components individually does not (the package under-declares + # inter-component dependencies). + target_link_libraries( + xrpl.libxrpl.telemetry + PUBLIC opentelemetry-cpp::opentelemetry-cpp + ) +endif() + add_module(xrpl tx) -target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger) +target_link_libraries( + xrpl.libxrpl.tx + PUBLIC xrpl.libxrpl.ledger xrpl.libxrpl.telemetry +) add_library(xrpl.libxrpl) set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl) @@ -233,6 +274,7 @@ target_link_modules( resource server shamap + telemetry tx ) diff --git a/cmake/XrplDocs.cmake b/cmake/XrplDocs.cmake index 7b3e9b3b306..6f81dcfd5f1 100644 --- a/cmake/XrplDocs.cmake +++ b/cmake/XrplDocs.cmake @@ -27,8 +27,12 @@ file( src/*.cpp src/*.md Builds/*.md - *.md ) +# Add only top-level .md files (README, CONTRIBUTING, etc.) without +# recursing into dot-directories like .claude/ whose files are not +# valid Doxygen/CMake sources. +file(GLOB doxygen_top_md CONFIGURE_DEPENDS "*.md") +list(APPEND doxygen_input ${doxygen_top_md}) list(APPEND doxygen_input external/README.md) set(dependencies "${doxygen_input}" "${doxyfile}") diff --git a/conan.lock b/conan.lock index c6a4070c771..5b6fa9620fa 100644 --- a/conan.lock +++ b/conan.lock @@ -10,11 +10,14 @@ "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1782392413.075713", "re2/20251105#8579cfd0bda4daf0683f9e3898f964b4%1782392402.431897", "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933", + "opentelemetry-cpp/1.26.0#9d81768342c78cb897345fd419b358d2%1776934712.672", "openssl/3.6.3#f806de8933e3bf6f01016c6a888cee2e%1783945160.863288", "nudb/2.0.9#11149c73f8f2baff9a0198fe25971fc7%1782392402.297166", + "nlohmann_json/3.11.3#45828be26eb619a2e04ca517bb7b828d%1701220705.259", "mpt-crypto/0.4.0-rc4#ffdba12f2332357f0d8b0ae944cfff52%1784138702.932355", "lz4/1.10.0#982d9b673900f665a1da109e09c17cab%1782392402.164188", "libiconv/1.17#9923bc6dc6f106646d6967e0039a5ada%1782392792.775744", + "libcurl/8.20.0#c90b0c91a33d9a79b519c1c70bafc823%1780907438.587", "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1782392402.420732", "libarchive/3.8.7#c446109bd1f1d8ba7936c94189bc50e6%1782392403.066892", "jemalloc/5.3.1#1fc58d55316041f10fbc1e8a2eae632a%1776700028.228", @@ -32,9 +35,15 @@ "zlib/1.3.2#1cb806da49011867778ffb6ac7190fcb%1782392402.122708", "strawberryperl/5.32.1.1#8d114504d172cfea8ea1662d09b6333e%1782395692.540639", "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933", + "pkgconf/2.5.1#93c2051284cba1279494a43a4fcfeae2%1757684701.089", + "opentelemetry-proto/1.7.0#ed6d5bd761bef0afb0ba09676420b9ea%1749461220.268", + "ninja/1.13.2#c8c5dc2a52ed6e4e42a66d75b4717ceb%1764096931.974", "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1782395690.33162", "msys2/cci.latest#d22fe7b2808f5fd34d0a7923ace9c54f%1770657326.649", + "meson/1.10.2#9d2d10681fe7fe61c788c58626c89b25%1775558003.754", "m4/1.4.19#34c4bbc3eeebe98ca6edf2f52d602e7d%1777282960.259", + "libtool/2.4.7#14e7739cc128bc1623d2ed318008e47e%1755679003.847", + "gnu-config/cci.20210814#466e9d4d7779e1c142443f7ea44b4284%1762363589.329", "cmake/4.3.3#840cf00ea09777e05c2050a50a82c722%1782392418.696091", "b2/5.4.2#ffd6084a119587e70f11cd45d1a386e2%1782392402.624226", "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56", @@ -60,6 +69,9 @@ ], "lz4/[>=1.9.4 <2]": [ "lz4/1.10.0#982d9b673900f665a1da109e09c17cab" + ], + "protobuf/[>=4.25.3 <7]": [ + "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81" ] }, "config_requires": [] diff --git a/conan/profiles/default b/conan/profiles/default index ae6e23c3c3d..cc937ce5111 100644 --- a/conan/profiles/default +++ b/conan/profiles/default @@ -25,3 +25,18 @@ compiler.libcxx={{ detect_api.detect_libcxx(compiler, version, compiler_exe) }} {# More info: https://docs.conan.io/2/reference/extensions/binary_compatibility.html #} user.package:cppstd_version=23 tools.info.package_id:confs+=["user.package:cppstd_version"] +{% if compiler == "gcc" and compiler_version < 13 %} +tools.build:cxxflags+=['-Wno-restrict'] +{% endif %} +{% if os == "Windows" %} +# opentelemetry-cpp's recipe removes the `shared` option on Windows and never +# sets BUILD_SHARED_LIBS, so its upstream CMake defaults the protobuf-generated +# `opentelemetry_proto` target to a DLL (opentelemetry_proto.dll). The rest of +# the project links statically and nothing deploys that DLL next to the +# executables, so the telemetry unit test fails to start with +# STATUS_DLL_NOT_FOUND (0xC0000135). Force the dependency to build fully static +# so no runtime DLL is produced. The conf is folded into the package id so a +# fresh static binary is built instead of reusing a previously cached one. +opentelemetry-cpp/*:tools.cmake.cmaketoolchain:extra_variables={"BUILD_SHARED_LIBS": "OFF"} +opentelemetry-cpp/*:tools.info.package_id:confs+=["tools.cmake.cmaketoolchain:extra_variables"] +{% endif %} diff --git a/conanfile.py b/conanfile.py index f883761f0e3..fdd74264408 100644 --- a/conanfile.py +++ b/conanfile.py @@ -22,6 +22,7 @@ class Xrpl(ConanFile): "rocksdb": [True, False], "shared": [True, False], "static": [True, False], + "telemetry": [True, False], "tests": [True, False], "unity": [True, False], "xrpld": [True, False], @@ -54,6 +55,7 @@ class Xrpl(ConanFile): "rocksdb": True, "shared": False, "static": True, + "telemetry": True, "tests": False, "unity": False, "xrpld": False, @@ -144,6 +146,10 @@ def requirements(self): self.requires("rocksdb/10.5.1") self.requires("secp256k1/0.7.1", transitive_headers=True) self.requires("sqlite3/3.53.0", force=True) + # OpenTelemetry C++ SDK for distributed tracing (optional). + # Provides OTLP/HTTP exporter, batch span processor, and trace API. + if self.options.telemetry: + self.requires("opentelemetry-cpp/1.26.0") self.requires("xxhash/0.8.3", transitive_headers=True) exports_sources = ( @@ -173,6 +179,7 @@ def generate(self): tc.variables["rocksdb"] = self.options.rocksdb tc.variables["BUILD_SHARED_LIBS"] = self.options.shared tc.variables["static"] = self.options.static + tc.variables["telemetry"] = self.options.telemetry tc.variables["unity"] = self.options.unity tc.variables["xrpld"] = self.options.xrpld tc.generate() @@ -226,3 +233,5 @@ def package_info(self): ] if self.options.rocksdb: libxrpl.requires.append("rocksdb::librocksdb") + if self.options.telemetry: + libxrpl.requires.append("opentelemetry-cpp::opentelemetry-cpp") diff --git a/docker/telemetry/.env.alerting.example b/docker/telemetry/.env.alerting.example new file mode 100644 index 00000000000..9822c635f1d --- /dev/null +++ b/docker/telemetry/.env.alerting.example @@ -0,0 +1,32 @@ +# rippled OTel alerting delivery — copy to `.env.alerting` and fill in. +# +# `.env.alerting` is gitignored; never commit a real Slack webhook or address. +# Grafana reads these when the stack starts and expands the ${VARS} referenced +# in grafana/provisioning/alerting/contactpoints.yaml. See the Alerting +# section of docs/telemetry-runbook.md. +# +# Any var left blank simply disables that delivery path — the stack still runs. + +# --- Slack --- +# Incoming-webhook URL from the Slack app (Incoming Webhooks feature). +# Used by both the warning (xrpld-default) and critical (xrpld-critical) tiers. +SLACK_WEBHOOK_URL= + +# Channel label. With an incoming webhook the target channel is fixed by the +# webhook itself; this only satisfies Grafana's Slack validator. Defaults to +# #xrpld-alerts if unset. +SLACK_CHANNEL=#xrpld-alerts + +# --- Email (critical tier only) --- +# Comma-separated recipient list for critical alerts. +ALERT_EMAIL_TO= + +# SMTP relay Grafana sends through. Email only delivers when SMTP is enabled +# and these point at a real relay (e.g. an internal smart host or an +# authenticated provider). +GF_SMTP_ENABLED=false +GF_SMTP_HOST=smtp.example.com:587 +GF_SMTP_USER= +GF_SMTP_PASSWORD= +GF_SMTP_FROM_ADDRESS=alerts@xrpld.local +GF_SMTP_FROM_NAME=xrpld Alerts diff --git a/docker/telemetry/.env.grafanacloud-alloy.example b/docker/telemetry/.env.grafanacloud-alloy.example new file mode 100644 index 00000000000..a43e09e6d92 --- /dev/null +++ b/docker/telemetry/.env.grafanacloud-alloy.example @@ -0,0 +1,27 @@ +# Grafana Alloy -> Grafana Cloud credentials for docker/telemetry/alloy/config.alloy. +# Copy to `.env.grafanacloud-alloy`, fill in, and source before starting Alloy. +# `.env.grafanacloud-alloy` is gitignored; never commit real tokens or stack ids. +# +# Prometheus values: Grafana Cloud -> Connections -> "Prometheus". +# OTLP values: Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)". + +# --- Host / systemd metrics (prometheus.remote_write) --- +# remote_write push URL, e.g. https://prometheus-prod-XX-.grafana.net/api/prom/push +GRAFANACLOUD_PROM_URL= +# Numeric instance/stack id (Basic-auth username). +GRAFANACLOUD_PROM_USER= +# Access-policy token with metrics:write (Basic-auth password). +GRAFANACLOUD_PROM_KEY= + +# --- xrpld OTLP (otelcol.exporter.otlphttp) --- +# OTLP/HTTP gateway URL including the /otlp path, +# e.g. https://otlp-gateway-prod-.grafana.net/otlp +GRAFANACLOUD_OTLP_URL= +# Numeric instance/stack id (Basic-auth username). +GRAFANACLOUD_OTLP_USER= +# Access-policy token with metrics:write + traces:write (Basic-auth password). +GRAFANACLOUD_OTLP_KEY= + +# --- Per-node label --- +# host label applied to this node's scraped metrics (e.g. the node's hostname). +XRPLD_HOST_LABEL= diff --git a/docker/telemetry/.env.grafanacloud.example b/docker/telemetry/.env.grafanacloud.example new file mode 100644 index 00000000000..53a1a2a7f39 --- /dev/null +++ b/docker/telemetry/.env.grafanacloud.example @@ -0,0 +1,16 @@ +# Grafana Cloud OTLP credentials — copy to `.env.grafanacloud` and fill in. +# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)". +# +# `.env.grafanacloud` is gitignored; never commit real tokens. + +# OTLP/HTTP gateway URL for your stack, including the /otlp path. +# Example: https://otlp-gateway-prod-us-east-0.grafana.net/otlp +GRAFANA_CLOUD_OTLP_ENDPOINT= + +# Numeric instance/stack id shown on the OTLP connection page +# (used as the Basic-auth username). +GRAFANA_CLOUD_INSTANCE_ID= + +# A Cloud Access Policy token with metrics:write, traces:write, logs:write +# (used as the Basic-auth password). +GRAFANA_CLOUD_API_TOKEN= diff --git a/docker/telemetry/.gitignore b/docker/telemetry/.gitignore new file mode 100644 index 00000000000..41139c450af --- /dev/null +++ b/docker/telemetry/.gitignore @@ -0,0 +1,13 @@ +# Runtime data generated by xrpld and telemetry stack +data/ + +# Env. file carrying environmental setup data for local or cloud runs. +.env.* + +# Keep examples +!.env.alerting.example +!.env.grafanacloud.example +!.env.grafanacloud-alloy.example + +# Do not commit grafana cloud versions with default filters +grafanacloud/*.json diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md new file mode 100644 index 00000000000..7cf2b19fa0d --- /dev/null +++ b/docker/telemetry/TESTING.md @@ -0,0 +1,708 @@ +# OpenTelemetry Integration Testing Guide + +This document describes how to verify the xrpld OpenTelemetry telemetry +pipeline end-to-end, from span generation through the observability stack +(otel-collector, Tempo, Prometheus, Grafana). + +--- + +## Prerequisites + +### Build xrpld with telemetry + +```bash +conan install . --build=missing -o telemetry=True +cmake --preset default -Dtelemetry=ON +cmake --build --preset default --target xrpld +``` + +The binary is at `.build/xrpld`. + +### Required tools + +- **Docker** with `docker compose` (v2) +- **curl** +- **jq** (JSON processor) + +### Verify binary + +```bash +.build/xrpld --version +``` + +--- + +## Test 1: Single-Node Standalone (Quick Verification) + +This test verifies RPC and transaction spans in standalone mode. Consensus +spans will not fire because standalone mode does not run consensus. + +### Step 1: Start the observability stack + +```bash +docker compose -f docker/telemetry/docker-compose.yml up -d +``` + +Wait for services to be ready: + +```bash +# otel-collector health +curl -sf http://localhost:13133/ && echo "collector ready" + +# Tempo readiness +curl -sf http://localhost:3200/ready >/dev/null && echo "tempo ready" +``` + +### Step 2: Start xrpld in standalone mode + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start +``` + +Wait a few seconds for the node to initialize. + +### Step 3: Exercise RPC spans + +```bash +# server_info +curl -s http://localhost:5005 \ + -d '{"method":"server_info"}' | jq .result.info.server_state + +# server_state +curl -s http://localhost:5005 \ + -d '{"method":"server_state"}' | jq .result.state.server_state + +# ledger +curl -s http://localhost:5005 \ + -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' | + jq .result.ledger_current_index +``` + +### Step 4: Submit a transaction + +Close the ledger first (required in standalone mode): + +```bash +curl -s http://localhost:5005 -d '{"method":"ledger_accept"}' +``` + +Submit a Payment from the genesis account: + +```bash +curl -s http://localhost:5005 -d '{ + "method": "submit", + "params": [{ + "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", + "tx_json": { + "TransactionType": "Payment", + "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF", + "Amount": "10000000" + } + }] +}' | jq .result.engine_result +``` + +Expected result: `"tesSUCCESS"`. + +Close the ledger again to finalize: + +```bash +curl -s http://localhost:5005 -d '{"method":"ledger_accept"}' +``` + +### Step 5: Verify traces in Tempo + +Wait 5 seconds for the batch export, then: + +```bash +TEMPO="http://localhost:3200" + +# Check xrpld service is registered +curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value' + +# Check RPC spans +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.http_request"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' + +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.process"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' + +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.command.server_info"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' + +# Check transaction spans +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="tx.process"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' +``` + +Or open Grafana Explore with Tempo datasource: http://localhost:3000 + +### Step 6: Teardown + +```bash +# Kill xrpld (Ctrl+C or) +kill $(pgrep -f 'xrpld.*xrpld-telemetry') + +# Stop observability stack +docker compose -f docker/telemetry/docker-compose.yml down + +# Clean xrpld data +rm -rf data/ +``` + +### Expected spans (standalone mode) + +| Span Name | Expected | Notes | +| --------------------------- | -------- | ----------------------------- | +| `rpc.http_request` | Yes | Every HTTP RPC call | +| `rpc.process` | Yes | Every RPC processing | +| `rpc.command.server_info` | Yes | server_info RPC | +| `rpc.command.server_state` | Yes | server_state RPC | +| `rpc.command.ledger` | Yes | ledger RPC | +| `rpc.command.submit` | Yes | submit RPC | +| `rpc.command.ledger_accept` | Yes | ledger_accept RPC | +| `tx.process` | Yes | Transaction submission | +| `tx.receive` | No | No peers in standalone | +| `consensus.*` | No | Consensus disabled standalone | + +--- + +## Test 2: 6-Node Consensus Network (Full Verification) + +This test verifies ALL span categories including consensus and peer +transaction relay, using a 6-node validator network. + +### Automated + +Run the integration test script: + +```bash +bash docker/telemetry/integration-test.sh +``` + +The script will: + +1. Start the observability stack +2. Generate 6 validator key pairs +3. Create config files for each node +4. Start all 6 nodes +5. Wait for consensus ("proposing" state) +6. Exercise RPC, submit transactions +7. Verify all span categories in Tempo +8. Verify spanmetrics in Prometheus +9. Print results and leave the stack running + +### Manual + +If you prefer to run the steps manually: + +#### Step 1: Start observability stack + +```bash +docker compose -f docker/telemetry/docker-compose.yml up -d +``` + +#### Step 2: Generate validator keys + +Start a temporary standalone xrpld: + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start & +TEMP_PID=$! +sleep 5 +``` + +Generate 6 key pairs: + +```bash +for i in $(seq 1 6); do + curl -s http://localhost:5005 \ + -d '{"method":"validation_create"}' | jq '.result' +done +``` + +Record the `validation_seed` and `validation_public_key` for each. +Kill the temporary node: + +```bash +kill $TEMP_PID +rm -rf data/ +``` + +#### Step 3: Create node configs + +For each node (1-6), create a config file. Template: + +```ini +[server] +port_rpc +port_peer + +[port_rpc] +port = {5004 + node_number} +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_peer] +port = {51234 + node_number} +ip = 0.0.0.0 +protocol = peer + +[node_db] +type=NuDB +path=/tmp/xrpld-integration/node{N}/nudb +online_delete=256 + +[database_path] +/tmp/xrpld-integration/node{N}/db + +[debug_logfile] +/tmp/xrpld-integration/node{N}/debug.log + +[validation_seed] +{seed from step 2} + +[validators_file] +/tmp/xrpld-integration/validators.txt + +[ips_fixed] +127.0.0.1 51235 +127.0.0.1 51236 +127.0.0.1 51237 +127.0.0.1 51238 +127.0.0.1 51239 +127.0.0.1 51240 + +[peer_private] +1 + +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +batch_size=512 +batch_delay_ms=2000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 + +[rpc_startup] +{ "command": "log_level", "severity": "warning" } + +[ssl_verify] +0 +``` + +#### Step 4: Create validators.txt + +```ini +[validators] +{public_key_1} +{public_key_2} +{public_key_3} +{public_key_4} +{public_key_5} +{public_key_6} +``` + +#### Step 5: Start all 6 nodes + +```bash +for i in $(seq 1 6); do + .build/xrpld --conf /tmp/xrpld-integration/node$i/xrpld.cfg --start & + echo $! >/tmp/xrpld-integration/node$i/xrpld.pid +done +``` + +#### Step 6: Wait for consensus + +Poll each node until `server_state` = `"proposing"`: + +```bash +for port in 5005 5006 5007 5008 5009 5010; do + while true; do + state=$(curl -s http://localhost:$port \ + -d '{"method":"server_info"}' | + jq -r '.result.info.server_state') + echo "Port $port: $state" + [ "$state" = "proposing" ] && break + sleep 5 + done +done +``` + +#### Step 7: Exercise RPC and submit transaction + +```bash +# RPC calls +curl -s http://localhost:5005 -d '{"method":"server_info"}' +curl -s http://localhost:5005 -d '{"method":"server_state"}' +curl -s http://localhost:5005 -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' + +# Submit transaction +curl -s http://localhost:5005 -d '{ + "method": "submit", + "params": [{ + "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", + "tx_json": { + "TransactionType": "Payment", + "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF", + "Amount": "10000000" + } + }] +}' +``` + +Wait 15 seconds for consensus and batch export. + +#### Step 8: Verify in Tempo + +See the "Verification Queries" section below. + +--- + +## Expected Span Catalog + +All 16 production span names instrumented across Phases 2-5: + +| Span Name | Source File | Phase | Key Attributes | How to Trigger | +| --------------------------- | ----------------- | ----- | ---------------------------------------------------------------------------------------- | ------------------------- | +| `rpc.http_request` | ServerHandler.cpp | 2 | -- | Any HTTP RPC call | +| `rpc.ws_upgrade` | ServerHandler.cpp | 2 | -- | WebSocket upgrade | +| `rpc.ws_message` | ServerHandler.cpp | 2 | -- | WebSocket RPC message | +| `rpc.process` | ServerHandler.cpp | 2 | -- | RPC processing | +| `rpc.command.` | RPCHandler.cpp | 2 | `command`, `version`, `rpc_role` | Any RPC command | +| `tx.process` | NetworkOPs.cpp | 3 | `xrpl.tx.hash`, `local`, `path` | Submit transaction | +| `tx.receive` | PeerImp.cpp | 3 | `xrpl.peer.id` | Peer relays transaction | +| `consensus.proposal.send` | RCLConsensus.cpp | 4 | `xrpl.consensus.round` | Consensus proposing phase | +| `consensus.ledger_close` | RCLConsensus.cpp | 4 | `xrpl.consensus.ledger.seq`, `xrpl.consensus.mode` | Ledger close event | +| `consensus.accept` | RCLConsensus.cpp | 4 | `xrpl.consensus.proposers`, `xrpl.consensus.round_time_ms` | Ledger accepted | +| `consensus.validation.send` | RCLConsensus.cpp | 4 | `xrpl.consensus.ledger.seq`, `xrpl.consensus.proposing` | Validation sent | +| `consensus.accept.apply` | RCLConsensus.cpp | 4 | `xrpl.consensus.close_time`, `close_time_correct`, `close_resolution_ms`, `state` | Ledger apply + close time | +| `tx.apply` | BuildLedger.cpp | 5 | `xrpl.ledger.tx_count`, `xrpl.ledger.tx_failed` | Ledger close (tx set) | +| `ledger.build` | BuildLedger.cpp | 5 | `xrpl.ledger.seq`, `xrpl.ledger.close_time`, `close_time_correct`, `close_resolution_ms` | Ledger build | +| `ledger.validate` | LedgerMaster.cpp | 5 | `xrpl.ledger.seq`, `xrpl.ledger.validations` | Ledger validated | +| `ledger.store` | LedgerMaster.cpp | 5 | `xrpl.ledger.seq` | Ledger stored | +| `peer.proposal.receive` | PeerImp.cpp | 5 | `xrpl.peer.id`, `proposal_trusted` | Peer sends proposal | +| `peer.validation.receive` | PeerImp.cpp | 5 | `xrpl.peer.id`, `validation_trusted` | Peer sends validation | + +--- + +## Verification Queries + +### Tempo API + +Base URL: `http://localhost:3200` + +```bash +TEMPO="http://localhost:3200" + +# List all services +curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value' + +# Query traces by operation +for op in "rpc.http_request" "rpc.ws_upgrade" "rpc.ws_message" "rpc.process" \ + "rpc.command.server_info" "rpc.command.server_state" "rpc.command.ledger" \ + "tx.process" "tx.receive" "tx.apply" \ + "consensus.proposal.send" "consensus.ledger_close" \ + "consensus.accept" "consensus.accept.apply" \ + "consensus.validation.send" \ + "ledger.build" "ledger.validate" "ledger.store" \ + "peer.proposal.receive" "peer.validation.receive"; do + count=$(curl -s "$TEMPO/api/search" \ + --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ + --data-urlencode "limit=5" | + jq '.traces | length') + printf "%-35s %s traces\n" "$op" "$count" +done +``` + +### Prometheus API + +Base URL: `http://localhost:9090` + +```bash +PROM="http://localhost:9090" + +# Span call counts (from spanmetrics connector) +curl -s "$PROM/api/v1/query?query=span_calls_total" | + jq '.data.result[] | {span: .metric.span_name, count: .value[1]}' + +# Latency histogram +curl -s "$PROM/api/v1/query?query=span_duration_milliseconds_count" | + jq '.data.result[] | {span: .metric.span_name, count: .value[1]}' + +# RPC calls by command +curl -s "$PROM/api/v1/query?query=span_calls_total{span_name=~\"rpc.command.*\"}" | + jq '.data.result[] | {command: .metric["command"], count: .value[1]}' + +# Deployment-tier labels present on metrics (set by the collector's +# resource/tier processor and promoted via resource_to_telemetry_conversion). +# Expect deployment_environment and xrpl_network_type on each series. +curl -s "$PROM/api/v1/query?query=span_calls_total" | + jq '.data.result[0].metric | {deployment_environment, xrpl_network_type, service_name}' +``` + +### Grafana + +Open http://localhost:3000 (anonymous admin access enabled). + +Pre-configured dashboards: + +- **RPC Performance**: Request rates, latency percentiles by command, top commands, WebSocket rate +- **Transaction Overview**: Transaction processing rates, apply duration, peer relay, failed tx rate +- **Consensus Health**: Consensus round duration, proposer counts, mode tracking, accept heatmap +- **Ledger Operations**: Build/validate/store rates and durations, TX apply metrics +- **Peer Network**: Proposal/validation receive rates, trusted vs untrusted breakdown (requires `trace_peer=1`) + +Pre-configured datasources: + +- **Tempo**: Trace data at `http://tempo:3200` +- **Prometheus**: Metrics at `http://prometheus:9090` +- **Loki**: Log data at `http://loki:3100` (via Grafana Explore) + +--- + +## Exporting to Grafana Cloud + +Instead of (or alongside) the local backends, the collector can forward +traces, metrics, and logs to a hosted **Grafana Cloud** stack. This is a +runtime choice layered on top of the base stack — xrpld and the base +`docker-compose.yml` are unchanged. + +### Step 1: Get Grafana Cloud OTLP credentials + +From **Grafana Cloud → Connections → OpenTelemetry (OTLP)**, note the OTLP +gateway endpoint (ends in `/otlp`), the numeric instance id, and an +access-policy token with `metrics:write`, `traces:write`, and `logs:write`. + +### Step 2: Fill in the env file + +```bash +cp docker/telemetry/.env.grafanacloud.example docker/telemetry/.env.grafanacloud +# edit .env.grafanacloud: +# GRAFANA_CLOUD_OTLP_ENDPOINT=https://otlp-gateway-.grafana.net/otlp +# GRAFANA_CLOUD_INSTANCE_ID= +# GRAFANA_CLOUD_API_TOKEN= +``` + +`.env.grafanacloud` is gitignored — never commit real tokens. + +### Step 3: Start the stack with cloud export enabled + +```bash +docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml up -d +``` + +The override swaps the collector onto `otel-collector-config.grafanacloud.yaml`, +which keeps the local Tempo/Prometheus/Loki exporters and adds one +OTLP/HTTP exporter to Grafana Cloud on all three pipelines. Bring the stack +up with just the base file to return to local-only. + +### Step 4: Verify data reaches Grafana Cloud + +After exercising RPC/transaction workflows (Tests 1 or 2), open your Grafana +Cloud instance and confirm: + +- **Traces**: Explore → hosted Tempo datasource → search `{resource.service.name="xrpld"}` +- **Metrics**: Explore → hosted Prometheus/Mimir → query `span_calls_total` +- **Logs**: Explore → hosted Loki → query `{job="xrpld"}` (requires `warning`+ file logging) + +If nothing appears, check the collector logs for auth/export errors: + +```bash +docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml \ + logs otel-collector | grep -iE 'grafanacloud|401|403|export' +``` + +A `401`/`403` means the instance id or token is wrong; a connection error +means the endpoint URL is wrong or missing the `/otlp` path. + +--- + +## Test 3: Log-Trace Correlation (Phase 8) + +Phase 8 injects `trace_id` and `span_id` into xrpld's log output when +a log line is emitted within an active OTel span. This test verifies the +end-to-end log-trace correlation pipeline. + +### Step 1: Verify trace_id in log output + +After running Test 1 or Test 2 (which generate RPC spans), check the +xrpld debug.log for trace context: + +```bash +grep 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' /path/to/debug.log +``` + +Expected: log lines with `trace_id=<32hex> span_id=<16hex>` between the +severity code and the message. Example: + +``` +2024-Jan-15 10:30:45.123456 UTC RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info +``` + +Lines emitted outside of an active span (background tasks, startup) will +NOT have trace context — this is expected. + +### Step 2: Cross-check trace_id in Tempo + +Extract a `trace_id` from the log and verify it exists in Tempo: + +```bash +TRACE_ID=$(grep -o 'trace_id=[a-f0-9]\{32\}' /path/to/debug.log | head -1 | cut -d= -f2) +echo "Checking trace: $TRACE_ID" +curl -s "http://localhost:3200/api/traces/$TRACE_ID" | jq '.batches | length' +``` + +Expected result: `> 0` (the trace exists in Tempo). + +### Step 3: Verify Loki log ingestion + +The OTel Collector's filelog receiver tails xrpld's debug.log and +exports parsed entries to Loki. Verify Loki has received entries: + +```bash +# Query Loki for any xrpld logs +curl -sG "http://localhost:3100/loki/api/v1/query" \ + --data-urlencode 'query={job="xrpld"}' \ + --data-urlencode 'limit=5' | jq '.data.result | length' +``` + +Expected: > 0 results. + +### Step 4: Verify Grafana Tempo-to-Loki correlation + +1. Open Grafana at http://localhost:3000 +2. Navigate to **Explore** -> select **Tempo** datasource +3. Search for a trace (e.g., operation `rpc.command.server_info`) +4. Click **"Logs for this trace"** in the trace detail view +5. Verify that Loki log lines appear, filtered by the trace's `trace_id` + +### Step 5: Verify Grafana Loki-to-Tempo correlation + +1. In Grafana **Explore**, select **Loki** datasource +2. Query: `{job="xrpld"} |= "trace_id="` +3. In the log results, click the **TraceID** derived field link +4. Verify it navigates to the full trace in Tempo + +### Expected results + +| Check | Expected | +| ------------------------------ | ---------------------------------------- | +| `trace_id=` in debug.log | Present in log lines within active spans | +| `span_id=` in debug.log | Present alongside trace_id | +| Logs without active span | No trace_id/span_id fields | +| trace_id in Tempo | Matches a valid trace | +| Loki log ingestion | Logs visible via LogQL | +| Tempo -> Loki "Logs for trace" | Shows correlated log lines | +| Loki -> Tempo TraceID link | Navigates to correct trace | + +--- + +## Troubleshooting + +### No traces in Tempo + +1. Check otel-collector logs: + ```bash + docker compose -f docker/telemetry/docker-compose.yml logs otel-collector + ``` +2. Verify xrpld telemetry config has `enabled=1` and correct endpoint +3. Check that otel-collector port 4318 is accessible: + ```bash + curl -sf http://localhost:4318 && echo "reachable" + ``` +4. Increase `batch_delay_ms` or decrease `batch_size` in xrpld config + +### Nodes not reaching "proposing" state + +1. Check that all peer ports (51235-51240) are not in use: + ```bash + for p in 51235 51236 51237 51238 51239 51240; do + ss -tlnp | grep ":$p " && echo "port $p in use" + done + ``` +2. Verify `[ips_fixed]` lists all 6 peer ports +3. Verify `validators.txt` has all 6 public keys +4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log` +5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network) + +### Transaction not processing + +1. Verify genesis account exists: + ```bash + curl -s http://localhost:5005 \ + -d '{"method":"account_info","params":[{"account":"rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"}]}' | + jq .result.account_data.Balance + ``` +2. Check submit response for error codes +3. In standalone mode, remember to call `ledger_accept` after submitting + +### No trace_id in log output (Phase 8) + +1. Verify xrpld was built with `telemetry=ON` (`-Dtelemetry=ON` in CMake) +2. Verify `enabled=1` in the `[telemetry]` config section +3. Log lines only contain trace context when emitted inside an active span. + Background logs (startup, periodic tasks outside spans) will not have + `trace_id`/`span_id`. +4. Ensure the trace category is enabled (e.g., `trace_rpc=1` for RPC logs) + +### No logs in Loki (Phase 8) + +1. Verify the log file mount in docker-compose.yml: + ```yaml + volumes: + - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro + ``` + The mount source defaults to the repo-relative `docker/telemetry/data/logs` + (where the telemetry configs write). Override `XRPLD_LOG_DIR` to tail logs + from another root. +2. Check OTel Collector logs for filelog receiver errors: + ```bash + docker compose -f docker/telemetry/docker-compose.yml logs otel-collector | grep -i "filelog\|loki\|error" + ``` +3. Verify Loki is running: + ```bash + curl -s http://localhost:3100/ready + ``` +4. Verify the filelog receiver glob pattern matches your log files: + The default pattern is `/var/log/xrpld/*/debug.log` + +### Grafana trace-log links not working (Phase 8) + +1. Verify `tracesToLogs` is configured in the Tempo datasource provisioning + (`docker/telemetry/grafana/provisioning/datasources/tempo.yaml`) +2. Verify `derivedFields` is configured in the Loki datasource provisioning + (`docker/telemetry/grafana/provisioning/datasources/loki.yaml`) +3. Restart Grafana after changing provisioning files: + ```bash + docker compose -f docker/telemetry/docker-compose.yml restart grafana + ``` + +### Spanmetrics not appearing in Prometheus + +1. Verify otel-collector config has `spanmetrics` connector +2. Check that the metrics pipeline is configured: + ```yaml + service: + pipelines: + metrics: + receivers: [spanmetrics] + exporters: [prometheus] + ``` +3. Verify Prometheus can reach collector: + ```bash + curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets' + ``` diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy new file mode 100644 index 00000000000..f56a9474268 --- /dev/null +++ b/docker/telemetry/alloy/config.alloy @@ -0,0 +1,268 @@ +// Grafana Alloy collector config for an xrpld node. +// +// WHY THIS FILE EXISTS +// -------------------- +// Some deployments feed telemetry through Grafana Alloy instead of the +// reference OpenTelemetry Collector (otel-collector-config.grafanacloud.yaml). +// xrpld sends OTLP (traces + native beast::insight metrics) to Alloy, which +// forwards to the Grafana Cloud OTLP gateway. That path carries traces and +// native metrics, but on its own produces NO span-derived RED metrics +// (span_calls_total / span_duration_milliseconds_*): those are NOT emitted by +// xrpld -- they are derived by a spanmetrics connector from the spans. Without +// the connector a node is missing from every span-based Grafana dashboard. +// +// This config therefore mirrors the two stages the reference collector has: +// 1. resource tagging (service.name, deployment.environment, xrpl.network.type) +// 2. spanmetrics (derives the RED metrics the dashboards query) +// See docker/telemetry/otel-collector-config.grafanacloud.yaml for the +// authoritative collector equivalent; keep the dimension list in sync with it. +// +// PIPELINE +// +// HOST / SYSTEMD METRICS: +// unix exporter + custom scrape --> relabel(host) --> prometheus.remote_write +// +// xrpld OTLP: +// receiver.otlp ─▶ processor.transform.tier ─┬─▶ connector.spanmetrics ─┐ +// │ │ +// └─▶ processor.batch ◀───────┘ +// │ traces + metrics +// ▼ +// exporter.otlphttp (GC OTLP gateway) +// +// The Grafana Cloud OTLP gateway converts OTLP resource attributes to +// Prometheus labels server-side, so no otelcol.exporter.prometheus is needed. +// +// CONFIGURATION -- NO SECRETS OR TENANT IDS ARE HARD-CODED IN THIS FILE. +// All credentials, endpoints, usernames and the per-node host label are read +// from the environment (same pattern as .env.grafanacloud.example). Copy +// .env.grafanacloud-alloy.example to .env.grafanacloud-alloy, fill it in, and +// source it before starting Alloy. Required variables: +// GRAFANACLOUD_PROM_URL Prometheus remote_write push URL +// GRAFANACLOUD_PROM_USER remote_write basic-auth username (numeric stack id) +// GRAFANACLOUD_PROM_KEY remote_write basic-auth password (access token) +// GRAFANACLOUD_OTLP_URL OTLP/HTTP gateway URL, including the /otlp path +// GRAFANACLOUD_OTLP_USER OTLP basic-auth username (numeric stack id) +// GRAFANACLOUD_OTLP_KEY OTLP basic-auth password (access token) +// XRPLD_HOST_LABEL host label for this node's scraped metrics +// +// PER-DEPLOYMENT EDITS: the deployment.environment and xrpl.network.type tier +// values in otelcol.processor.transform are literals (OTTL cannot read env +// vars) -- edit them to match this node's tier and network. + +logging { + level = "info" +} + +// =========================================================================== +// HOST / SYSTEMD METRICS (Prometheus remote_write path) +// =========================================================================== + +prometheus.remote_write "grafanacloud" { + endpoint { + url = sys.env("GRAFANACLOUD_PROM_URL") + + basic_auth { + username = sys.env("GRAFANACLOUD_PROM_USER") + password = sys.env("GRAFANACLOUD_PROM_KEY") + } + } +} + +prometheus.exporter.unix "default" { + enable_collectors = ["systemd", "textfile"] + systemd { + unit_include = "(rippled|xrpld).*" + } + textfile { + directory = "/usr/local/bin/" + } +} + +prometheus.scrape "system_metrics" { + targets = prometheus.exporter.unix.default.targets + scrape_interval = "15s" + forward_to = [prometheus.relabel.hostname.receiver] +} + +prometheus.scrape "custom_rippled_metrics" { + targets = [{ + __address__ = "localhost:9256", + }] + scrape_interval = "15s" + forward_to = [prometheus.relabel.hostname.receiver] +} + +prometheus.relabel "hostname" { + forward_to = [prometheus.remote_write.grafanacloud.receiver] + + rule { + action = "replace" + source_labels = ["instance"] + replacement = sys.env("XRPLD_HOST_LABEL") + target_label = "host" + } +} + +// =========================================================================== +// xrpld OTLP -> Grafana Cloud (traces + native metrics + derived spanmetrics) +// =========================================================================== + +// Receiver: xrpld sends OTLP/HTTP on 4318 and gRPC on 4317 (localhost only). +otelcol.receiver.otlp "xrpld" { + grpc { + endpoint = "127.0.0.1:4317" + } + http { + endpoint = "127.0.0.1:4318" + } + + output { + // Both signals pass through resource tagging first so they leave with + // identical resource identity. + traces = [otelcol.processor.transform.tier.input] + metrics = [otelcol.processor.transform.tier.input] + } +} + +// Resource tagging (reference: resource/tier + resource/stripsdk). +// * service.name -> "xrpld". Also corrects the service_name="true" +// bool-binding bug on stale binaries: even an un-rebuilt node exports a +// correct service.name once it passes through here. +// * deployment.environment -> set from XRPLD_DEPLOYMENT_ENV (the collector +// IS the environment, so it is authoritative -> upsert). +// * xrpl.network.type -> set only when absent (don't overwrite the node's +// own value). OTTL `where ... == nil` gives insert (not upsert) semantics. +// * telemetry.sdk.* -> deleted (SDK noise). +otelcol.processor.transform "tier" { + error_mode = "ignore" + + // NOTE: statements are OTTL (raw strings) -- River sys.env() does NOT expand + // inside them, and OTTL has no env() converter. The tier values below are + // plain deployment labels (not secrets), so they are literals here. + // PER-DEPLOYMENT EDIT: set deployment.environment and xrpl.network.type to + // match this node's tier and network. + trace_statements { + context = "resource" + statements = [ + `set(attributes["service.name"], "xrpld")`, + `set(attributes["deployment.environment"], "prod")`, + `set(attributes["xrpl.network.type"], "mainnet") where attributes["xrpl.network.type"] == nil`, + `delete_key(attributes, "telemetry.sdk.language")`, + `delete_key(attributes, "telemetry.sdk.name")`, + `delete_key(attributes, "telemetry.sdk.version")`, + ] + } + + metric_statements { + context = "resource" + statements = [ + `set(attributes["service.name"], "xrpld")`, + `set(attributes["deployment.environment"], "prod")`, + `set(attributes["xrpl.network.type"], "mainnet") where attributes["xrpl.network.type"] == nil`, + `delete_key(attributes, "telemetry.sdk.language")`, + `delete_key(attributes, "telemetry.sdk.name")`, + `delete_key(attributes, "telemetry.sdk.version")`, + ] + } + + output { + // Traces fan out: to the batch/gateway path AND into the spanmetrics + // connector so the RED metrics are derived from the same tagged spans. + traces = [ + otelcol.processor.batch.xrpld.input, + otelcol.connector.spanmetrics.xrpld.input, + ] + // Native metrics go straight to the batch/gateway path. + metrics = [otelcol.processor.batch.xrpld.input] + } +} + +// Spanmetrics connector (reference: connectors.spanmetrics, namespace "span"). +// Derives span_calls_total and span_duration_milliseconds_* that every span +// dashboard queries. The dimension list and buckets are copied from the +// reference otel-collector-config.grafanacloud.yaml -- keep them in sync (a +// panel that groups by a dimension missing here renders empty). +otelcol.connector.spanmetrics "xrpld" { + namespace = "span" + + histogram { + unit = "ms" + explicit { + buckets = ["1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "5s"] + } + } + + // RPC / transaction dimensions. + dimension { name = "command" } + dimension { name = "rpc_status" } + dimension { name = "tx_type" } + dimension { name = "ter_result" } + dimension { name = "stage" } + dimension { name = "txq_status" } + dimension { name = "load_type" } + dimension { name = "is_batch" } + + // Consensus dimensions. + dimension { name = "consensus_mode" } + dimension { name = "close_time_correct" } + dimension { name = "consensus_state" } + dimension { name = "mode_new" } + dimension { name = "consensus_stalled" } + dimension { name = "consensus_phase" } + dimension { name = "consensus_result" } + + // Overlay / peer dimensions. + dimension { name = "local" } + dimension { name = "suppressed" } + dimension { name = "proposal_trusted" } + dimension { name = "validation_trusted" } + + // gRPC surface dimensions. + dimension { name = "method" } + dimension { name = "grpc_role" } + dimension { name = "grpc_status" } + + // ledger.acquire dimensions. + dimension { name = "outcome" } + dimension { name = "acquire_reason" } + + output { + // Derived span metrics rejoin the metric stream at the batch processor. + metrics = [otelcol.processor.batch.xrpld.input] + } +} + +// Batch traces + metrics (native and span-derived) before export. +otelcol.processor.batch "xrpld" { + timeout = "1s" + send_batch_size = 1024 + + output { + traces = [otelcol.exporter.otlphttp.grafanacloud.input] + metrics = [otelcol.exporter.otlphttp.grafanacloud.input] + } +} + +// Grafana Cloud OTLP gateway auth + exporter. The gateway converts OTLP +// resource attributes to Prometheus labels server-side. +otelcol.auth.basic "grafanacloud" { + username = sys.env("GRAFANACLOUD_OTLP_USER") + password = sys.env("GRAFANACLOUD_OTLP_KEY") +} + +otelcol.exporter.otlphttp "grafanacloud" { + client { + endpoint = sys.env("GRAFANACLOUD_OTLP_URL") + auth = otelcol.auth.basic.grafanacloud.handler + } + retry_on_failure { + enabled = true + max_elapsed_time = "5m" + } + sending_queue { + enabled = true + num_consumers = 4 + queue_size = 1000 + } +} diff --git a/docker/telemetry/docker-compose.grafanacloud.yaml b/docker/telemetry/docker-compose.grafanacloud.yaml new file mode 100644 index 00000000000..b78f188d53d --- /dev/null +++ b/docker/telemetry/docker-compose.grafanacloud.yaml @@ -0,0 +1,28 @@ +# Compose override — enable Grafana Cloud export at runtime. +# +# Layer this on top of the base stack to swap the collector onto the +# Grafana Cloud dual-export config and inject the OTLP credentials. The +# base docker-compose.yml is unchanged, so plain `up` stays local-only. +# +# Usage: +# 1. Put your three Grafana Cloud values in docker/telemetry/.env.grafanacloud +# (template: .env.grafanacloud.example). That file is gitignored. +# 2. Bring the stack up with BOTH compose files: +# docker compose -f docker/telemetry/docker-compose.yml \ +# -f docker/telemetry/docker-compose.grafanacloud.yaml up -d +# 3. To go back to local-only, bring the stack up with just the base file. +# +# Log mount: inherited from the base compose (./data/logs -> /var/log/xrpld). +# Both nodes' xrpld cfgs write to docker/telemetry/data/logs//debug.log, +# so no per-box remap is needed here. + +services: + otel-collector: + volumes: + # Mount the Grafana Cloud collector config over the default path + # (replaces local-only config for this run; file on disk untouched). + - ./otel-collector-config.grafanacloud.yaml:/etc/otel-collector-config.yaml:ro + # Secrets from the env file, injected as container env and + # resolved by the collector via ${env:...}. + env_file: + - .env.grafanacloud diff --git a/docker/telemetry/docker-compose.workload.yaml b/docker/telemetry/docker-compose.workload.yaml new file mode 100644 index 00000000000..818c65f4c58 --- /dev/null +++ b/docker/telemetry/docker-compose.workload.yaml @@ -0,0 +1,103 @@ +# Docker Compose workload harness for Phase 10 telemetry validation. +# +# Runs a 5-node validator cluster with full OTel telemetry stack: +# - 5 rippled validator nodes (consensus network) +# - OTel Collector (traces + native OTLP metrics) +# - Tempo (trace backend + search API) +# - Prometheus (metrics) +# - Loki (log aggregation for log-trace correlation) +# - Grafana (dashboards + trace/log exploration) +# +# Usage: +# # Start the harness (requires pre-built xrpld image or mount binary): +# docker compose -f docker/telemetry/docker-compose.workload.yaml up -d +# +# # Or use the orchestrator: +# docker/telemetry/workload/run-full-validation.sh +# +# Prerequisites: +# - xrpld binary built with -DXRPL_ENABLE_TELEMETRY=ON +# - Validator keys generated via generate-validator-keys.sh +# - Node configs generated by run-full-validation.sh +# +# Note: No Docker healthchecks are defined here. The orchestrator script +# (run-full-validation.sh) polls each service endpoint directly from the +# host, which avoids issues with missing curl/wget in container images. + +services: + # --------------------------------------------------------------------------- + # Telemetry Backend Stack + # --------------------------------------------------------------------------- + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otel-collector-config.yaml"] + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP (traces + beast::insight metrics) + - "8889:8889" # Prometheus metrics endpoint + - "13133:13133" # Health check + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + # Mount the validation workdir so the filelog receiver can tail node + # logs. run-full-validation.sh sets XRPLD_LOG_DIR to its workdir; the + # default matches that workdir so a bare `docker compose up` also works. + - ${XRPLD_LOG_DIR:-/tmp/xrpld-validation}:/var/log/xrpld:ro + depends_on: + - tempo + networks: + - workload-net + + tempo: + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo.yaml"] + ports: + - "3200:3200" # Tempo HTTP API + volumes: + - ./tempo.yaml:/etc/tempo.yaml:ro + - tempo-data:/var/tempo + networks: + - workload-net + + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + depends_on: + - otel-collector + networks: + - workload-net + + loki: + image: grafana/loki:3.4.2 + ports: + - "3100:3100" # Loki HTTP API + command: ["-config.file=/etc/loki/local-config.yaml"] + networks: + - workload-net + + grafana: + image: grafana/grafana:latest + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "3000:3000" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - tempo + - prometheus + - loki + networks: + - workload-net + +volumes: + tempo-data: + +networks: + workload-net: + driver: bridge diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml new file mode 100644 index 00000000000..cffe3d139ac --- /dev/null +++ b/docker/telemetry/docker-compose.yml @@ -0,0 +1,153 @@ +# Docker Compose stack for xrpld OpenTelemetry observability. +# +# Provides services for local development: +# - otel-collector: receives OTLP traces from xrpld, batches and +# forwards them to Tempo. Also tails xrpld log files +# via filelog receiver and exports to Loki. Listens on ports +# 4317 (gRPC) and 4318 (HTTP). +# - tempo: Grafana Tempo tracing backend, queryable via Grafana Explore +# on port 3000. Recommended for production (S3/GCS storage, TraceQL). +# - loki: Grafana Loki log aggregation backend for centralized log +# ingestion and log-trace correlation (Phase 8). +# - grafana: dashboards on port 3000, pre-configured with Tempo, +# Prometheus, and Loki datasources. +# +# Usage: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# +# Configure xrpld to export traces by adding to xrpld.cfg: +# [telemetry] +# enabled=1 +# endpoint=http://localhost:4318/v1/traces + +services: + # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol, + # batches them for efficiency, and forwards to Tempo for storage. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.121.0 + command: ["--config=/etc/otel-collector-config.yaml"] + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP (traces + native OTel metrics) + - "8889:8889" # Prometheus metrics (spanmetrics + OTLP) + # StatsD UDP port removed — beast::insight now uses native OTLP. + # Uncomment if using server=statsd fallback: + # - "8125:8125/udp" + volumes: + # Mount collector pipeline config (receivers → processors → exporters) + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + # Mount the xrpld log root for the filelog receiver. The telemetry + # configs write to docker/telemetry/data/logs//debug.log, so + # the default source is the repo-relative ./data/logs — user-owned and + # needing no root, so `docker compose up` works with no setup. Override + # XRPLD_LOG_DIR to point at another root (e.g. the integration test sets + # it to its own workdir). Mounted read-only so the collector only tails. + - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro + depends_on: + - tempo + - loki + networks: + - xrpld-telemetry + + # Grafana Tempo: distributed tracing backend that stores and indexes + # spans. Queryable via TraceQL in Grafana Explore. + tempo: + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo.yaml"] + ports: + - "3200:3200" # Tempo HTTP API (health check, query) + volumes: + # Mount Tempo storage and ingestion config + - ./tempo.yaml:/etc/tempo.yaml:ro + # Persistent volume for trace data (WAL + blocks) + - tempo-data:/var/tempo + networks: + - xrpld-telemetry + + # Grafana Loki for centralized log ingestion and log-trace + # correlation. Loki 3.x supports native OTLP ingestion, so the OTel + # Collector exports via otlphttp to Loki's /otlp endpoint. + # Query logs via Grafana Explore -> Loki at http://localhost:3000. + loki: + image: grafana/loki:3.4.2 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + volumes: + - loki-data:/loki + networks: + - xrpld-telemetry + + prometheus: + # Pinned to the v2.53 LTS line for reproducible, config-stable runs. + image: prom/prometheus:v2.53.3 + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + depends_on: + - otel-collector + networks: + - xrpld-telemetry + + # Grafana: visualization UI with Tempo pre-configured as a datasource. + # Anonymous admin access enabled for local development convenience. + grafana: + image: grafana/grafana:11.5.2 + # Alerting secrets/addresses (Slack webhook, alert email) come from this + # gitignored file; Grafana expands the ${VARS} referenced in the alerting + # provisioning YAML. Absent file = unset vars = receivers have no live + # destination, which is fine for a local stack. See .env.alerting.example. + env_file: + - path: .env.alerting + required: false + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth + # Remote image rendering: point Grafana at the renderer container. + # These belong on the grafana service (the server delegates renders); + # the callback URL is how the renderer fetches the panel from grafana. + - GF_RENDERING_SERVER_URL=http://renderer:8081/render + - GF_RENDERING_CALLBACK_URL=http://grafana:3000/ + # SMTP for the critical-tier email receiver and the Slack webhook / email + # address for the contact points all come from the env_file above, which + # injects them straight into the container environment for Grafana to + # expand. They are deliberately NOT duplicated here: a compose-level + # `environment:` entry is interpolated from the host shell and would + # override (and blank out) the env_file values. + ports: + - "3000:3000" # Grafana web UI + volumes: + # Auto-provision Tempo datasource and search filters on startup + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - tempo + - prometheus + - loki + - renderer + networks: + - xrpld-telemetry + + # Grafana image renderer: a sidecar that renders panels/dashboards to PNG + # for image export and alerting. Grafana calls it at http://renderer:8081. + renderer: + image: grafana/grafana-image-renderer:latest + ports: + - "8081:8081" # Renderer HTTP endpoint (called by grafana) + networks: + - xrpld-telemetry +# Named volume for Tempo trace storage (WAL and compacted blocks). +# Data persists across container restarts. Remove with: +# docker compose -f docker/telemetry/docker-compose.yml down -v +volumes: + tempo-data: + prometheus-data: + loki-data: + +# Isolated bridge network so services communicate by container name +# (e.g., the collector reaches Tempo at http://tempo:4317). +networks: + xrpld-telemetry: + driver: bridge diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json new file mode 100644 index 00000000000..17a69e34881 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -0,0 +1,1337 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Validation Send Rate", + "description": "###### What this is:\n*How often this node issues a ledger validation to the network. Each validation asserts the node fully validated one ledger.*\n\n###### How it's computed:\n*Per-second rate of validation events sent, summed per node over a 5-minute window.*\n\n###### Reading it:\n*A flat line at the network's ledger cadence is normal; higher is not better and lower means the node is validating less often.*\n\n###### Healthy range:\n*Roughly one validation per closed ledger (about 0.2-0.3/s on a healthy chain).*\n\n###### Watch for:\n*A drop to zero on a validator means it stopped validating; a value well below the close rate means validation is lagging.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::createValidationSpan`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Consensus Mode Over Time", + "description": "###### What this is:\n*Ledger-close activity split by the node's consensus mode (Proposing, Observing, Wrong Ledger, Switched Ledger).*\n\n###### How it's computed:\n*Per-second rate of ledger-close events grouped by consensus mode, per node, over a 5-minute window.*\n\n###### Reading it:\n*One dominant band is normal; a validator should sit almost entirely in Proposing, a stock node in Observing.*\n\n###### Healthy range:\n*Nearly all activity in a single expected mode.*\n\n###### Watch for:\n*Sustained time in Wrong Ledger or Switched Ledger indicates the node is out of sync or flapping between chains.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onClose`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (consensus_mode, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.ledger_close\"}[$__rate_interval])), \"series\", \"$1\", \"consensus_mode\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Consensus Proposals Sent Rate", + "description": "###### What this is:\n*How often this node proposes its candidate transaction set to peers during consensus.*\n\n###### How it's computed:\n*Per-second rate of proposal-send events, summed per node over a 5-minute window.*\n\n###### Reading it:\n*A steady line tracking the ledger cadence is healthy; only proposing (validator) nodes show activity.*\n\n###### Healthy range:\n*Steady output in step with the close rate for a proposing node; zero for a non-proposing node.*\n\n###### Watch for:\n*A proposing validator dropping to zero, or erratic spikes suggesting repeated re-proposals within rounds.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::propose`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.proposal.send\"}[$__rate_interval])), \"series\", \"Proposals / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Proposals / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Accept vs Close Rate", + "description": "###### What this is:\n*Two rates side by side: ledgers accepted after consensus versus ledger closes initiated.*\n\n###### How it's computed:\n*Per-second rate of accept events and of close events, each summed per node over a 5-minute window.*\n\n###### Reading it:\n*The two lines should overlap; read any gap between them as closes that did not finish accepting.*\n\n###### Healthy range:\n*Both lines equal and tracking the network close cadence.*\n\n###### Watch for:\n*A persistent gap where closes exceed accepts points to consensus rounds failing or timing out before acceptance.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan / RCLConsensus::Adaptor::onClose`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\"}[$__rate_interval])), \"series\", \"Accepts / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.ledger_close\"}[$__rate_interval])), \"series\", \"Closes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Validation vs Close Rate", + "description": "###### What this is:\n*Two rates side by side: validations sent versus ledger closes, so you can see whether every closed ledger gets validated.*\n\n###### How it's computed:\n*Per-second rate of validation-send events and of close events, each summed per node over a 5-minute window.*\n\n###### Reading it:\n*The lines should overlap; a validation line below the close line means validation is falling behind.*\n\n###### Healthy range:\n*Both lines equal at the network close cadence.*\n\n###### Watch for:\n*Validations persistently trailing closes, which means the node validates fewer ledgers than it closes.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::createValidationSpan / RCLConsensus::Adaptor::onClose`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.validation.send\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.ledger_close\"}[$__rate_interval])), \"series\", \"Closes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Time to Reach Consensus (p50/p95)", + "description": "###### What this is:\n*Wall-clock time for the network to agree a ledger, at the median and 95th percentile.*\n\n###### How it's computed:\n*The recorded per-round agreement time is aggregated to its 50th and 95th percentile over the window.*\n\n###### Reading it:\n*Lower is better; the p95 line shows the worst typical rounds while p50 shows the common case.*\n\n###### Healthy range:\n*Roughly 2-4 seconds on mainnet; workload- and network-dependent.*\n\n###### Watch for:\n*Rising percentiles, or a p95 that pulls far above p50, signal slow or contentious rounds under load or poor peer connectivity.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .5)", + "legendFormat": "P50 Time to Consensus [{{resource.service.instance.id}}]", + "refId": "A" + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "refId": "B", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .95)", + "legendFormat": "P95 Time to Consensus [{{resource.service.instance.id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Time to Consensus (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Average Time to Reach Consensus", + "description": "###### What this is:\n*Mean wall-clock time for the network to agree a ledger.*\n\n###### How it's computed:\n*The recorded per-round agreement time averaged over the window.*\n\n###### Reading it:\n*Lower is better; watch the trend rather than any single point.*\n\n###### Healthy range:\n*Roughly 2-4 seconds on mainnet; workload-dependent.*\n\n###### Watch for:\n*A steady upward drift indicates the network is taking longer to converge, often from load or connectivity problems.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.round_time_ms)", + "legendFormat": "Avg Time to Consensus [{{resource.service.instance.id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Time to Consensus (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "Consensus Rounds per Ledger (Establish Count)", + "description": "###### What this is:\n*How many establish-phase iterations each ledger needed before validators converged.*\n\n###### How it's computed:\n*Ledgers are counted over the window and grouped by their iteration count; each series is one count value.*\n\n###### Reading it:\n*Most weight on the '1' series is ideal; weight shifting to 2, 3 or more shows harder convergence.*\n\n###### Healthy range:\n*The large majority of ledgers converging in one iteration.*\n\n###### Watch for:\n*A growing share of ledgers needing several iterations, indicating disagreement or network stress.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::startEstablishTracing`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "refId": "A", + "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.establish_count)", + "legendFormat": "{{span.establish_count}} Rounds" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "axisLabel": "Ledgers in Window", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "decimals": 0 + }, + "overrides": [] + }, + "id": 8 + }, + { + "title": "Previous Round Time per Ledger", + "description": "###### What this is:\n*Duration of the immediately preceding consensus round, at the 95th percentile.*\n\n###### How it's computed:\n*The prior round's recorded duration is aggregated to its 95th percentile over the window.*\n\n###### Reading it:\n*Lower is better; complements the time-to-consensus panels with the previous round's cost.*\n\n###### Healthy range:\n*Comparable to the current time-to-consensus, roughly a few seconds.*\n\n###### Watch for:\n*A rising p95 means recent rounds have been expensive, often preceding a slowdown in ledger cadence.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::startRoundTracing`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.previous_round_time_ms, .95)", + "legendFormat": "P95 Previous Round [{{resource.service.instance.id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Round Time (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Position Update Duration", + "description": "###### What this is:\n*Time this node spends each round tallying disputes and updating its consensus position, at the 95th percentile.*\n\n###### How it's computed:\n*Per-round position-update durations are aggregated to their 95th percentile over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; short, flat durations are normal.*\n\n###### Healthy range:\n*A few milliseconds under normal load.*\n\n###### Watch for:\n*Sustained high durations point to heavy dispute resolution or slow convergence on close time.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::updateOurPositions`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.update_positions\"}[5m]))), \"series\", \"P95 Update\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "Ledger Close Duration", + "description": "###### What this is:\n*Time from when consensus triggers a ledger close to when the close completes, at the 95th percentile.*\n\n###### How it's computed:\n*Per-close durations are aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; compare against the round time to see how much of it the close accounts for.*\n\n###### Healthy range:\n*A few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising p95 indicates the close step is becoming a bottleneck, often under heavy transaction volume.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onClose`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.ledger_close\"}[5m]))), \"series\", \"P95 Close Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 11 + }, + { + "title": "Ledger Apply Duration (doAccept)", + "description": "###### What this is:\n*Time spent applying the agreed transaction set to build the new ledger, at the 95th percentile.*\n\n###### How it's computed:\n*Per-apply durations are aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; this is the cost of executing the ledger's transactions.*\n\n###### Healthy range:\n*A few to tens of milliseconds, scaling with the number of transactions in the ledger.*\n\n###### Watch for:\n*A rising p95 alongside high transaction counts signals apply-stage load, whether organic or from a flood of transactions.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.accept.apply\"}[5m]))), \"series\", \"P95 Apply Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "Consensus Accept Duration Heatmap", + "description": "###### What this is:\n*Distribution of how long the accept step takes across rounds, over time.*\n\n###### How it's computed:\n*Accept durations are bucketed by duration band and counted per 5-minute window, drawn as a heatmap.*\n\n###### Reading it:\n*A tight low band is healthy; brighter cells higher up mean more slow rounds.*\n\n###### Healthy range:\n*Most rounds concentrated in the lowest duration bands.*\n\n###### Watch for:\n*A widening or upward-drifting hot band flags outlier rounds that take abnormally long.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Duration (ms)", + "unit": "ms" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "id": 13 + }, + { + "title": "Close-Time Agreement Rate (Agreed vs Disagreed)", + "description": "###### What this is:\n*How often validators agreed on the ledger close time versus agreed to disagree.*\n\n###### How it's computed:\n*Per-second rate of rounds split into Agreed and Disagreed, per node, over the rate interval.*\n\n###### Reading it:\n*Mostly Agreed is healthy; occasional Disagreed is normal when validator clocks differ slightly.*\n\n###### Healthy range:\n*Overwhelmingly Agreed.*\n\n###### Watch for:\n*A sustained rise in Disagreed points to clock drift or latency spread across the validator set.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (close_time_correct, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{span_name=\"consensus.accept.apply\", consensus_mode=~\"$consensus_mode\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"close_time_correct\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Rounds / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Close Time Vote Bins & Resolution", + "description": "###### What this is:\n*Two related values: how many distinct close-time votes were seen, and the current close-time rounding granularity in ms.*\n\n###### How it's computed:\n*Both values are averaged over the window and plotted on dual axes.*\n\n###### Reading it:\n*Fewer vote bins and a fine resolution mean tight agreement; more bins and a coarse resolution mean disagreement.*\n\n###### Healthy range:\n*Few bins with a fine (about 10s) resolution.*\n\n###### Watch for:\n*Rising bins with the resolution widening (toward 120s) shows validators struggling to agree on close time.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Count / Milliseconds", + "spanNulls": 1800000 + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Vote Bins" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Resolution" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_time_vote_bins)", + "legendFormat": "Avg Vote Bins", + "refId": "A" + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_resolution_ms)", + "legendFormat": "Avg Resolution (ms)", + "refId": "B" + } + ], + "id": 15 + }, + { + "title": "Close-Time Resolution Change (per Round)", + "description": "###### What this is:\n*Whether the close-time rounding granularity moved Coarser, Finer, or stayed Steady versus the previous round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by direction of change.*\n\n###### Reading it:\n*Mostly Steady is healthy; Coarser means widening bins (more disagreement), Finer means tightening.*\n\n###### Healthy range:\n*Predominantly Steady.*\n\n###### Watch for:\n*Frequent Coarser shifts indicate the network is repeatedly failing to agree on close time.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", + "legendFormat": "{{span.resolution_direction}}", + "refId": "A" + } + ], + "id": 16, + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "increased", + "renamePattern": "Coarser (more disagreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "decreased", + "renamePattern": "Finer (better agreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "unchanged", + "renamePattern": "Steady" + } + } + ] + }, + { + "title": "Close-Time Proposal Spread (Distinct Positions per Round)", + "description": "###### What this is:\n*How spread out validators' proposed close times were, as the number of distinct positions per round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by their distinct-position count; each bar is one count value.*\n\n###### Reading it:\n*Weight on '1 distinct position' means everyone agreed; weight on 2 or more means the proposals split.*\n\n###### Healthy range:\n*Most rounds at a single distinct position.*\n\n###### Watch for:\n*A shift toward two or more distinct positions signals growing clock drift or latency across validators.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.close_time_vote_bins)", + "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", + "refId": "A" + } + ], + "id": 17 + }, + { + "title": "Consensus Outcome Distribution (per Round)", + "description": "###### What this is:\n*How consensus rounds concluded: Agreed, Moved On (partial), Expired (timeout), or No Consensus.*\n\n###### How it's computed:\n*Rounds over the window are counted and grouped by outcome, shown as shares of a pie.*\n\n###### Reading it:\n*An almost fully Agreed pie is healthy; other slices represent degraded outcomes.*\n\n###### Healthy range:\n*Nearly 100 percent Agreed.*\n\n###### Watch for:\n*A growing Moved On or Expired share signals network stress, disagreement, or connectivity loss.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value", "percent"] + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, consensus_state, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\", consensus_state!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"consensus_state\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short" + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Consensus Failures Over Time", + "description": "###### What this is:\n*Rate of non-normal consensus outcomes (moved-on plus expired) over time.*\n\n###### How it's computed:\n*Per-second rate of the moved-on and expired outcomes, summed per node over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is healthy; any sustained line is a concern.*\n\n###### Healthy range:\n*Zero under normal operation.*\n\n###### Watch for:\n*Spikes or a persistent nonzero rate indicate consensus instability that can precede ledger stalls or forks.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\", consensus_state=\"moved_on\"}[$__rate_interval])), \"series\", \"moved_on\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\", consensus_state=\"expired\"}[$__rate_interval])), \"series\", \"expired\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Failures / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 19 + }, + { + "title": "Consensus Stall Rate", + "description": "###### What this is:\n*Rate at which consensus health checks report a stalled condition versus not stalled.*\n\n###### How it's computed:\n*Per-second rate of consensus checks split by stalled flag, summed per node over a 5-minute window.*\n\n###### Reading it:\n*The Not Stalled line should carry all the weight; any Stalled line is an early warning.*\n\n###### Healthy range:\n*Stalled at zero.*\n\n###### Watch for:\n*A nonzero stalled rate surfaces stalls before they show up as validated-ledger-age alarms.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::haveConsensus`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.check\", consensus_stalled=\"true\"}[$__rate_interval])), \"series\", \"Stalled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.check\", consensus_stalled=\"false\"}[$__rate_interval])), \"series\", \"Not Stalled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Checks / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 20 + }, + { + "title": "Consensus Mode-Change Rate by Target Mode", + "description": "###### What this is:\n*How often the node switches consensus mode, broken down by the mode it switched into.*\n\n###### How it's computed:\n*Per-second rate of mode-change events grouped by target mode, per node, over a 5-minute window.*\n\n###### Reading it:\n*Near-zero is healthy; a stable node rarely changes mode.*\n\n###### Healthy range:\n*Essentially zero mode changes in steady state.*\n\n###### Watch for:\n*Frequent switches into Wrong Ledger or Switched Ledger mark an unstable node at risk of forking.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onModeChange`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (mode_new, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.mode_change\"}[$__rate_interval])), \"series\", \"$1\", \"mode_new\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Mode Changes / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 21 + }, + { + "title": "Ledger History Mismatch Rate by Reason", + "description": "###### What this is:\n*Rate of built-versus-validated ledger mismatches, broken down by why they diverged.*\n\n###### How it's computed:\n*Per-second rate of mismatch events grouped by reason, per node, over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is healthy; the reason label tells you the nature of any divergence.*\n\n###### Healthy range:\n*Zero under normal operation.*\n\n###### Watch for:\n*Any sustained rate marks a fork; the reason distinguishes close-time disagreement, sync drift, and transaction-processing differences.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgerHistoryMismatch`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Mismatches / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 22 + } + ], + "schemaVersion": 39, + "tags": ["consensus"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "label": "Tempo", + "query": "tempo", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "consensus_mode", + "label": "Consensus Mode", + "description": "Filter by consensus mode (Proposing, Observing, Wrong Ledger, Switched Ledger)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"consensus.ledger_close\"}, consensus_mode)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "close_time_correct", + "label": "Close Time Agreed", + "type": "custom", + "query": "true,false", + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "options": [ + { + "text": "All", + "value": "$__all", + "selected": true + }, + { + "text": "true", + "value": "true", + "selected": false + }, + { + "text": "false", + "value": "false", + "selected": false + } + ] + }, + { + "name": "resolution_direction", + "label": "Resolution Direction", + "type": "custom", + "query": "increased,decreased,unchanged", + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "options": [ + { + "text": "All", + "value": "$__all", + "selected": true + }, + { + "text": "increased", + "value": "increased", + "selected": false + }, + { + "text": "decreased", + "value": "decreased", + "selected": false + }, + { + "text": "unchanged", + "value": "unchanged", + "selected": false + } + ] + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Consensus Health", + "uid": "consensus-health", + "description": "What this shows: Consensus health for XRPL nodes: how reliably and quickly the network agrees each ledger, and where agreement breaks down. \u2014 Use it to: Spot stalled or slow consensus rounds and pinpoint the phase where agreement is failing.", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json new file mode 100644 index 00000000000..c0c057caf30 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -0,0 +1,604 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Transaction queue depth and capacity, fee-escalation levels, and load-factor breakdown that together set the minimum fee to enter the open ledger. \u2014 Use it to: Understand why the minimum transaction fee is rising and how full the open ledger and queue are.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Transaction Queue Depth", + "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_count\"}, \"series\", \"Queue Depth\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_max_size\"}, \"series\", \"Max Capacity\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Transactions", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Transactions Per Ledger", + "description": "###### What this is:\n*Transactions already placed in the current open ledger versus the expected per-ledger target.*\n\n###### How it's computed:\n*Instantaneous gauge readings of in-ledger count and the target count that governs fee escalation.*\n\n###### Reading it:\n*Staying at or below the expected target is normal; exceeding it triggers open-ledger fee escalation.*\n\n###### Healthy range:\n*At or under the expected per-ledger target.*\n\n###### Watch for:\n*In-ledger count persistently above target, indicating sustained congestion pushing fees up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_in_ledger\"}, \"series\", \"In Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_per_ledger\"}, \"series\", \"Expected Per Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Transactions", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Fee Escalation Levels", + "description": "###### What this is:\n*The fee levels that govern queue admission: reference (baseline), minimum processing, median, and open-ledger levels.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each fee level, shown on a log scale.*\n\n###### Reading it:\n*Open-ledger level near the reference level means cheap entry; a large gap above reference means escalation is active.*\n\n###### Healthy range:\n*Open-ledger level at or near reference during normal traffic.*\n\n###### Watch for:\n*Open-ledger level spiking far above reference, the hallmark of congestion or a fee-bidding war.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_reference_fee_level\"}, \"series\", \"Reference Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_min_processing_fee_level\"}, \"series\", \"Min Processing Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_med_fee_level\"}, \"series\", \"Median Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_open_ledger_fee_level\"}, \"series\", \"Open Ledger Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Fee Level", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "scaleDistribution": { + "type": "log", + "log": 2 + }, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Load Factor Breakdown", + "description": "###### What this is:\n*The combined load factor and its server, fee-escalation, and fee-queue contributors as unitless fee multipliers (1.0 = no load).*\n\n###### How it's computed:\n*Instantaneous gauge readings of each load-factor component.*\n\n###### Reading it:\n*Values at 1.0 mean base fees; higher values raise the fee to transact.*\n\n###### Healthy range:\n*Around 1.0 under normal conditions.*\n\n###### Watch for:\n*Combined factor climbing well above 1.0, showing the node is charging premium fees due to congestion or overload.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor\"}, \"series\", \"Combined Load Factor\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_server\"}, \"series\", \"Server\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_fee_escalation\"}, \"series\", \"Fee Escalation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_fee_queue\"}, \"series\", \"Fee Queue\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Multiplier", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "Load Factor Components", + "description": "###### What this is:\n*The individual load-factor inputs, local server load, network load, and cluster load, as unitless multipliers.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each component.*\n\n###### Reading it:\n*All at 1.0 means no load pressure from any source; a raised component identifies where load originates.*\n\n###### Healthy range:\n*Around 1.0 for each component.*\n\n###### Watch for:\n*A single component rising sharply, which pinpoints whether the pressure is local, network-wide, or cluster-driven.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_local\"}, \"series\", \"Local\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_net\"}, \"series\", \"Network\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_cluster\"}, \"series\", \"Cluster\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Multiplier", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Queue Abandonment Rate (Expired)", + "description": "###### What this is:\n*Transactions dropped from the queue because their last-ledger deadline passed before they could be included.*\n\n###### How it's computed:\n*Per-second rate of the cumulative expired-transaction counter over the dashboard's rate interval.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means submitters under-bid the escalating fee and their transactions timed out.*\n\n###### Healthy range:\n*Near 0 expirations per second.*\n\n###### Watch for:\n*Sustained expiry rate, a demand-frustration signal often coinciding with fee spikes or spam that crowds out honest traffic.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqExpired (caller TxQ.cpp)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(txq_expired_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Expired / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Expired / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Queue Admission Rejections (Dropped)", + "description": "###### What this is:\n*Transactions refused entry to the queue, broken down by reason such as queue_full.*\n\n###### How it's computed:\n*Per-second rate of the cumulative dropped-transaction counter over the dashboard's rate interval, split by reason.*\n\n###### Reading it:\n*Near zero is healthy; queue_full rejections mean the queue is at capacity and applying backpressure.*\n\n###### Healthy range:\n*Near 0 rejections per second.*\n\n###### Watch for:\n*A burst of queue_full drops, distinct from expiry, indicating the node is being flooded faster than it can drain.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqDropped (caller TxQ.cpp)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(txq_dropped_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Dropped / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["transactions", "fees"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Fee Market & TxQ", + "uid": "fee-market", + "version": 1, + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/fix_server_info_panels.py b/docker/telemetry/grafana/dashboards/fix_server_info_panels.py new file mode 100644 index 00000000000..7dd902bdf8e --- /dev/null +++ b/docker/telemetry/grafana/dashboards/fix_server_info_panels.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Fix three Server Info panels on node-health (legends + colors), matching the +conventions of the user's reference "Validated Ledger Seq — Current (Stat)" +panel. Throwaway helper (not committed). + +Operates on a dashboard JSON that was pulled FRESH from Grafana Cloud, so the +user's live layout and their Validated Ledger Seq / Build Version panels are +preserved verbatim -- only these three panels are edited in place: + + - Server State : pie -> horizontal bargauge, one bar per state (node count), + each bar coloured by state via value mappings (red/orange/yellow/blue/green). + - Peer Count : bargauge -> stat, value_and_name + per-node legend + gradient + colour (clones the reference stat panel's options/color). + - Uptime : bargauge -> stat, same stat treatment, unit seconds. + +Build Version and the Validated Ledger Seq * panels are intentionally left +untouched (the user owns those). +""" + +import json +import sys + +FILTERS = ( + 'service_instance_id=~"$node", deployment_environment=~"$deployment_environment", ' + 'xrpl_network_type=~"$xrpl_network_type", service_name=~"$service_name", ' + 'xrpl_work_item=~"$xrpl_work_item", xrpl_branch=~"$xrpl_branch", ' + 'xrpl_node_role=~"$xrpl_node_role"' +) + +DS = {"type": "prometheus", "uid": "${DS_PROMETHEUS}"} + +# state value -> (name, colour) for the Server State bars. +STATES = [ + ("0", "DISCONNECTED", "red"), + ("1", "CONNECTED", "orange"), + ("2", "SYNCING", "yellow"), + ("3", "TRACKING", "blue"), + ("4", "FULL", "green"), +] + + +def find(panels, title): + for p in panels: + if p.get("title") == title: + return p + if "panels" in p: + r = find(p["panels"], title) + if r: + return r + return None + + +def fix_server_state(p): + """Pie -> bargauge, one bar per state (node count), state-coloured.""" + expr = 'count_values("state", server_info{%s, metric="server_state"})' % FILTERS + for num, name, _ in STATES: + expr = 'label_replace(%s, "state", "%s", "state", "%s")' % (expr, name, num) + p["type"] = "bargauge" + p["targets"] = [ + { + "datasource": DS, + "expr": expr, + "instant": True, + "legendFormat": "{{state}}", + "refId": "A", + } + ] + p["options"] = { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "showUnfilled": True, + "valueMode": "color", + } + # colour each bar by its state name (value mapping sets text colour). + p["fieldConfig"] = { + "defaults": { + "color": {"mode": "fixed"}, + "displayName": "${__field.labels.state}", + "mappings": [ + {"type": "value", "options": {name: {"index": i, "color": colour}}} + for i, (_, name, colour) in enumerate(STATES) + ], + "unit": "short", + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": name}, + "properties": [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": colour}} + ], + } + for _, name, colour in STATES + ], + } + + +def to_reference_stat(p, expr, unit): + """bargauge -> stat, cloning the reference 'Current (Stat)' panel: per-node + legend, value_and_name text, continuous colour.""" + p["type"] = "stat" + p["targets"] = [ + { + "datasource": DS, + "expr": expr, + "instant": True, + "legendFormat": "{{service_instance_id}}", + "refId": "A", + } + ] + p["options"] = { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "textMode": "value_and_name", + "wideLayout": True, + } + p["fieldConfig"] = { + "defaults": { + "color": {"mode": "continuous-RdYlGr"}, + "thresholds": { + "mode": "absolute", + "steps": [{"color": "green", "value": 0}], + }, + "unit": unit, + }, + "overrides": [], + } + + +def fix(dash): + panels = dash["panels"] + fix_server_state(find(panels, "Server State")) + to_reference_stat( + find(panels, "Peer Count"), 'server_info{%s, metric="peers"}' % FILTERS, "short" + ) + to_reference_stat( + find(panels, "Uptime"), 'server_info{%s, metric="uptime"}' % FILTERS, "s" + ) + return dash + + +def _dump(obj, lvl=0): + pad = " " * lvl + pad1 = " " * (lvl + 1) + if isinstance(obj, dict): + if not obj: + return "{}" + return ( + "{\n" + + ",\n".join( + "%s%s: %s" % (pad1, json.dumps(k), _dump(v, lvl + 1)) + for k, v in obj.items() + ) + + "\n" + + pad + + "}" + ) + if isinstance(obj, list): + if not obj: + return "[]" + if all(isinstance(x, (str, int, float, bool)) or x is None for x in obj): + return "[" + ", ".join(json.dumps(x) for x in obj) + "]" + return ( + "[\n" + + ",\n".join("%s%s" % (pad1, _dump(x, lvl + 1)) for x in obj) + + "\n" + + pad + + "]" + ) + return json.dumps(obj) + + +# Runtime-only keys the Grafana API returns that must NOT be written to the +# repo file (they are instance state, not dashboard definition). +_RUNTIME_KEYS = ("id", "version", "liveNow") + + +def main(): + """argv: [dest2.json ...] + + The live pull is the source of truth for layout + the user's panels. For each + dest we start from that dest's EXISTING top-level shape (to keep repo-only + keys like ``links`` and avoid importing runtime keys), then replace only the + ``panels`` array with the fixed live layout.""" + live = json.load(open(sys.argv[1])) + fix(live) + for dest in sys.argv[2:]: + base = json.load(open(dest)) # keep dest's own top-level keys + base["panels"] = live["panels"] # adopt live layout + fixed panels + for k in _RUNTIME_KEYS: + base.pop(k, None) + out = _dump(base) + "\n" + json.loads(out) + open(dest, "w").write(out) + print("wrote", dest, "(%d top-level panels)" % len(base["panels"])) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json new file mode 100644 index 00000000000..516d90f5c02 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -0,0 +1,690 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue. \u2014 Use it to: Find job types that are backing up or running slowly and causing internal processing delays.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Current Job Latency (p99 Gauge)", + "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`", + "type": "gauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Job Throughput Rate (Per Second)", + "description": "###### What this is:\n*Rate of jobs queued, started, and finished across all job types.*\n\n###### How it's computed:\n*Per-second rate of each cumulative job counter over a 5-minute window.*\n\n###### Reading it:\n*Queued, started, and finished tracking together means the queue keeps up.*\n\n###### Healthy range:\n*Workload-dependent; the three rates should stay roughly equal.*\n\n###### Watch for:\n*Queued rate persistently above finished rate, which indicates a growing backlog.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued / recordJobStarted / recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Per-Job-Type Queued Rate", + "description": "###### What this is:\n*Rate of jobs entering the queue, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the queued-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Identifies which job types generate the most queue activity.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single job type surging unexpectedly, which can point to a flood of a particular request or peer message.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Per-Job-Type Finish Rate", + "description": "###### What this is:\n*Rate of jobs completing, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the finished-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Compare against the queued rate per type to spot which types are falling behind.*\n\n###### Healthy range:\n*Workload-dependent; should match the per-type queued rate.*\n\n###### Watch for:\n*A type whose finish rate lags its queued rate, revealing where the backlog concentrates.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Job Queue Wait Time", + "description": "###### What this is:\n*Distribution of how long jobs sit in the queue before a worker picks them up (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; a widening gap between p75 and p99 signals occasional stalls.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait, an early sign of worker-thread saturation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p75 Wait\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Job Execution Time", + "description": "###### What this is:\n*Distribution of how long jobs run once started (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p75 Exec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Per-Job-Type Execution Time (p99)", + "description": "###### What this is:\n*The 10 slowest job types ranked by p99 execution time.*\n\n###### How it's computed:\n*p99 derived from the run-time histogram per job_type over a 5-minute window, top 10 selected.*\n\n###### Reading it:\n*Highlights which job types cost the most CPU time at the tail.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A job type whose p99 grows over time, indicating a slow or degrading operation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "Transaction Overflow Rate", + "description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate \u2014 the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "custom": { + "axisLabel": "Overflows / Min", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 8 + } + ], + "schemaVersion": 39, + "tags": ["node", "jobqueue"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "job_type", + "label": "Job Type", + "description": "Filter by job type", + "type": "query", + "query": "label_values(job_queued_total, job_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Job Queue Analysis", + "uid": "job-queue", + "version": 1, + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json new file mode 100644 index 00000000000..63ec3507e18 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -0,0 +1,1009 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange. \u2014 Use it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Ledger Data \u2014 Ledger", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Data \u2014 Transaction", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Data \u2014 Account State", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Traffic \u2014 Ledger", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Traffic \u2014 Transaction", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Traffic \u2014 Account State", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject \u2014 Ledger", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject \u2014 Transaction", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject \u2014 Account State", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Messages \u2014 Ledger", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Messages \u2014 Transaction", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Messages \u2014 Account State", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Messages \u2014 Specials", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject \u2014 Specials", + "description": "###### What this is:\n*Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["ledger", "sync"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Ledger Data & Sync", + "uid": "ledger-data-sync", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json new file mode 100644 index 00000000000..926e54af632 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Ledger Build Rate", + "description": "###### What this is:\n*How many new ledgers this node finishes building per second.*\n\n###### How it's computed:\n*Per-second rate of completed ledger-build operations, averaged over 5 minutes and split by node.*\n\n###### Reading it:\n*A steady flat line; the value should track the network close cadence.*\n\n###### Healthy range:\n*About 0.2-0.3 ledgers/sec on mainnet (roughly one every 3-5s); workload-dependent on test networks.*\n\n###### Watch for:\n*A drop toward zero (node fell out of sync or stalled) or a value well above the network rate (rebuilding history).*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[$__rate_interval])), \"series\", \"Builds / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Ledger Build Duration", + "description": "###### What this is:\n*The time taken to build a single ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of ledger-build durations over a 5-minute window, per node.*\n\n###### Reading it:\n*Lower is better; the line should stay well under the ledger interval.*\n\n###### Healthy range:\n*Typically tens to low hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained rises approaching the close interval, which indicate heavy transaction sets or disk/I/O pressure.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m]))), \"series\", \"P95 Build Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Validation Rate", + "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Ledger Build Duration Heatmap", + "description": "###### What this is:\n*The full distribution of ledger-build times over the window, not just a single percentile.*\n\n###### How it's computed:\n*Counts of ledger builds falling in each duration band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; scattered high cells mean occasional slow builds.*\n\n###### Healthy range:\n*Most mass concentrated in the low-millisecond bands; workload-dependent.*\n\n###### Watch for:\n*A second cluster of hot cells at high durations (bimodal build times) hidden by percentile charts.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Duration (ms)" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + } + }, + { + "title": "Transaction Apply Duration", + "description": "###### What this is:\n*The time spent applying the agreed transaction set into the new ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of transaction-apply durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is a large share of total build time.*\n\n###### Healthy range:\n*A few to tens of milliseconds; scales with transaction volume per ledger.*\n\n###### Watch for:\n*Spikes during large or expensive transaction sets, which push out overall ledger build time.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[5m]))), \"series\", \"P95 tx.apply\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Transaction Apply Rate", + "description": "###### What this is:\n*How often the transaction-apply phase runs per second (once per ledger build).*\n\n###### How it's computed:\n*Per-second rate of transaction-apply operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should track the ledger build rate almost exactly.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent.*\n\n###### Watch for:\n*Divergence from the build rate, which would indicate a metric or pipeline anomaly.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[$__rate_interval])), \"series\", \"tx.apply / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Store Rate", + "description": "###### What this is:\n*How often completed ledgers are written into ledger history per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-store operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should match the build rate during normal operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; can burst higher while backfilling history.*\n\n###### Watch for:\n*A store rate below the build rate (storage falling behind) or a stall at zero.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::storeLedger`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.store\"}[$__rate_interval])), \"series\", \"Stores / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Build vs Close Duration", + "description": "###### What this is:\n*Ledger build time compared with the total consensus ledger-close time, both at the 95th percentile.*\n\n###### How it's computed:\n*Two 95th-percentile duration series over 5 minutes: ledger construction and the full consensus close, per node.*\n\n###### Reading it:\n*Build should sit below close; the gap is consensus overhead outside construction.*\n\n###### Healthy range:\n*Close a bit above build; both workload-dependent and under the round interval.*\n\n###### Watch for:\n*A widening gap (consensus-pipeline overhead growing) or build time approaching close time.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`buildLedgerImpl ; RCLConsensus::Adaptor::onClose`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m]))), \"series\", \"P95 ledger.build\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.ledger_close\"}[5m]))), \"series\", \"P95 consensus.ledger_close\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["ledger"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Ledger Operations", + "uid": "ledger-operations", + "refresh": "10s", + "description": "What this shows: Ledger construction, validation, and storage activity and timing for this node. \u2014 Use it to: Confirm ledgers are being built, validated, and stored on schedule and find the slow stage when they are not." +} diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json new file mode 100644 index 00000000000..af9831d2ea8 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -0,0 +1,988 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Peer connectivity and overlay bandwidth for this node: peer counts, disconnects, total bytes and messages exchanged, and the transaction, proposal, and validation traffic that flows across the peer-to-peer network. \u2014 Use it to: Gauge peer health and overall overlay bandwidth and see which message types dominate network traffic.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Active Peers", + "description": "###### What this is:\n*Number of active inbound and outbound peer connections the node currently holds.*\n\n###### How it's computed:\n*Current value of the inbound and outbound active-peer counts per node.*\n\n###### Reading it:\n*Outbound is what the node dials out; inbound is what others open to it. Both should be stable.*\n\n###### Healthy range:\n*roughly 10-21 outbound and 0-85 inbound on mainnet, depending on config.*\n\n###### Watch for:\n*Outbound dropping toward zero (isolation) or inbound pinned at the limit with churn (connection pressure).*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Inbound Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Outbound Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Peers", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Peer Disconnects", + "description": "###### What this is:\n*Peer connections that dropped, as a per-interval increase.*\n\n###### How it's computed:\n*Disconnect events per rate-interval, per node.*\n\n###### Reading it:\n*A flat or slowly rising line is normal; the slope matters more than the absolute value.*\n\n###### Healthy range:\n*workload-dependent; slow, steady growth.*\n\n###### Watch for:\n*Sharp step-ups in the slope (network instability, resource exhaustion, or many peers dropping the node at once).*\n\n###### Source:\n[OverlayImpl.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.h)\n\n###### Function:\n`OverlayImpl::Stats ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(increase(overlay_peer_disconnects{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Disconnects per interval", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Total Network Bytes", + "description": "###### What this is:\n*Total bytes received and sent across all peer connections.*\n\n###### How it's computed:\n*Per-second in/out byte rate per node.*\n\n###### Reading it:\n*Overall bandwidth footprint; in and out usually track network activity together.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Sudden sustained jumps not matched by ledger or transaction activity (relay storms or a noisy peer).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Total Network Messages", + "description": "###### What this is:\n*Total messages received and sent across all peer connections.*\n\n###### How it's computed:\n*Per-second in/out message rate per node.*\n\n###### Reading it:\n*Overall message throughput of the overlay; complements the byte totals.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Message count climbing far faster than bytes (many tiny messages, possible flooding).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Transaction Traffic", + "description": "###### What this is:\n*Transaction relay messages in and out, plus duplicate transaction messages received.*\n\n###### How it's computed:\n*Per-second message rate for the transaction and transaction-duplicate categories.*\n\n###### Reading it:\n*In/out rise with network transaction volume; duplicates are transactions the node already had.*\n\n###### Healthy range:\n*workload-dependent; duplicates a modest fraction of inbound.*\n\n###### Watch for:\n*Duplicate inbound approaching or exceeding unique inbound (redundant relay), or a sharp spike suggesting transaction flooding.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Proposal Traffic", + "description": "###### What this is:\n*Consensus proposal messages in/out, plus untrusted and duplicate proposal messages received.*\n\n###### How it's computed:\n*Per-second message rate for the proposal, proposal-untrusted, and proposal-duplicate categories.*\n\n###### Reading it:\n*Trusted in/out track consensus rounds; untrusted come from validators not on this node's trusted list.*\n\n###### Healthy range:\n*workload-dependent; untrusted and duplicates low relative to trusted.*\n\n###### Watch for:\n*High untrusted (trusted-list misconfiguration) or high duplicates (inefficient relay or proposal spam).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_untrusted_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Untrusted In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Validation Traffic", + "description": "###### What this is:\n*Validation messages in/out, plus untrusted and duplicate validation messages received.*\n\n###### How it's computed:\n*Per-second message rate for the validation, validation-untrusted, and validation-duplicate categories.*\n\n###### Reading it:\n*Trusted validations should arrive steadily each ledger; untrusted come from non-trusted validators.*\n\n###### Healthy range:\n*workload-dependent; untrusted and duplicates low relative to trusted.*\n\n###### Watch for:\n*Rising untrusted or duplicate validations (trusted-list health issues or validation spam).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_untrusted_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Untrusted In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Overlay Traffic by Category (Bytes In)", + "description": "###### What this is:\n*Top overlay traffic categories ranked by inbound bytes, excluding the all-traffic total.*\n\n###### How it's computed:\n*Top categories by inbound byte rate per node.*\n\n###### Reading it:\n*Shows which message types dominate receive bandwidth right now.*\n\n###### Healthy range:\n*workload-dependent; transactions, proposals, and validations typically lead on a synced node.*\n\n###### Watch for:\n*A fetch or ledger-data category topping the list (sync activity) or an unexpected category dominating.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(topk(10, label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_overlay_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_overlay_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ping_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ping_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(status_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"status_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(havetxset_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"havetxset_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledgerdata_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledgerdata_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_set_candidate_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_set_candidate_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"set_get_bytes_in\",\"\",\"\")), \"series\", \"$1\", \"__name__\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "transactions_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transactions" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proposals_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Proposals" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "validations_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Validations" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "overhead_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Overhead" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "overhead_overlay_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Overhead Overlay" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ping_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "getobject_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Get Object" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "havetxset_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Have Tx Set" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledgerdata_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Data" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Data Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Data Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_account_state_node_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Account State Node Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_account_state_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Account State Node Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_transaction_node_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transaction Node Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_transaction_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transaction Node Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_transaction_set_candidate_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Tx Set Candidate Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_account_state_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Account State Node Share (Legacy)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_transaction_set_candidate_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Tx Set Candidate Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_transaction_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transaction Node Share (Legacy)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "set_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Set Get" + } + ] + } + ] + } + }, + { + "title": "Duplicate Traffic (Wasted Bandwidth)", + "description": "###### What this is:\n*Throughput of duplicate transaction, proposal, and validation traffic: messages the node had already seen and discarded.*\n\n###### How it's computed:\n*Per-second rate of the duplicate byte counters for each category, in and out.*\n\n###### Reading it:\n*Lower is better; this is bandwidth spent on redundant relays.*\n\n###### Healthy range:\n*workload-dependent; a small fraction of total traffic.*\n\n###### Watch for:\n*Duplicate rate climbing toward the same order as useful traffic (poor relay topology or redundant flooding).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Duplicate Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals Duplicate Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations Duplicate Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Throughput", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "All Traffic Categories (Detail)", + "description": "###### What this is:\n*The busiest overlay categories by inbound byte rate over time, excluding the all-traffic total.*\n\n###### How it's computed:\n*Per-second inbound byte rate for the busiest categories, ranked, excluding the all-traffic total.*\n\n###### Reading it:\n*Time-series companion to the category bar view; shows how the traffic mix shifts over the window.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A category ramping up and staying high, or the mix suddenly changing (sync, spam, or a misbehaving peer).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(15, label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_cas_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_cas_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_fetch_pack_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_fetch_pack_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_ledger_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_ledger_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transactions_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(have_transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"have_transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_set_candidate_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_set_candidate_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_set_candidate_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_set_candidate_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_cluster_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_cluster_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_manifest_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_manifest_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_overlay_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_overlay_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proof_path_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proof_path_request_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proof_path_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proof_path_response_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_duplicate_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_untrusted_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_untrusted_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(replay_delta_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"replay_delta_request_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(replay_delta_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"replay_delta_response_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(requested_transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"requested_transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"set_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(set_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"set_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(squelch_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"squelch_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(squelch_ignored_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"squelch_ignored_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(squelch_suppressed_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"squelch_suppressed_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(transactions_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"transactions_duplicate_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(unknown_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"unknown_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_duplicate_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_untrusted_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_untrusted_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validator_lists_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validator_lists_bytes_in\",\"\",\"\")), \"series\", \"$1\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Throughput", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["network"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Network Traffic", + "uid": "network-traffic", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json new file mode 100644 index 00000000000..5b2b33c05e6 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -0,0 +1,5107 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Operational health of an XRPL node: sync state, ledger progress, caches, storage, job queue, and network economy. \u2014 Use it to: Get an at-a-glance read on whether the node is healthy, synced, and keeping up with the network.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How old the most recently validated ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the validated-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the network close interval.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*Values above 20 seconds mean the node is falling behind the network.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 67, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Validated Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Validated Ledger Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How old the most recently published ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the published-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; should track close to the validated ledger age.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*A growing gap above the validated age indicates the publish pipeline is backing up.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 68, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Published Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Published Ledger Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How often the node requests a ledger from its peers.*\n\n###### How it's computed:\n*Per-second rate of ledger-fetch requests over a 5-minute window.*\n\n###### Reading it:\n*Near zero in steady state; elevated while catching up.*\n\n###### Healthy range:\n*Close to zero once fully synced.*\n\n###### Watch for:\n*A sustained high rate means the node is repeatedly missing ledgers and back-filling from peers.*\n\n###### Source:\n[InboundLedgers.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedgers.cpp)\n\n###### Function:\n`InboundLedgersImp`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 69, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetches / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Fetch Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which a locally built ledger hash fails to match the network-validated hash.*\n\n###### How it's computed:\n*Per-second rate of history-mismatch events over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is the only healthy reading.*\n\n###### Healthy range:\n*Zero.*\n\n###### Watch for:\n*Any nonzero value indicates consensus divergence or database corruption and warrants immediate investigation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgerHistoryMismatch`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 70, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Mismatches / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger History Mismatches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The lag between published and validated ledger ages, in seconds.*\n\n###### How it's computed:\n*Published ledger age minus validated ledger age, as a single derived value.*\n\n###### Reading it:\n*Near zero is healthy; a positive value is how far publishing trails validation.*\n\n###### Healthy range:\n*Close to zero.*\n\n###### Watch for:\n*A growing gap means the publish pipeline is falling behind and subscribers may see stale data.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 71, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Publish Gap\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Publish Gap", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Latency of the I/O service loop, at the 95th percentile in milliseconds.*\n\n###### How it's computed:\n*I/O loop latency samples aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; it reflects how promptly queued I/O work runs.*\n\n###### Healthy range:\n*A few milliseconds; brief spikes are tolerable.*\n\n###### Watch for:\n*Sustained values above a few hundred ms indicate thread-pool saturation or blocking operations.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 72, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(ios_latency_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 I/O Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "I/O Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 121, + "title": "Operating Mode", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Fraction of recent wall-clock time the node spent in each operating mode (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Per-second rate of each per-mode duration counter divided by the sum of all five mode rates, giving each mode's time share.*\n\n###### Reading it:\n*The Full share should sit at or near 1.0 and dominate; other shares should be near 0.*\n\n###### Healthy range:\n*Full share close to 1.0.*\n\n###### Watch for:\n*Share accumulating in Syncing, Connected, or Disconnected means the node is not staying fully synced.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time Share", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 74, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + } + ], + "title": "Operating Mode (Time Share)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative count of transitions into each operating mode.*\n\n###### How it's computed:\n*Current value of each per-mode transition counter, plotted as lines.*\n\n###### Reading it:\n*Flat lines are healthy; steps up mean the node changed mode.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Transitions", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 75, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + } + ], + "title": "Operating Mode Transitions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of change of time spent in Full and Tracking modes, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of the Full and Tracking duration counters, scaled to seconds.*\n\n###### Reading it:\n*The Full line near 1.0 means the node gains a second of Full-mode time each wall-clock second.*\n\n###### Healthy range:\n*Full rate close to 1.0, Tracking near 0.*\n\n###### Watch for:\n*A Full rate below 1.0 means the node is spending time in other modes.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate (s/s)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 76, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Full Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Tracking Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "State Duration Rate (Full vs Tracking)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 77, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"% Time in Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "% Time in Full", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The node's operating mode over time as a colored timeline (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Current value of the server-state gauge mapped to a mode name; equal consecutive samples are merged into a single band.*\n\n###### Reading it:\n*A solid green Full band across the window is the goal; other colors mark periods the node was not fully synced.*\n\n###### Healthy range:\n*Continuously Full (green).*\n\n###### Watch for:\n*Bands of Syncing, Connected, or Disconnected, which pinpoint exactly when the node dropped out of Full.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "Disconnected" + } + }, + "type": "value" + }, + { + "options": { + "1": { + "color": "orange", + "text": "Connected" + } + }, + "type": "value" + }, + { + "options": { + "2": { + "color": "yellow", + "text": "Syncing" + } + }, + "type": "value" + }, + { + "options": { + "3": { + "color": "blue", + "text": "Tracking" + } + }, + "type": "value" + }, + { + "options": { + "4": { + "color": "green", + "text": "Full" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 78, + "options": { + "alignValue": "left", + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", + "legendFormat": "State [{{service_instance_id}}]", + "refId": "A" + } + ], + "title": "Operating Mode (State Timeline)", + "type": "state-timeline" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 122, + "title": "NodeStore I/O", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Operations / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 86 + }, + "id": 80, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Reads Total\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]), \"series\", \"Reads Hit (cache)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval]), \"series\", \"Writes Total\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "NodeStore Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Operations / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 86 + }, + "id": 81, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / ignoring(metric) rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Cache Hit Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "NodeStore Cache Hit Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 82, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_load\"}, \"series\", \"Write Load\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_queue\"}, \"series\", \"Read Queue\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Write Load & Read Queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Throughput", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 83, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_read_bytes\"}[$__rate_interval]), \"series\", \"Bytes Read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_written_bytes\"}[$__rate_interval]), \"series\", \"Bytes Written\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Bytes Read/Written", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Threads / Bundle", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 110 + }, + "id": 84, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_running\"}, \"series\", \"Read Threads Running\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_total\"}, \"series\", \"Read Threads Total\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_request_bundle\"}, \"series\", \"Read Request Bundle\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "NodeStore Read Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Threads / Bundle", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 110 + }, + "id": 85, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6, \"series\", \"Read Busy Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "NodeStore Read Busy Ratio", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 123, + "title": "Jobs", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of jobs currently waiting in the internal work queue.*\n\n###### How it's computed:\n*Current value of the job-count gauge, plotted over time.*\n\n###### Reading it:\n*Lower is better; a near-flat low line means the node keeps up with its workload.*\n\n###### Healthy range:\n*Low single digits at idle, brief bumps under load.*\n\n###### Watch for:\n*A sustained high depth means the node cannot process work fast enough, common during replay, heavy RPC, or a request flood.*\n\n###### Source:\n[JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Jobs", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 123 + }, + "id": 87, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(jobq_job_count{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Job Queue Depth\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Job Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Execution time for the most critical job types at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type execution durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; each line is one job type's run time.*\n\n###### Healthy range:\n*A few to tens of milliseconds for most jobs; workload-dependent.*\n\n###### Watch for:\n*Spikes on key jobs (accept, transaction, write) indicate processing bottlenecks.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 123 + }, + "id": 88, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Accept Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Advance Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Transaction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"writeObjects\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Write Objects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"heartbeat\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Heartbeat\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"sweep\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Sweep\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"trustedValidation\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Validation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"trustedProposal\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Proposal\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"publishNewLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Publish New Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"clientRPC\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Client RPC\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Ledger Data\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "K" + } + ], + "title": "Key Jobs Execution Time (q$quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time critical jobs wait in the queue before running, at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type queue-wait durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; each line is one job type's wait time.*\n\n###### Healthy range:\n*Near zero to a few milliseconds when the queue is keeping up.*\n\n###### Watch for:\n*High waits mean the queue is backlogged and jobs are scheduled late.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Wait Time (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 135 + }, + "id": 89, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Accept Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Advance Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Transaction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"writeObjects\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Write Objects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"heartbeat\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Heartbeat\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"sweep\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Sweep\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"trustedValidation\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Validation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"trustedProposal\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Proposal\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"publishNewLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Publish New Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"clientRPC\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Client RPC\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Ledger Data\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "K" + } + ], + "title": "Key Jobs Dequeue Wait Time (q$quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Execution time for every non-special job type at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type execution durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; this is the full picture behind the Key Jobs view.*\n\n###### Healthy range:\n*A few to tens of milliseconds for most jobs.*\n\n###### Watch for:\n*Broad elevation across many job types indicates systemic processing load.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 147 + }, + "id": 90, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (service_instance_id, le, job_type, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "All Jobs Execution Time (Detail)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Queue wait time for every non-special job type at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type queue-wait durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; this is the full picture behind the Key Jobs wait view.*\n\n###### Healthy range:\n*Near zero to a few milliseconds when the queue keeps up.*\n\n###### Watch for:\n*High waits across many job types indicate systemic queue congestion.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Wait Time (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 163 + }, + "id": 91, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (service_instance_id, le, job_type, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "All Jobs Dequeue Wait (Detail)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 179 + }, + "id": 124, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Hit rates for the SLE, Ledger, and AcceptedLedger caches, from 0 to 1.*\n\n###### How it's computed:\n*Current values of the per-cache hit-rate gauges, plotted as lines.*\n\n###### Reading it:\n*Higher is better; each line is the fraction of lookups served from cache.*\n\n###### Healthy range:\n*Above roughly 0.8 in steady state.*\n\n###### Watch for:\n*Low or falling hit rates indicate cache thrashing and extra back-end reads.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hit Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 180 + }, + "id": 93, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"SLE_hit_rate\"}, \"series\", \"SLE Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"ledger_hit_rate\"}, \"series\", \"Ledger Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"AL_hit_rate\"}, \"series\", \"AcceptedLedger Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "Cache Hit Rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Entry counts for the TreeNode cache and track set, the FullBelow cache, and the AcceptedLedger cache.*\n\n###### How it's computed:\n*Current values of the per-cache size gauges, plotted as lines.*\n\n###### Reading it:\n*Stable lines are normal; sizes grow with working set and shrink after sweeps.*\n\n###### Healthy range:\n*Stable within configured limits.*\n\n###### Watch for:\n*Unbounded growth suggests memory pressure or a cache not being swept.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 180 + }, + "id": 94, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"treenode_cache_size\"}, \"series\", \"TreeNode Cache\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"treenode_track_size\"}, \"series\", \"TreeNode Track\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"fullbelow_size\"}, \"series\", \"FullBelow\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"AL_size\"}, \"series\", \"AcceptedLedger Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + } + ], + "title": "Cache Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of entries in the FullBelowCache, which tracks subtrees known to be fully present locally.*\n\n###### How it's computed:\n*Current value of the cache size gauge, plotted over time.*\n\n###### Reading it:\n*A stable size is normal; it grows during acquisition and is trimmed by sweeps.*\n\n###### Healthy range:\n*Stable within its configured bound.*\n\n###### Watch for:\n*Unbounded growth suggests the cache is not being swept.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 192 + }, + "id": 95, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(node_family_full_below_cache_size{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"FullBelowCache Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "FullBelowCache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Hit-rate percentage for the FullBelowCache.*\n\n###### How it's computed:\n*Current value of the cache hit-rate gauge.*\n\n###### Reading it:\n*Higher is better; it shows how often cached subtree knowledge is reused.*\n\n###### Healthy range:\n*Above roughly 50 percent in steady state.*\n\n###### Watch for:\n*A low hit rate during steady state means redundant subtree work and warrants investigation.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 192 + }, + "id": 96, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": true + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": true, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "FullBelowCache Hit Rate", + "type": "gauge" + } + ], + "title": "Caches", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 180 + }, + "id": 125, + "title": "Server Info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current state of each XRPL node. Green = FULL (healthy); orange/yellow = syncing in progress; red = disconnected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "dark-red", + "index": 0, + "text": "DISCONNECTED" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "CONNECTED" + }, + "2": { + "color": "orange", + "index": 2, + "text": "SYNCING" + }, + "3": { + "color": "light-blue", + "index": 3, + "text": "TRACKING" + }, + "4": { + "color": "green", + "index": 4, + "text": "FULL" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DISCONNECTED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CONNECTED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SYNCING" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "TRACKING" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FULL" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 181 + }, + "id": 98, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", + "instant": false, + "legendFormat": "{{service_instance_id}} [{{xrpl_branch}}]", + "queryType": "range", + "range": true, + "refId": "A" + } + ], + "title": "Server State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the server process has been running, in seconds.*\n\n###### How it's computed:\n*Current value of the uptime gauge.*\n\n###### Reading it:\n*Higher is generally better; a reset to a small value means the process restarted.*\n\n###### Healthy range:\n*Continuously increasing.*\n\n###### Watch for:\n*An unexpected drop to near zero indicates a restart or crash.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "dthms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 181 + }, + "id": 99, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"uptime\"}", + "instant": true, + "legendFormat": "{{service_instance_id}}", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Total connected peers, inbound plus outbound.*\n\n###### How it's computed:\n*Current value of the peer-count gauge.*\n\n###### Reading it:\n*A stable count in the healthy range is good; too few limits connectivity.*\n\n###### Healthy range:\n*Workload- and config-dependent, typically 10 or more.*\n\n###### Watch for:\n*A sudden drop points to network or connectivity problems; an unusually high inbound count can indicate connection-flood pressure.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "green", + "value": 10 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 193 + }, + "id": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peers\"}", + "instant": true, + "legendFormat": "{{service_instance_id}}", + "refId": "A" + } + ], + "title": "Peer Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Sequence number of the current open ledger.*\n\n###### How it's computed:\n*Current value of the open-ledger index gauge.*\n\n###### Reading it:\n*Should climb steadily; the gap above the validated sequence is the ledgers in flight.*\n\n###### Healthy range:\n*One or two ahead of the validated sequence.*\n\n###### Watch for:\n*A large or growing gap above the validated sequence means validation is lagging behind ledger creation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 193 + }, + "id": 103, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"ledger_current_index\"}, \"series\", \"Current Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Current Ledger Index", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "service_instance_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Seq" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 205 + }, + "id": 58, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})", + "instant": true, + "legendFormat": "Spread", + "queryType": "instant", + "range": false, + "refId": "A" + } + ], + "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 205 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort_desc(scalar(max(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})) - server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})", + "instant": true, + "legendFormat": "{{service_instance_id}}", + "queryType": "instant", + "range": false, + "refId": "A" + } + ], + "title": "Validated Ledger Seq \u2014 Current (Stat)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The running server's build version string.*\n\n###### How it's computed:\n*Read from the version label of the build-info metric (its value is always 1).*\n\n###### Reading it:\n*Confirms which version each node is running.*\n\n###### Healthy range:\n*The expected release version across all nodes.*\n\n###### Watch for:\n*A node on an unexpected or mismatched version in a fleet.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerBuildInfoGauge`", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 212 + }, + "id": 102, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count by (version) (build_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})", + "instant": false, + "legendFormat": "{{version}}", + "queryType": "range", + "range": true, + "refId": "A" + } + ], + "title": "Build Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ms", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 212 + }, + "id": 105, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}, \"series\", \"Converge Time ms\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Last Close \u2014 Converge Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Proposers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 224 + }, + "id": 104, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_proposers\"}, \"series\", \"Proposers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Last Close \u2014 Proposers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*On-disk size of the NuDB object-store back end, in bytes.*\n\n###### How it's computed:\n*Current value of the NuDB storage-size gauge, plotted over time.*\n\n###### Reading it:\n*A smoothly growing line is normal; the slope is the data growth rate.*\n\n###### Healthy range:\n*Gradual growth consistent with retained ledger history.*\n\n###### Watch for:\n*A sudden jump in growth rate can indicate runaway storage or an unexpected back-fill.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStorageDetailGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Size", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "decbytes" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 224 + }, + "id": 106, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(storage_detail{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"nudb_bytes\"}, \"series\", \"NuDB Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "NuDB Storage Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Live instance counts for the busiest internal object types.*\n\n###### How it's computed:\n*Current per-type instance counts, showing the top 15 types over time.*\n\n###### Reading it:\n*Stable lines are healthy; each line is one object type's live count.*\n\n###### Healthy range:\n*Steady counts that rise and fall with load.*\n\n###### Watch for:\n*A single type climbing without bound suggests memory pressure or a leak.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerObjectCountGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Instances", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 236 + }, + "id": 107, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": ["last", "max"], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(15, object_count{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", type=~\"$type\"}), \"series\", \"$1\", \"type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Object Instance Counts", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 252 + }, + "id": 126, + "title": "Complete Ledgers & DB", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The contiguous ranges of ledgers the node holds locally.*\n\n###### How it's computed:\n*Current start and end bounds of each complete range, listed as table rows.*\n\n###### Reading it:\n*Fewer ranges is better; one continuous range means an unbroken history.*\n\n###### Healthy range:\n*A single range covering the configured retention window.*\n\n###### Watch for:\n*Many fragmented ranges indicate gaps in stored history from missed or failed fetches.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCompleteLedgersGauge`", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 253 + }, + "id": 109, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "complete_ledgers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}", + "format": "table", + "instant": true, + "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}]", + "refId": "A" + } + ], + "title": "Complete Ledger Ranges", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Sizes of the relational databases in KB (total, ledger, transaction).*\n\n###### How it's computed:\n*Current values of the per-database size gauges, plotted as lines.*\n\n###### Reading it:\n*Smoothly growing lines are normal; the split shows where storage is used.*\n\n###### Healthy range:\n*Gradual growth consistent with retained history.*\n\n###### Watch for:\n*An abrupt change in growth rate can indicate storage pressure or a pruning issue.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Size (KB)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "deckbytes" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 253 + }, + "id": 110, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"db_kb_total\"}, \"series\", \"Total KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"db_kb_ledger\"}, \"series\", \"Ledger KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"db_kb_transaction\"}, \"series\", \"Transaction KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "Database Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 265 + }, + "id": 111, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"historical_perminute\"}, \"series\", \"Fetches/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Historical Fetch Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative count of peers disconnected for exceeding resource limits.*\n\n###### How it's computed:\n*Current value of the resource-disconnect gauge, plotted over time.*\n\n###### Reading it:\n*A flat line is healthy; steps up mean peers were dropped for overuse.*\n\n###### Healthy range:\n*Flat or very slowly rising.*\n\n###### Watch for:\n*A rising line indicates peers are being throttled off, consistent with abusive or misbehaving peers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 265 + }, + "id": 112, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peer_disconnects_resources\"}, \"series\", \"Resource Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Peer Disconnects (Resources)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 277 + }, + "id": 127, + "title": "Ledger Economy", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The node's local load-based fee factor that scales its minimum transaction cost; baseline 256 at idle.*\n\n###### How it's computed:\n*Current value of the local load-fee economy gauge.*\n\n###### Reading it:\n*Steady at the baseline (256) is normal; higher values mean the node is raising its fee in response to load.*\n\n###### Healthy range:\n*Around 256 (the normal baseline) when idle.*\n\n###### Watch for:\n*A climbing factor, which indicates the node is under transaction load pressure.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:drops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 278 + }, + "id": 114, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"base_fee_xrp\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Base Fee\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Base Fee (drops)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The minimum XRP balance required to keep an account on the ledger, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-base economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network reserve base.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:drops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 278 + }, + "id": 115, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"reserve_base_xrp\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Reserve Base\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Reserve Base (drops)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The additional XRP reserve required per owned ledger object, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-increment economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network owner reserve increment.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:drops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 290 + }, + "id": 116, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"reserve_inc_xrp\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Reserve Inc\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Reserve Inc (drops)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Seconds since the last validated ledger closed, plotted over time.*\n\n###### How it's computed:\n*Current value of the ledger-age economy gauge, sampled each interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the ledger close interval. Mirrors the Validated Ledger Age panel.*\n\n###### Healthy range:\n*Under about 10 seconds.*\n\n###### Watch for:\n*Growth beyond the expected close interval, meaning the node is not keeping up with validated ledgers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ledger Age (s)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 290 + }, + "id": 117, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"ledger_age_seconds\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Ledger Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The network transaction throughput reported by the ledger economy metrics.*\n\n###### How it's computed:\n*Current value of the transaction-rate economy gauge, plotted over time.*\n\n###### Reading it:\n*Reflects how many transactions are being processed; higher means busier.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A sudden sustained surge can indicate a transaction flood; a drop to zero can indicate the node stopped processing.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Transactions", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 306 + }, + "id": 118, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"transaction_rate\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Tx Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Transaction Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time to fetch a missing ledger from peers, at the 95th percentile.*\n\n###### How it's computed:\n*Per-acquire durations aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; populated mainly during sync or back-fill.*\n\n###### Healthy range:\n*Low when synced; higher and more active while catching up.*\n\n###### Watch for:\n*A spike signals the node is falling behind or recovering from a fork.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 322 + }, + "id": 119, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire\"}[5m]))), \"series\", \"P95 Acquire\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Acquire Duration (Inbound Fetch)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of completed ledger fetches split by outcome (complete or failed).*\n\n###### How it's computed:\n*Per-second rate of finished acquisitions grouped by outcome, per node, over a 5-minute window.*\n\n###### Reading it:\n*Complete should dominate; the failed line should stay near zero.*\n\n###### Healthy range:\n*Complete tracking fetch demand, failed near zero.*\n\n###### Watch for:\n*A rising failed rate means the node cannot fetch needed ledgers from its peers.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Acquisitions / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 322 + }, + "id": 120, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Acquire Rate by Outcome", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["node", "health"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "type", + "label": "Object Type", + "description": "Filter by internal object type (CountedObject class name)", + "type": "query", + "query": "label_values(object_count, type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "quantile", + "label": "Quantile", + "type": "custom", + "query": "0.5,0.75,0.95,0.99", + "current": { + "text": "0.95", + "value": "0.95" + }, + "options": [ + { + "text": "0.5", + "value": "0.5", + "selected": false + }, + { + "text": "0.75", + "value": "0.75", + "selected": false + }, + { + "text": "0.95", + "value": "0.95", + "selected": true + }, + { + "text": "0.99", + "value": "0.99", + "selected": false + } + ], + "includeAll": false, + "multi": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Node Health", + "uid": "node-health", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json new file mode 100644 index 00000000000..17fa7a8cd73 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -0,0 +1,767 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Fine-grained breakdown of peer-to-peer overlay traffic beyond the main network view: squelch relay control, protocol overhead, validator-list distribution, transaction-set exchange, transaction availability, ledger-proof and replay traffic, and unclassified messages. \u2014 Use it to: Drill into individual overlay message categories to diagnose relay efficiency, overhead, and catch-up traffic.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Squelch Traffic (Messages)", + "description": "###### What this is:\n*Squelch relay-control messages in/out, plus messages suppressed by squelch and squelch directives that were ignored. Squelch reduces redundant message forwarding between peers.*\n\n###### How it's computed:\n*Per-second message rate for the squelch, squelch-suppressed, and squelch-ignored categories, in and out.*\n\n###### Reading it:\n*High suppressed counts mean squelch is saving bandwidth; ignored should stay low.*\n\n###### Healthy range:\n*workload-dependent; suppressed far above ignored.*\n\n###### Watch for:\n*High ignored counts (peers not honoring squelch) or squelch traffic itself dominating.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Squelch In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Squelch Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_suppressed_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Suppressed In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_suppressed_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Suppressed Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_ignored_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ignored In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_ignored_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ignored Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Overhead Traffic Breakdown (Bytes)", + "description": "###### What this is:\n*Overlay protocol overhead bytes split into base overhead, intra-cluster overhead, and validator-manifest distribution overhead.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the overhead, overhead-cluster, and overhead-manifest categories.*\n\n###### Reading it:\n*Base overhead is routine; cluster and manifest rise around cluster syncs and manifest changes.*\n\n###### Healthy range:\n*workload-dependent; low and stable.*\n\n###### Watch for:\n*Sustained high cluster or manifest overhead (frequent cluster state churn or manifest reissue).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Base Overhead In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Base Overhead Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_cluster_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Cluster In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_cluster_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Cluster Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_manifest_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Manifest In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_manifest_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Manifest Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Validator List Traffic", + "description": "###### What this is:\n*Bytes and messages exchanged distributing validator lists (trusted-list configuration) between peers.*\n\n###### How it's computed:\n*Per-second in/out byte and message rate for the validator-lists category.*\n\n###### Reading it:\n*Bursts when lists update or new peers connect; quiet otherwise.*\n\n###### Healthy range:\n*workload-dependent; occasional bursts.*\n\n###### Watch for:\n*Continuous high volume (repeated list re-fetching or churn).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Bytes/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "binBps" + } + ] + } + ] + } + }, + { + "title": "Set Get/Share Traffic (Bytes)", + "description": "###### What this is:\n*Transaction-set fetch (get) and share bytes exchanged during ledger close.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the set-get and set-share categories.*\n\n###### Reading it:\n*Some exchange each ledger is normal as peers reconcile transaction sets.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High set-get (peers frequently missing transaction sets: possible sync delays).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Get In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_get_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Get Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Share In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_share_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Share Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Have/Requested Transactions (Messages)", + "description": "###### What this is:\n*Transaction-availability messages: advertisements that a peer has certain transactions, and explicit requests for transaction data.*\n\n###### How it's computed:\n*Per-second in/out message rate for the have-transactions and requested-transactions categories.*\n\n###### Reading it:\n*Compare requested versus have to gauge how well transactions are propagating.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Requested far exceeding have (peers behind on transaction propagation).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(have_transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Have TX In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(have_transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Have TX Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(requested_transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requested TX In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(requested_transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requested TX Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Unknown / Unclassified Traffic", + "description": "###### What this is:\n*Overlay traffic that matches no known message category, in bytes and messages.*\n\n###### How it's computed:\n*Current in/out byte and message counts for the unknown category.*\n\n###### Reading it:\n*Should be at or near zero.*\n\n###### Healthy range:\n*zero.*\n\n###### Watch for:\n*Any sustained non-zero value (protocol version mismatch, corrupted messages, or an unclassified new message type).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Bytes/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + } + }, + { + "title": "Proof Path Traffic", + "description": "###### What this is:\n*Proof-path request/response bytes used to verify individual ledger entries without downloading the whole ledger.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the proof-path request and response categories.*\n\n###### Reading it:\n*Rises when peers verify specific state, often during catch-up.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High sustained request volume (heavy state-verification load).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_request_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_response_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Replay Delta Traffic", + "description": "###### What this is:\n*Replay-delta request/response bytes used to efficiently replay ledger state changes during catch-up.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the replay-delta request and response categories.*\n\n###### Reading it:\n*Active during catch-up and replay; quiet when synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous replay traffic (node repeatedly replaying rather than staying current).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_request_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_response_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["network", "peer"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Overlay Traffic Detail", + "uid": "overlay-traffic-detail", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json new file mode 100644 index 00000000000..969b7ea0852 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -0,0 +1,495 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Consensus proposals and validations received from peers, and transaction relay efficiency for this node. \u2014 Use it to: Check that peers are feeding this node proposals, validations, and transactions, and gauge relay overhead.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Peer Proposal Receive Rate", + "description": "###### What this is:\n*How many consensus proposals this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received proposals over 5 minutes, per node.*\n\n###### Reading it:\n*A steady rate roughly proportional to the number of proposing validators.*\n\n###### Healthy range:\n*Workload-dependent; scales with validator count and connectivity.*\n\n###### Watch for:\n*A drop toward zero (isolation from the network) or a sudden flood far above baseline (proposal spam).*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Proposals Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Proposals / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Peer Validation Receive Rate", + "description": "###### What this is:\n*How many ledger validations this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received validations over 5 minutes, per node.*\n\n###### Reading it:\n*Steady and proportional to the number of validators the node hears from.*\n\n###### Healthy range:\n*Workload-dependent; roughly one burst per validator per closed ledger.*\n\n###### Watch for:\n*A fall toward zero (loss of validator connectivity) or an abnormal surge from untrusted sources.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Validations Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Validations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Proposals Trusted vs Untrusted", + "description": "###### What this is:\n*The share of received proposals that come from trusted (UNL) validators versus everyone else.*\n\n###### How it's computed:\n*Received-proposal rate split by trust status over 5 minutes, shown as proportions.*\n\n###### Reading it:\n*A healthy node with a good UNL shows a solid trusted slice.*\n\n###### Healthy range:\n*Workload-dependent; a meaningful trusted fraction is expected.*\n\n###### Watch for:\n*A collapsing trusted share or a large untrusted volume, consistent with proposal flooding from non-UNL peers.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Validations Trusted vs Untrusted", + "description": "###### What this is:\n*The share of received validations from trusted (UNL) validators versus untrusted sources.*\n\n###### How it's computed:\n*Received-validation rate split by trust status over 5 minutes, shown as proportions.*\n\n###### Reading it:\n*The trusted slice should dominate for a well-configured node.*\n\n###### Healthy range:\n*Workload-dependent; trusted validations expected to be the majority.*\n\n###### Watch for:\n*A shrinking trusted share or a spike of untrusted validations, which can indicate misconfiguration or abuse.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Reduce-Relay Peer Selection", + "description": "###### What this is:\n*How transaction relay picks peers: chosen relay sources, suppressed peers, and peers with the feature off.*\n\n###### How it's computed:\n*Current peer counts in each category (selected, suppressed, not-enabled), per node.*\n\n###### Reading it:\n*A high suppressed-to-selected ratio means relay is saving bandwidth as intended.*\n\n###### Healthy range:\n*Workload-dependent; suppressed should exceed selected in a well-connected mesh.*\n\n###### Watch for:\n*A large not-enabled count (older peers forcing full relay) or selected climbing while suppressed falls.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerReduceRelayGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"selected_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Selected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"suppressed_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Suppressed\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"not_enabled_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Not Enabled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Peer Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Reduce-Relay Missing-Tx Frequency", + "description": "###### What this is:\n*How often a peer has to fetch a transaction it missed because relay suppressed it.*\n\n###### How it's computed:\n*The reported frequency of on-demand missing-transaction fetches, per node.*\n\n###### Reading it:\n*Lower is better; near-flat means suppression is well tuned.*\n\n###### Healthy range:\n*Workload-dependent; a low, stable value is expected.*\n\n###### Watch for:\n*A rising trend, meaning suppression is too aggressive and the on-demand fetch path is growing.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerReduceRelayGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"missing_tx_freq\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missing Tx Freq\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Frequency", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["network", "peer"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "proposal_trusted", + "label": "Proposal Trusted", + "description": "Filter by proposal trust status (true = from trusted validator)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"peer.proposal.receive\"}, proposal_trusted)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "validation_trusted", + "label": "Validation Trusted", + "description": "Filter by validation trust status (true = from trusted validator)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"peer.validation.receive\"}, validation_trusted)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Peer Network", + "uid": "peer-network", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json new file mode 100644 index 00000000000..2b8891612e2 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -0,0 +1,573 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Peer connection quality: latency, divergence, version distribution and upgrade guidance, resource disconnects, and inbound/outbound balance. \u2014 Use it to: Assess how good this node's peer set is and whether connectivity or an upgrade needs attention.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "P90 Peer Latency", + "description": "###### What this is:\n*90th-percentile round-trip latency to connected peers, in milliseconds.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the p90 peer latency.*\n\n###### Reading it:\n*Lower is better; green under 200ms, yellow to 500ms, red above.*\n\n###### Healthy range:\n*Under 200ms.*\n\n###### Watch for:\n*Rising latency, which points to network congestion or geographically distant, poorly performing peers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"peer_latency_p90_ms\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"P90 Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 200 + }, + { + "color": "red", + "value": 500 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "Insane/Diverged Peers", + "description": "###### What this is:\n*Count of connected peers whose ledger state has diverged from the network.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the diverged-peer count.*\n\n###### Reading it:\n*Zero is healthy; any count means those peers disagree on ledger state.*\n\n###### Healthy range:\n*0 diverged peers.*\n\n###### Watch for:\n*A persistent non-zero count, which can indicate peers on a fork or misbehaving peers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"peers_insane_count\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Insane Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Higher Version Peers %", + "description": "###### What this is:\n*Percentage of connected peers running a newer rippled version than this node.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the higher-version peer percentage.*\n\n###### Reading it:\n*A high percentage suggests this node is behind and should be upgraded.*\n\n###### Healthy range:\n*Under 30%.*\n\n###### Watch for:\n*A majority of peers on a newer version, a strong upgrade signal.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"peers_higher_version_pct\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Higher Version %\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Upgrade Recommended", + "description": "###### What this is:\n*A flag indicating whether an upgrade is advised based on peer version analysis (Yes/No).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the upgrade-recommended flag.*\n\n###### Reading it:\n*No is healthy; Yes means most peers run a newer version.*\n\n###### Healthy range:\n*No.*\n\n###### Watch for:\n*A Yes state, indicating the node risks falling out of step with the network.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"upgrade_recommended\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Upgrade\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "No", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "Yes", + "color": "red" + } + } + } + ], + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Resource Disconnects", + "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Disconnects", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Inbound vs Outbound Peers", + "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Inbound\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Outbound\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Inbound.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "Outbound.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + } + ] + } + } + ], + "schemaVersion": 39, + "tags": ["network", "peer"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Peer Quality", + "uid": "peer-quality", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/retype_server_info.py b/docker/telemetry/grafana/dashboards/retype_server_info.py new file mode 100644 index 00000000000..960b5827e62 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/retype_server_info.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Change the render type of four Server Info tiles on node-health, and append +the user's live-authored ledger-seq panels. Throwaway helper (not committed). + +Transforms (type + query/options only; gridPos kept): + - Build Version : stat -> piechart, slices = count by (version, xrpl_branch) + - Server State : stat -> piechart, slices = node count per state (keep 0-4 + value/color mappings) + - Peer Count : stat -> bargauge (horizontal), one bar per node + - Uptime : stat -> bargauge (horizontal), one bar per node + +The user added three panels directly on Grafana Cloud (a row + two ledger-seq +stat panels); those are re-appended verbatim from /tmp/nh_added_panels.json so a +repo write does not drop them. +""" + +import json +import sys + +FILTERS = ( + 'service_instance_id=~"$node", deployment_environment=~"$deployment_environment", ' + 'xrpl_network_type=~"$xrpl_network_type", service_name=~"$service_name", ' + 'xrpl_work_item=~"$xrpl_work_item", xrpl_branch=~"$xrpl_branch", ' + 'xrpl_node_role=~"$xrpl_node_role"' +) + +ADDED_PANELS = "/tmp/nh_added_panels.json" + + +def find(panels, title): + for p in panels: + if p.get("title") == title: + return p + if "panels" in p: + r = find(p["panels"], title) + if r: + return r + return None + + +def to_piechart(p, expr, legend): + """Convert a panel to a piechart with one target + legend.""" + p["type"] = "piechart" + p["targets"] = [ + { + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "expr": expr, + "legendFormat": legend, + "refId": "A", + } + ] + p["options"] = { + "legend": {"displayMode": "list", "placement": "right", "values": ["value"]}, + "pieType": "pie", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "tooltip": {"mode": "single", "sort": "desc"}, + } + # piechart uses field displayName for slice labels, not the xrpl_ident stat form + p.setdefault("fieldConfig", {}).setdefault("defaults", {}).pop("displayName", None) + + +def to_bargauge(p, expr): + """Convert a panel to a horizontal bar gauge, one bar per node.""" + p["type"] = "bargauge" + for t in p["targets"]: + t["expr"] = expr + t["instant"] = False + t["legendFormat"] = "{{service_instance_id}}" + p["options"] = { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "showUnfilled": True, + "valueMode": "color", + } + # bar label is the per-node legend, not the stat xrpl_ident bracket + p.setdefault("fieldConfig", {}).setdefault("defaults", {})[ + "displayName" + ] = "{{service_instance_id}}" + + +def retype(dash): + panels = dash["panels"] + + # 1) Build Version -> pie by (version, xrpl_branch) + bv = find(panels, "Build Version") + to_piechart( + bv, + "count by (version, xrpl_branch) (build_info{%s})" % FILTERS, + "{{version}} {{xrpl_branch}}", + ) + + # 2) Server State -> pie of node count per state. server_state is a metric + # VALUE (0-4), not a label, so count_values buckets nodes by that value + # into a new "state" label; the nested label_replace chain then renames + # 0-4 to DISCONNECTED..FULL so the pie slices read as state names. (Value + # mappings can't help here -- they map field values, not legend labels.) + ss = find(panels, "Server State") + state_expr = ( + 'count_values("state", server_info{%s, metric="server_state"})' % FILTERS + ) + for num, name in ( + ("0", "DISCONNECTED"), + ("1", "CONNECTED"), + ("2", "SYNCING"), + ("3", "TRACKING"), + ("4", "FULL"), + ): + state_expr = 'label_replace(%s, "state", "%s", "state", "%s")' % ( + state_expr, + name, + num, + ) + to_piechart(ss, state_expr, "{{state}}") + + # 3) Peer Count -> bargauge per node + pc = find(panels, "Peer Count") + to_bargauge(pc, 'server_info{%s, metric="peers"}' % FILTERS) + + # 4) Uptime -> bargauge per node + up = find(panels, "Uptime") + to_bargauge(up, 'server_info{%s, metric="uptime"}' % FILTERS) + + # 5) Re-append the user's live-authored panels if not already present. + have = {p.get("title") for p in panels} + added = json.load(open(ADDED_PANELS)) + for p in added: + if p.get("title") not in have: + panels.append(p) + + return dash + + +# --- serializer matching the dashboards' on-disk byte format --------------- +def _dump(obj, lvl=0): + pad = " " * lvl + pad1 = " " * (lvl + 1) + if isinstance(obj, dict): + if not obj: + return "{}" + return ( + "{\n" + + ",\n".join( + "%s%s: %s" % (pad1, json.dumps(k), _dump(v, lvl + 1)) + for k, v in obj.items() + ) + + "\n" + + pad + + "}" + ) + if isinstance(obj, list): + if not obj: + return "[]" + if all(isinstance(x, (str, int, float, bool)) or x is None for x in obj): + return "[" + ", ".join(json.dumps(x) for x in obj) + "]" + return ( + "[\n" + + ",\n".join("%s%s" % (pad1, _dump(x, lvl + 1)) for x in obj) + + "\n" + + pad + + "]" + ) + return json.dumps(obj) + + +def main(): + for path in sys.argv[1:]: + d = json.load(open(path)) + retype(d) + out = _dump(d) + "\n" + json.loads(out) + open(path, "w").write(out) + print("%s: retyped 4 panels, +%d user panels" % (path, 3)) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json new file mode 100644 index 00000000000..14ebc4a3d68 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -0,0 +1,794 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: RPC request throughput and response timing, pathfinding cost, gRPC read traffic, and resource-manager enforcement for this node. \u2014 Use it to: Watch client request load and cost, and confirm resource limits are protecting the node.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "RPC Request Rate", + "description": "###### What this is:\n*How many RPC requests the server counts per second.*\n\n###### How it's computed:\n*Per-second rate of the RPC request counter over 5 minutes, per node.*\n\n###### Reading it:\n*A steady line proportional to client demand; cross-checks the trace-based RPC dashboard.*\n\n###### Healthy range:\n*Workload-dependent; tracks client activity.*\n\n###### Watch for:\n*A sudden surge above baseline, consistent with a client flooding the RPC endpoint.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(rpc_requests_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requests / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "title": "RPC Response Time", + "description": "###### What this is:\n*The 95th-percentile end-to-end RPC response time, including HTTP handling.*\n\n###### How it's computed:\n*95th-percentile of measured response times over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is broader than command-only latency.*\n\n###### Healthy range:\n*A few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating handler overload or slow downstream work.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Time\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "RPC Response Size", + "description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_milliseconds_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_size_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "decbytes", + "custom": { + "axisLabel": "Size (Bytes)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "RPC Response Time Distribution", + "description": "###### What this is:\n*RPC response time at the 90th, 95th, and 99th percentiles together.*\n\n###### How it's computed:\n*Three response-time quantiles over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; the gap between P90 and P99 shows the long tail.*\n\n###### Healthy range:\n*Percentiles clustered low; workload-dependent.*\n\n###### Watch for:\n*A widening P99, revealing long-tail or bimodal slow requests.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.9, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P90\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Fast Duration", + "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Full Duration", + "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Resource Warnings Rate", + "description": "###### What this is:\n*How often the resource manager warns a peer or client for excessive usage, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-warning events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.1/sec, red above 1/sec.*\n\n###### Healthy range:\n*Near zero in normal operation.*\n\n###### Watch for:\n*A rising rate, consistent with aggressive clients that may need throttling.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(warn_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Warnings / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "Resource Drops Rate", + "description": "###### What this is:\n*How often the resource manager drops or blocks a peer or client for abuse, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-drop events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.01/sec, red above 0.1/sec.*\n\n###### Healthy range:\n*Zero when no abusive consumers are present.*\n\n###### Watch for:\n*Non-zero values, meaning the node is actively rejecting abusive connections.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(drop_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Drops / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "gRPC Request Rate by Method (Spans)", + "description": "###### What this is:\n*How many gRPC calls of each method the node serves per second.*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Non-zero only when reporting/Clio-style clients use the gRPC API.*\n\n###### Healthy range:\n*Workload-dependent; zero without gRPC traffic.*\n\n###### Watch for:\n*A single method spiking, consistent with a heavy or misbehaving gRPC consumer.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Calls / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "gRPC Latency P95 by Method (Spans)", + "description": "###### What this is:\n*The 95th-percentile latency of each gRPC method.*\n\n###### How it's computed:\n*95th-percentile of gRPC call durations over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Lower is better; identifies slow gRPC read paths.*\n\n###### Healthy range:\n*Workload-dependent; scales with ledger data volume served.*\n\n###### Watch for:\n*A method whose latency climbs, indicating an expensive read path.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m]))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "gRPC Error Rate by Status (Spans)", + "description": "###### What this is:\n*gRPC call rate broken down by outcome status (success or error).*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over 5 minutes, grouped by status and node.*\n\n###### Reading it:\n*Success should dominate; the error rate should stay low.*\n\n###### Healthy range:\n*Workload-dependent; errors near the floor.*\n\n###### Watch for:\n*A rising error rate, indicating gRPC clients hitting failures.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (grpc_status, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"grpc\\\\..*\", grpc_status!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"grpc_status\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Calls / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Compute Duration (Spans)", + "description": "###### What this is:\n*The 95th-percentile time to compute paths for a single request.*\n\n###### How it's computed:\n*95th-percentile of path-computation durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; complements the fast/full pathfinding timers with span-level detail.*\n\n###### Healthy range:\n*Workload-dependent; grows with order-book depth and request complexity.*\n\n###### Watch for:\n*A rising trend under pathfinding load, indicating expensive path computation.*\n\n###### Source:\n[PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`PathRequest::doUpdate`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.compute\"}[5m]))), \"series\", \"P95 Compute\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Request & Discovery Rate (Spans)", + "description": "###### What this is:\n*The rate of client path requests versus path-discovery passes.*\n\n###### How it's computed:\n*Per-second rates of path requests and discovery passes over 5 minutes, per node.*\n\n###### Reading it:\n*Discovery cost tracks request demand; useful for subscription-heavy nodes.*\n\n###### Healthy range:\n*Workload-dependent; both zero without pathfinding clients.*\n\n###### Watch for:\n*Discovery rate climbing out of proportion to requests, a cost driver for subscription-heavy nodes.*\n\n###### Source:\n[PathFind.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/handlers/orderbook/PathFind.cpp) \u00b7 [PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`doPathFind ; PathRequest::findPaths`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.request\"}[$__rate_interval])), \"series\", \"Requests / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.discover\"}[$__rate_interval])), \"series\", \"Discoveries / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["rpc", "pathfinding"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "grpc_method", + "label": "gRPC Method", + "description": "Filter by gRPC method (GetLedger, GetLedgerData, GetLedgerDiff, GetLedgerEntry)", + "type": "query", + "query": "label_values(span_calls_total{span_name=~\"grpc\\\\..*\"}, method)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "RPC & Pathfinding", + "uid": "rpc-pathfinding", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json new file mode 100644 index 00000000000..1eea7ff51f9 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -0,0 +1,1115 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 11, + "type": "row", + "title": "Aggregate RPC (all commands)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "title": "RPC Request Rate by Command", + "description": "###### What this is:\n*How many times each RPC command runs per second.*\n\n###### How it's computed:\n*Per-second rate of command executions over 5 minutes, grouped by command name and node.*\n\n###### Reading it:\n*Compare bars/lines across commands to see which endpoints drive load.*\n\n###### Healthy range:\n*Workload-dependent; dominated by whichever commands clients call most.*\n\n###### Watch for:\n*A single command spiking far above its norm, consistent with a client hammering one endpoint.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "RPC Latency P95 by Command", + "description": "###### What this is:\n*The 95th-percentile response time for each RPC command.*\n\n###### How it's computed:\n*95th-percentile of per-command execution durations over 5 minutes, grouped by command and node.*\n\n###### Reading it:\n*Lower is better; watch the slowest commands.*\n\n###### Healthy range:\n*Fast lookups a few ms; heavy queries tens to hundreds of ms. Workload-dependent.*\n\n###### Watch for:\n*A command whose latency climbs steadily, pointing to an expensive or degrading query path.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))), \"series\", \"P95 $1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "RPC Error Rate", + "description": "###### What this is:\n*The percentage of each command's calls that finished with an error.*\n\n###### How it's computed:\n*Error calls divided by total calls per command over 5 minutes, expressed as a percent.*\n\n###### Reading it:\n*Green under 1%, yellow 1-5%, red above 5%.*\n\n###### Healthy range:\n*Below 1% per command in normal operation.*\n\n###### Watch for:\n*A command sitting red, indicating malformed input, overload, or a broken handler.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])) * 100, \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "RPC Latency Heatmap", + "description": "###### What this is:\n*The full distribution of RPC response times over the window.*\n\n###### How it's computed:\n*Counts of requests in each latency band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; multiple bands mean mixed fast/slow requests.*\n\n###### Healthy range:\n*Most mass in the low-latency bands; workload-dependent.*\n\n###### Watch for:\n*A distinct high-latency cluster (bimodal behavior) that percentiles alone would hide.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Duration (ms)", + "unit": "ms" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "id": 4 + }, + { + "title": "Overall RPC Throughput", + "description": "###### What this is:\n*Two layers of the request pipeline: connections accepted by the HTTP handler versus requests parsed and dispatched.*\n\n###### How it's computed:\n*Per-second rates of the outer request-accept layer and the inner processing layer over 5 minutes, per node.*\n\n###### Reading it:\n*The two lines should track each other closely.*\n\n###### Healthy range:\n*Workload-dependent; both lines roughly equal.*\n\n###### Watch for:\n*A gap between accept and process, meaning requests are queued or rejected before dispatch.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession ; ServerHandler::processRequest`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.http_request\"}[$__rate_interval])), \"series\", \"rpc.http_request / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.process\"}[$__rate_interval])), \"series\", \"rpc.process / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "RPC Success vs Error", + "description": "###### What this is:\n*Aggregate rate of successful versus failed RPC commands across all command types.*\n\n###### How it's computed:\n*Per-second rates of ok-status and error-status command completions over 5 minutes, per node.*\n\n###### Reading it:\n*Successful commands report an ok status and should dominate; the error line should stay near the floor.*\n\n###### Healthy range:\n*Error line near zero relative to success; workload-dependent.*\n\n###### Watch for:\n*A sustained error line, which warrants drilling into the per-command breakdown.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_OK\"}[$__rate_interval])), \"series\", \"Success\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])), \"series\", \"Error\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Commands / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Top Commands by Volume", + "description": "###### What this is:\n*The ten most-called RPC commands over the recent window.*\n\n###### How it's computed:\n*Total invocation counts per command over the last 5 minutes, ranked and capped at ten.*\n\n###### Reading it:\n*The top bars are the hottest endpoints driving node load.*\n\n###### Healthy range:\n*Workload-dependent; typically led by lightweight status queries.*\n\n###### Watch for:\n*An unexpected command dominating the ranking, consistent with automated abuse.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short" + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "WebSocket Message Rate", + "description": "###### What this is:\n*How many RPC messages arrive over WebSocket connections per second.*\n\n###### How it's computed:\n*Per-second rate of processed WebSocket RPC messages over 5 minutes, per node.*\n\n###### Reading it:\n*Non-zero only when clients use WebSocket; zero is normal for HTTP-only nodes.*\n\n###### Healthy range:\n*Workload-dependent; zero if no WebSocket clients.*\n\n###### Watch for:\n*A sudden surge from a single source, consistent with a chatty or abusive subscription client.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval])), \"series\", \"WS Messages / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + }, + "id": 8 + }, + { + "title": "RPC Resource Cost by Command", + "description": "###### What this is:\n*RPC traffic grouped by resource-cost category rather than by command name.*\n\n###### How it's computed:\n*Per-second rate of commands over 5 minutes, grouped by load/cost category.*\n\n###### Reading it:\n*Cheap categories should dominate; costly ones should be a small slice.*\n\n###### Healthy range:\n*Workload-dependent; low-cost categories carry most traffic.*\n\n###### Watch for:\n*Growth in high-cost categories such as exception or malformed requests, which point to problematic clients.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, load_type, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"rpc.command.*\", load_type!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"load_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Batch vs Single RPC Requests", + "description": "###### What this is:\n*The rate of batched RPC requests versus single-command requests.*\n\n###### How it's computed:\n*Per-second rates of batch and single requests over 5 minutes, per node.*\n\n###### Reading it:\n*Single requests usually dominate; batch is used by bulk clients.*\n\n###### Healthy range:\n*Workload-dependent; batch typically a minority.*\n\n###### Watch for:\n*A batch rate climbing sharply, consistent with bulk automation or amplification attempts.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processRequest`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"rpc.process\", is_batch=\"true\"}[$__rate_interval])), \"series\", \"Batch\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"rpc.process\", is_batch=\"false\"}[$__rate_interval])), \"series\", \"Single\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "id": 12, + "type": "row", + "title": "Per-Method Detail", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "panels": [] + }, + { + "title": "RPC Call Rate (All Methods)", + "description": "###### What this is:\n*Overall rate of RPC method calls that started, finished, and errored, across all methods.*\n\n###### How it's computed:\n*Per-second rate of each counter over a 5-minute window, summed per node.*\n\n###### Reading it:\n*Started should closely track finished; errored should be a small fraction.*\n\n###### Healthy range:\n*Workload-dependent; started \u2248 finished, errored near zero.*\n\n###### Watch for:\n*A growing gap between started and finished (calls hanging), or an errored line that rises with load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted / recordRpcFinished / recordRpcErrored`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 51 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Errored/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 13 + }, + { + "title": "Per-Method Call Rate (Top 10)", + "description": "###### What this is:\n*The ten busiest RPC methods by call rate.*\n\n###### How it's computed:\n*Per-second start rate over 5 minutes, per method, showing the top ten.*\n\n###### Reading it:\n*Identifies which methods dominate load; the mix shifts with client behaviour.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single method suddenly dominating, which can signal a runaway client or abusive query pattern.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 59 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Per-Method Error Rate (Top 10)", + "description": "###### What this is:\n*The ten RPC methods producing the most errors.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Normally near zero; persistent errors point to a specific failing method.*\n\n###### Healthy range:\n*Near zero for well-behaved traffic.*\n\n###### Watch for:\n*Sustained errors concentrated on one method \u2014 a broken client, a bad input, or probing.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 67 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 15 + }, + { + "title": "RPC Latency - All Methods", + "description": "###### What this is:\n*Aggregate RPC handler latency across all methods (p75 and p99).*\n\n###### How it's computed:\n*Percentiles of the method-duration histogram over a 5-minute window.*\n\n###### Reading it:\n*p75 reflects typical responsiveness; p99 captures the slow tail.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond for light commands; heavier commands run longer.*\n\n###### Watch for:\n*A rising p99 while p75 stays flat \u2014 a subset of calls degrading, often from expensive queries.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p75\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 16 + }, + { + "title": "Per-Method Latency (p99, Top 10 Slowest)", + "description": "###### What this is:\n*The ten slowest RPC methods by tail latency.*\n\n###### How it's computed:\n*p99 of each method's duration histogram over 5 minutes, top ten.*\n\n###### Reading it:\n*Surfaces which specific methods are expensive.*\n\n###### Healthy range:\n*Method-dependent; ledger/account queries are heavier than status calls.*\n\n###### Watch for:\n*A method whose p99 climbs over time, or an unexpectedly cheap method appearing here.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 83 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, histogram_quantile(0.99, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m])))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 17 + }, + { + "title": "RPC Error Ratio by Method", + "description": "###### What this is:\n*The methods with the highest error rates, for spotting failure hotspots.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Highlights where failures concentrate.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*One method with a persistently high error rate \u2014 malformed requests or targeted probing.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 91 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) / (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) > 0)), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Error Ratio", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Current RPC Latency (p99 Gauge)", + "description": "###### What this is:\n*Current tail latency (p99) of RPC handling across all methods, as a live gauge.*\n\n###### How it's computed:\n*p99 of the method-duration histogram over the recent window.*\n\n###### Reading it:\n*A single at-a-glance number for current RPC responsiveness.*\n\n###### Healthy range:\n*Low-millisecond under normal load.*\n\n###### Watch for:\n*Sustained elevation, indicating the node is under query pressure.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`", + "type": "gauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99 Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [] + }, + "id": 19 + } + ], + "schemaVersion": 39, + "tags": ["rpc"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "command", + "label": "RPC Command", + "description": "Filter by RPC command name (e.g., server_info, submit)", + "type": "query", + "query": "label_values(span_calls_total{span_name=~\"rpc.command.*\"}, command)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "method", + "label": "RPC Method", + "description": "Filter by RPC method", + "type": "query", + "query": "label_values(rpc_method_started_total, method)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "RPC Performance", + "uid": "rpc-performance", + "refresh": "10s", + "description": "What this shows: Per-command and per-method RPC performance: call rates, error rates, and latency distributions. \u2014 Use it to: Identify slow or failing RPC commands and track client-facing request latency." +} diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json new file mode 100644 index 00000000000..d08f49bc9b9 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -0,0 +1,1110 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Transaction Apply Failed Rate", + "description": "###### What this is:\n*Transactions that fail during the apply (transactor) stage, per second \u2014 i.e. the transactor ran but the result was not tesSUCCESS.*\n\n###### How it's computed:\n*Per-second rate of tx.transactor spans at the apply stage whose ter_result is not tesSUCCESS, over the dashboard rate interval, per node.*\n\n###### Reading it:\n*A steady low background is normal (tefPAST_SEQ, tecUNFUNDED, etc.); the transactor legitimately rejects many transactions.*\n\n###### Healthy range:\n*A low, stable rate; workload-dependent.*\n\n###### Watch for:\n*A sharp rise above baseline, which points to a submission pattern that is systematically failing at apply.*\n\n###### Source:\n[Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/tx/detail/Transactor.cpp)\n\n###### Function:\n`Transactor::operator()`", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval])), \"series\", \"Failed / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Transaction Processing Latency by Type", + "description": "###### What this is:\n*The 95th-percentile time to process a transaction, broken down by transaction type.*\n\n###### How it's computed:\n*95th-percentile of processing durations over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Lower is better; compare types to find the expensive ones.*\n\n###### Healthy range:\n*A few milliseconds for simple payments; workload- and type-dependent.*\n\n###### Watch for:\n*A type whose latency climbs, pointing to expensive processing or resource pressure.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Transaction Rate by Type", + "description": "###### What this is:\n*How many transactions of each type enter processing per second.*\n\n###### How it's computed:\n*Per-second processing rate over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Shows the transaction mix; payments usually lead.*\n\n###### Healthy range:\n*Workload-dependent; mix reflects network activity.*\n\n###### Watch for:\n*A single type spiking far above baseline, consistent with a spam campaign of that type.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 12 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "TX / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Transaction Results by Type", + "description": "###### What this is:\n*The non-success result codes transactions return, by transaction type.*\n\n###### How it's computed:\n*Per-second rate of failing transactions over 5 minutes, grouped by type and result code.*\n\n###### Reading it:\n*Fewer failures is better; use it to see which types fail and why.*\n\n###### Healthy range:\n*Workload-dependent; a modest background of expected failures is normal.*\n\n###### Watch for:\n*A surge of one failure code for one type, indicating a systemic issue or abusive submissions.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (tx_type, ter_result, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\", ter_result=~\"$ter_result\", ter_result!=\"tesSUCCESS\"}[$__rate_interval])), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"ter_result\", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "axisLabel": "Failed TX / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Transaction Receive vs Suppressed", + "description": "###### What this is:\n*Raw transactions received from peers, split by whether they were duplicates suppressed before processing.*\n\n###### How it's computed:\n*Per-second rate of received transactions over the window, split by suppressed flag and node.*\n\n###### Reading it:\n*A large suppressed share is normal, since the same transaction arrives from many peers.*\n\n###### Healthy range:\n*Workload-dependent; suppressed typically exceeds newly processed.*\n\n###### Watch for:\n*A collapse in suppression (duplicate filtering failing) or an overall receive flood.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::handleTransaction`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (suppressed, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{span_name=\"tx.receive\", tx_type=~\"$tx_type\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Suppressed $1\", \"suppressed\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Transaction Processing Rate", + "description": "###### What this is:\n*Transactions entering the processing pipeline versus raw transactions arriving from peers.*\n\n###### How it's computed:\n*Per-second rates of processed transactions and received peer transactions over 5 minutes, per node.*\n\n###### Reading it:\n*Received sits above processed; the difference is deduplicated relay traffic.*\n\n###### Healthy range:\n*Workload-dependent; both scale with network volume.*\n\n###### Watch for:\n*A receive rate spiking well above processing, consistent with relay flooding.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp) \u00b7 [PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction ; PeerImp::handleTransaction`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"tx.process / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.receive\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"tx.receive / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Transaction Path Distribution", + "description": "###### What this is:\n*The split of transactions by origin: submitted locally versus relayed from peers.*\n\n###### How it's computed:\n*Processed-transaction rate over 5 minutes, split by local-origin flag and shown as proportions.*\n\n###### Reading it:\n*Most traffic on a network node is peer-relayed; local dominates on a submission node.*\n\n###### Healthy range:\n*Workload-dependent on the node's role.*\n\n###### Watch for:\n*An unexpected surge in local submissions, consistent with a local client flooding the node.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (local, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"Local $1\", \"local\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "id": 7 + }, + { + "title": "Transaction Processing Duration Heatmap", + "description": "###### What this is:\n*The full distribution of transaction processing times over the window.*\n\n###### How it's computed:\n*Counts of transactions in each latency band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; spread bands mean uneven processing time.*\n\n###### Healthy range:\n*Most mass in the low-latency bands; workload-dependent.*\n\n###### Watch for:\n*A distinct slow cluster (multi-modal timing) hidden by percentile charts.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "yAxis": { + "axisLabel": "Duration (ms)", + "unit": "ms" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "id": 8 + }, + { + "title": "Transactor Duration by Type (p95)", + "description": "###### What this is:\n*The 95th-percentile execution time of the core transactor step, by transaction type.*\n\n###### How it's computed:\n*95th-percentile of transactor durations over 5 minutes, grouped by type and node.*\n\n###### Reading it:\n*Lower is better; reveals which types are most expensive to execute.*\n\n###### Healthy range:\n*Sub-millisecond to a few ms for most types; workload-dependent.*\n\n###### Watch for:\n*A type whose execution time grows, indicating expensive logic or ledger contention.*\n\n###### Source:\n[Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`Transactor::operator()`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 52 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.transactor\", tx_type=~\"$tx_type\"}[5m]))), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "TxQ Accept: Applied Ratio per Node (State Timeline)", + "description": "###### What this is:\n*The applied fraction of TxQ accepts per node over time: the share of queued transactions that were included in a ledger versus removed on failure.*\n\n###### How it's computed:\n*Per node, applied accepts divided by applied-plus-failed accepts over the window.*\n\n###### Reading it:\n*Green (>=90% applied) is a healthy drain; yellow is degraded; red means accepts are mostly failing.*\n\n###### Healthy range:\n*At or near 100% applied when the queue is draining healthily; workload-dependent.*\n\n###### Watch for:\n*A node dropping into yellow or red, which signals queue pressure, under-bidding, or fee escalation on that node.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::accept`", + "type": "state-timeline", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 60 + }, + "options": { + "mergeValues": true, + "showValue": "auto", + "alignValue": "center", + "rowHeight": 0.85, + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "{{service_instance_id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "custom": { + "fillOpacity": 80, + "lineWidth": 0 + } + }, + "overrides": [] + }, + "id": 11 + }, + { + "title": "Tx Apply Pipeline Latency by Stage (p95)", + "description": "###### What this is:\n*The 95th-percentile duration of each apply-pipeline stage.*\n\n###### How it's computed:\n*95th-percentile of per-stage durations over 5 minutes, grouped by stage and node.*\n\n###### Reading it:\n*Lower is better; isolates which stage dominates processing time.*\n\n###### Healthy range:\n*Each stage typically sub-millisecond to a few ms; workload-dependent.*\n\n###### Watch for:\n*One stage's latency rising, pinpointing the bottleneck in transaction handling.*\n\n###### Source:\n[applySteps.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/applySteps.cpp) \u00b7 [Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`makeStageSpan ; Transactor::operator()`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 68 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(label_replace(label_replace(label_replace(histogram_quantile(0.95, sum by (le, stage, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\"}[5m]))), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\"), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "Tx Apply Pipeline Failure Rate by Stage", + "description": "###### What this is:\n*How many transactions fail at each apply-pipeline stage per second.*\n\n###### How it's computed:\n*Per-second rate of non-success outcomes over 5 minutes, grouped by stage and node.*\n\n###### Reading it:\n*Shows whether rejections concentrate in preflight, preclaim, or apply.*\n\n###### Healthy range:\n*Workload-dependent; a modest background of expected rejections is normal.*\n\n###### Watch for:\n*A failure spike concentrated in one stage, consistent with malformed or spam submissions.*\n\n###### Source:\n[applySteps.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/applySteps.cpp) \u00b7 [Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`makeStageSpan ; Transactor::operator()`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 76 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(label_replace(label_replace(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval])), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\"), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Failed Spans / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 13 + }, + { + "title": "Tx Apply Pipeline Latency by Type and Stage (p95)", + "description": "###### What this is:\n*The 95th-percentile stage duration broken down by both transaction type and pipeline stage.*\n\n###### How it's computed:\n*95th-percentile durations over 5 minutes, grouped by type and stage; higher cardinality than the by-stage view.*\n\n###### Reading it:\n*Lower is better; shows which stage dominates each type's latency.*\n\n###### Healthy range:\n*Workload-dependent; most type/stage pairs sub-millisecond to a few ms.*\n\n###### Watch for:\n*A specific type/stage combination rising sharply, pinpointing a costly path.*\n\n###### Source:\n[applySteps.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/applySteps.cpp) \u00b7 [Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`makeStageSpan ; Transactor::operator()`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 84 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(label_replace(label_replace(label_replace(histogram_quantile(0.95, sum by (le, tx_type, stage, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", tx_type=~\"$tx_type\", stage=~\"$stage\"}[5m]))), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\"), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"stage\", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Transaction Apply Duration per Ledger", + "description": "###### What this is:\n*The 95th-percentile time to apply the agreed transaction set into each new ledger.*\n\n###### How it's computed:\n*95th-percentile of transaction-apply durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; a major component of ledger build time.*\n\n###### Healthy range:\n*A few to tens of milliseconds; scales with transactions per ledger.*\n\n###### Watch for:\n*Rising durations during heavy or expensive transaction sets.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 92 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[5m]))), \"series\", \"tx.apply\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 15 + }, + { + "title": "TxQ Enqueue Rate by Transaction Type", + "description": "###### What this is:\n*How many transactions are queued (rather than applied immediately) per second, by type.*\n\n###### How it's computed:\n*Per-second rate of queue-enqueue operations over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Shows the demand mix hitting the queue and how it shifts as the queue fills.*\n\n###### Healthy range:\n*Workload-dependent; low or zero when the network is not congested.*\n\n###### Watch for:\n*A spam burst of one type, an early indicator of fee escalation.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::apply`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 100 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.enqueue\"}[$__rate_interval])), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Enqueues / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 16 + }, + { + "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", + "description": "###### What this is:\n*The fraction of transactions that applied straight to the open ledger instead of being queued.*\n\n###### How it's computed:\n*Direct-apply rate divided by direct-apply plus enqueue rate over 5 minutes, per node.*\n\n###### Reading it:\n*Higher is better; near 1.0 means the network is not congested.*\n\n###### Healthy range:\n*Close to 1.0 when not congested; falls as demand exceeds capacity.*\n\n###### Watch for:\n*A falling ratio, the cleanest single signal the network has entered sustained fee escalation.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::tryDirectApply ; TxQ::apply`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 108 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.apply_direct\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.apply_direct\"}[$__rate_interval])) + sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.enqueue\"}[$__rate_interval])), 1), \"series\", \"Direct-Apply Fraction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "custom": { + "axisLabel": "Bypass Fraction", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Queue Accept (Drain) Duration per Ledger", + "description": "###### What this is:\n*The 95th-percentile time to drain queued transactions into a newly closed ledger.*\n\n###### How it's computed:\n*95th-percentile of queue-drain durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; rising time signals queue pressure at ledger close.*\n\n###### Healthy range:\n*A few milliseconds when the queue is light; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating a large or contended queue slowing ledger close.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::accept`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 108 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept\"}[5m]))), \"series\", \"Drain\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 19 + }, + { + "title": "Queue Cleanup Rate (Expired Entries)", + "description": "###### What this is:\n*How often expired transactions are removed from the queue each ledger, per second.*\n\n###### How it's computed:\n*Per-second rate of queue-cleanup operations over 5 minutes, per node.*\n\n###### Reading it:\n*Low is healthy; a rising rate means submitters are abandoning transactions.*\n\n###### Healthy range:\n*Near zero when not congested; workload-dependent.*\n\n###### Watch for:\n*A rising rate, a demand-frustration signal where submitters under-bid the escalating fee and gave up.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::processClosedLedger`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 116 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.cleanup\"}[$__rate_interval])), \"series\", \"Cleanups / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Cleanups / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 20 + } + ], + "schemaVersion": 39, + "tags": ["transactions"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "tx_origin", + "label": "TX Origin", + "description": "Filter by transaction origin (true = local submit, false = peer relay)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"tx.process\"}, local)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "tx_type", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(span_calls_total{span_name=\"tx.process\", tx_type!=\"\"}, tx_type)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "label": "TX Type" + }, + { + "name": "ter_result", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(span_calls_total{span_name=\"tx.process\", ter_result!=\"\"}, ter_result)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "label": "Result Code" + }, + { + "name": "stage", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(span_calls_total{span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage!=\"\"}, stage)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "label": "Apply Stage" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Transaction Overview", + "uid": "transaction-overview", + "refresh": "10s", + "description": "What this shows: Transaction flow through this node: receipt, processing, results, per-stage timing, and queue behavior. \u2014 Use it to: Trace transactions from arrival to ledger, and locate stalls in processing or the queue." +} diff --git a/docker/telemetry/grafana/dashboards/validate_dashboards.py b/docker/telemetry/grafana/dashboards/validate_dashboards.py new file mode 100755 index 00000000000..e921d02c4f2 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/validate_dashboards.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Dashboard lint: cumulative metrics must be rate()-wrapped; tier filters present.""" + +import json, re, sys + +# Prometheus gauges that hold a CUMULATIVE total -> must be rate()/increase()-wrapped. +CUMULATIVE_PREFIXES = ( + "total_bytes_", + "total_messages_", + "transactions_messages_", + "transactions_duplicate_messages_", + "proposals_", + "validations_", + "overlay_peer_disconnects", + "squelch_", + "overhead_", + "validator_lists_", + "set_get_", + "set_share_", + "have_transactions_", + "requested_transactions_", + "proof_path_", + "replay_delta_", + "ledger_data_", + "ledger_get_", + "ledger_share_", + "ledger_transaction_", + "ledger_account_state_", + "getobject_", + "jq_trans_overflow_total", +) +# nodestore_state{metric=...} cumulative sub-series (raw only inside these metric= labels). +NODESTORE_CUMULATIVE = ( + "node_reads_total", + "node_reads_hit", + "node_writes", + "node_read_bytes", + "node_written_bytes", + "node_reads_duration_us", +) +# state_accounting_*_duration are cumulative µs. +STATE_DURATION = re.compile(r"state_accounting_\w+_duration") +REQUIRED_FILTERS = ( + "$node", + "$deployment_environment", + "$xrpl_network_type", + "$service_name", +) + + +def iter_panels(dash): + for p in dash.get("panels", []): + yield p + for sub in p.get("panels", []) or []: + yield sub + + +def expr_is_wrapped(expr): + return ( + "rate(" in expr + or "increase(" in expr + or "irate(" in expr + or "histogram_quantile(" in expr + ) + + +def check(path, forbid_5m): + errs = [] + try: + dash = json.load(open(path)) + except Exception as e: + return [f"{path}: INVALID JSON: {e}"] + for p in iter_panels(dash): + title = p.get("title", "") + for tg in p.get("targets", []) or []: + expr = (tg.get("expr") or "").strip() + if not expr: + continue + # tier filters + if "{" in expr: + for f in REQUIRED_FILTERS: + if f not in expr: + errs.append(f"{path} [{title}]: missing {f} in expr") + # cumulative metric plotted raw? + hits = [m for m in CUMULATIVE_PREFIXES if m in expr] + nod = [m for m in NODESTORE_CUMULATIVE if f'"{m}"' in expr] + statedur = STATE_DURATION.search(expr) + if (hits or nod or statedur) and not expr_is_wrapped(expr): + who = hits or nod or [statedur.group(0)] + errs.append( + f"{path} [{title}]: cumulative metric {who} plotted RAW (needs rate/increase)" + ) + if "deriv(" in expr or "idelta(" in expr: + errs.append( + f"{path} [{title}]: uses deriv()/idelta() on a cumulative series" + ) + if forbid_5m and ("[5m]" in expr or "[1h]" in expr): + # Only a fixed [5m]/[1h] inside a top-level rate()/increase()/irate() on a + # plain counter is a violation. Legitimate fixed-window exprs -- histogram + # buckets (rate(..._bucket[5m])), avg_over_time windows, and subqueries + # ([5m:]) -- are intentional and must not be flagged. + if not ("_bucket" in expr or "avg_over_time" in expr or ":]" in expr): + errs.append( + f"{path} [{title}]: hardcoded range window; use [$__rate_interval]" + ) + # unit check + unit = p.get("fieldConfig", {}).get("defaults", {}).get("unit", "") + if unit == "mps": + errs.append(f"{path} [{title}]: invalid Grafana unit 'mps' (use 'cps')") + return errs + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("--")] + forbid_5m = "--no-5m" in sys.argv + all_errs = [] + for path in args: + all_errs += check(path, forbid_5m) + if all_errs: + print("\n".join(all_errs)) + print(f"\nFAIL: {len(all_errs)} violation(s)") + sys.exit(1) + print(f"OK: {len(args)} dashboard(s) passed") + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json new file mode 100644 index 00000000000..4a6b387ca5f --- /dev/null +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -0,0 +1,1148 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Validator agreement rates, validation counts, amendment and UNL status, server state tracking, and ledger close rate. \u2014 Use it to: Confirm the validator is participating, agreeing with the network, and closing ledgers at the expected cadence.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Validation Agreement", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Agreement % (1h)", + "description": "###### What this is:\n*Share of ledgers over the last hour where this validator agreed with the network consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 1-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; green at 95%+, yellow from 80%, red below.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*Values below 80%, meaning the validator frequently disagrees with consensus.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreement_pct_1h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreement 1h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreement % (24h)", + "description": "###### What this is:\n*Share of ledgers over the last 24 hours where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 24-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; a smoother, longer-term view than the 1h stat.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A sustained dip below 90%, which can indicate configuration drift or a network partition.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreement_pct_24h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreement 24h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreements vs Missed (1h)", + "description": "###### What this is:\n*Counts of agreed versus missed validations over the last hour.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 1-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate; missed should be small.*\n\n###### Healthy range:\n*Missed near 0.*\n\n###### Watch for:\n*A high missed count, meaning the validator is skipping consensus rounds.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreements_1h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreements 1h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"missed_1h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missed 1h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Missed.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "title": "Agreements vs Missed (24h)", + "description": "###### What this is:\n*Counts of agreed versus missed validations over the last 24 hours.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 24-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate over the full day.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A growing missed share, signalling longer-term reliability problems.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreements_24h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreements 24h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"missed_24h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missed 24h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Missed.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "title": "Validation Rates", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Validation Rate", + "description": "###### What this is:\n*Validations this node sends per minute.*\n\n###### How it's computed:\n*Per-second rate of the sent-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should track the ledger close cadence; roughly one validation per closed ledger.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*A drop toward zero, meaning the validator has stopped participating.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsSent (caller RCLConsensus.cpp)`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_sent_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Sent/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "green", + "value": 8 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Validations Checked Rate", + "description": "###### What this is:\n*Validations received from peers and checked per minute.*\n\n###### How it's computed:\n*Per-second rate of the checked-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Reflects how much validation traffic the network is delivering to this node.*\n\n###### Healthy range:\n*Workload-dependent; scales with trusted validator count.*\n\n###### Watch for:\n*A sudden collapse, which suggests peer connectivity loss or network isolation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsChecked (caller NetworkOPs.cpp)`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_checked_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Checked/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Amendment Blocked", + "description": "###### What this is:\n*Whether the node is amendment-blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the amendment-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means an enabled amendment is unsupported by this build.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which halts validation and requires a software upgrade.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 50 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"amendment_blocked\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Blocked\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "OK", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "BLOCKED", + "color": "red" + } + } + } + ], + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "UNL Expiry (days)", + "description": "###### What this is:\n*Days remaining until the current UNL (trusted validator list) expires.*\n\n###### How it's computed:\n*Instantaneous gauge reading of days-to-expiry.*\n\n###### Reading it:\n*Higher is safer; green at 30+, yellow under 7, red at expiry.*\n\n###### Healthy range:\n*30+ days.*\n\n###### Watch for:\n*Fewer than 7 days, after which the node loses its trusted validator set if not renewed.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 58 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"unl_expiry_days\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"UNL Expiry\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "d", + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 7 + }, + { + "color": "green", + "value": 30 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "UNL Blocked", + "description": "###### What this is:\n*Whether the node's UNL is blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the UNL-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means validator trust cannot be established.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which can stop the node participating in consensus.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 66 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"unl_blocked\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"UNL Blocked\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "OK", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "BLOCKED", + "color": "red" + } + } + } + ], + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreement/Missed Counters (Rate)", + "description": "###### What this is:\n*Rate of cumulative agreement and missed-validation counters per minute.*\n\n###### How it's computed:\n*Per-second rate of each monotonic counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Agreements should dominate; the missed line should stay low.*\n\n###### Healthy range:\n*Missed rate near 0.*\n\n###### Watch for:\n*A rising missed rate, complementing the windowed agreement percentages above.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationTotalsCounters`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 74 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validation_agreements_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Agreements/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validation_missed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Missed/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "custom": { + "axisLabel": "Per Minute", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Missed.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "title": "Server State & Consensus", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Validation Quorum", + "description": "###### What this is:\n*Minimum number of trusted validations required to declare a ledger fully validated.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the current quorum requirement.*\n\n###### Reading it:\n*Tracks the quorum derived from the active validator list; changes when the list changes.*\n\n###### Healthy range:\n*Stable at the network-appropriate value.*\n\n###### Watch for:\n*An unexpected drop, which can weaken consensus safety guarantees.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 83 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"validation_quorum\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Quorum\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "State Value Timeline", + "description": "###### What this is:\n*Numeric encoding of the server operating state (disconnected, connected, syncing, tracking, full, validating, proposing) over time.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the encoded state value.*\n\n###### Reading it:\n*A flat line at the full-operation state is healthy; steps show transitions.*\n\n###### Healthy range:\n*Steady at the highest (full) state.*\n\n###### Watch for:\n*Frequent transitions, useful for correlating state flapping with other metrics.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 91 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"state_value\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "State", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Time in Current State", + "description": "###### What this is:\n*How long the server has held its current operating state, in seconds.*\n\n###### How it's computed:\n*Current value of the time-in-state gauge.*\n\n###### Reading it:\n*Not yet wired in the code; the value currently always reads 0.*\n\n###### Healthy range:\n*Not applicable; the value is always 0 today.*\n\n###### Watch for:\n*n/a until implemented.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 99 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"time_in_current_state_seconds\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Time in State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "s", + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "State Changes Rate", + "description": "###### What this is:\n*Rate of server operating-state changes per hour.*\n\n###### How it's computed:\n*Per-hour rate of the state-change counter, averaged over a 1-hour window.*\n\n###### Reading it:\n*Near zero is healthy; each increment is one state transition.*\n\n###### Healthy range:\n*Near 0 changes per hour.*\n\n###### Watch for:\n*Frequent transitions, which point to network instability or configuration problems.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementStateChanges (caller NetworkOPs.cpp)`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 107 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(3600 * sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(state_changes_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Changes/hr\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix:/hr", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Ledgers Closed Rate", + "description": "###### What this is:\n*Ledgers closed per minute by this node.*\n\n###### How it's computed:\n*Per-second rate of the ledgers-closed counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should match the network's steady close cadence.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*Deviation from the expected cadence, which indicates consensus timing trouble or the node falling behind.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgersClosed (caller RCLConsensus.cpp)`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 115 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Closed/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "green", + "value": 8 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreement % (7d)", + "description": "###### What this is:\n*Share of ledgers over the trailing 7 days where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 7-day agreement percentage.*\n\n###### Reading it:\n*The long-term reliability window; higher is better.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A gradual decline, which reflects chronic rather than transient disagreement.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 123 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreement_pct_7d\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreement 7d\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreements vs Missed (7d)", + "description": "###### What this is:\n*Agreed versus missed validation counts over the trailing 7 days.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 7-day agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate across the week.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A rising missed trend, signalling sustained validator unreliability.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 131 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreements_7d\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreements 7d\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"missed_7d\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missed 7d\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["validator", "health"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Validator Health", + "uid": "validator-health", + "refresh": "10s" +} diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml new file mode 100644 index 00000000000..922caed0810 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -0,0 +1,65 @@ +# Grafana contact-point provisioning for rippled OTel alerts. +# +# Phase 9: Internal metric gap fill — alerting on health-critical metrics. +# +# A contact point is where a firing alert is delivered. Two are defined: +# xrpld-default — Slack only; receives warning-severity alerts. +# xrpld-critical — Slack + email; receives critical-severity alerts. +# The severity split is wired in policies.yaml. +# +# Secrets and personal addresses are NOT hard-coded. To enable delivery, +# replace the disabled placeholder values below with a real Slack webhook +# and alert email — e.g. by copying docker/telemetry/.env.alerting.example +# to .env.alerting (gitignored) and swapping the placeholder lines to +# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of +# docs/telemetry-runbook.md for setup. +# +# The defaults are deliberately non-empty, invalid-but-valid-shaped +# placeholders (an unroutable webhook host and a .invalid email). Grafana's +# alerting provisioning validator REQUIRES a non-empty Slack url and email +# addresses, and it does NOT support ${VAR:-default} expansion — an unset +# ${VAR} expands to empty and crashes Grafana on startup. The placeholders +# keep the whole stack booting with zero configuration; alerts simply route +# nowhere until a real destination is supplied. +# +# Email delivery additionally requires SMTP configured on the Grafana +# service (GF_SMTP_* in docker-compose.yml). + +apiVersion: 1 + +contactPoints: + # --- Warning tier: Slack only --- + - orgId: 1 + name: xrpld-default + receivers: + - uid: xrpld-slack-default + type: slack + settings: + # Disabled placeholder: an unroutable webhook host. A non-empty url + # selects Slack webhook mode (no recipient/token required) and keeps + # provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to + # enable delivery. + url: https://hooks.slack.invalid/disabled + title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}" + disableResolveMessage: false + + # --- Critical tier: Slack + email --- + - orgId: 1 + name: xrpld-critical + receivers: + - uid: xrpld-slack-critical + type: slack + settings: + # Disabled placeholder — see xrpld-slack-default above. + url: https://hooks.slack.invalid/disabled + title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}" + disableResolveMessage: false + - uid: xrpld-email-critical + type: email + settings: + # Disabled placeholder: a .invalid address keeps the required + # `addresses` field non-empty so provisioning validates. Replace + # with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver. + addresses: alerts-disabled@xrpld.invalid + singleEmail: true + disableResolveMessage: false diff --git a/docker/telemetry/grafana/provisioning/alerting/policies.yaml b/docker/telemetry/grafana/provisioning/alerting/policies.yaml new file mode 100644 index 00000000000..b1c4c92c705 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/alerting/policies.yaml @@ -0,0 +1,44 @@ +# Grafana notification-policy provisioning for rippled OTel alerts. +# +# Phase 9: Internal metric gap fill — alerting on health-critical metrics. +# +# The notification policy tree decides which contact point receives a firing +# alert and how alerts are batched. Routing is split by severity: +# - default (root) → xrpld-default (Slack) — catches warnings. +# - severity = critical → xrpld-critical (Slack + email) — child route. +# A critical alert matches the child route and is delivered to Slack AND email; +# everything else falls through to the root receiver. +# +# Grouping by alertname + service_instance_id means one notification per +# (alert, node) pair, so a fleet-wide problem does not collapse into a single +# ambiguous message. + +apiVersion: 1 + +policies: + - orgId: 1 + receiver: xrpld-default + group_by: + - alertname + - service_instance_id + # Wait before sending the first notification for a new group, so a burst of + # related series is batched together. + group_wait: 30s + # Wait before adding new alerts to an already-notified group. + group_interval: 5m + # Minimum time before re-sending a notification for an alert that is still + # firing. + repeat_interval: 4h + routes: + # Critical alerts page harder: Slack + email. matchers uses the + # PromQL-style label match "severity = critical" set on the rules. + - receiver: xrpld-critical + matchers: + - severity = critical + group_by: + - alertname + - service_instance_id + group_wait: 30s + group_interval: 5m + # Re-page critical alerts more often than the 4h default while still firing. + repeat_interval: 1h diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 00000000000..cf396da0c50 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,410 @@ +# Grafana alert-rule provisioning for rippled OTel metrics. +# +# Phase 9: Internal metric gap fill — alerting on health-critical metrics. +# +# Six rules across three node-fatal subsystems: consensus/ledger health, +# validator health, and the job queue. Every metric referenced here is +# introduced by phase 9's MetricsRegistry. +# +# Rule shape (Grafana server-side evaluation): +# A Prometheus query — a 5-minute rate / histogram_quantile, aggregated +# `by (service_instance_id)` so each node evaluates +# independently. Alert rules run headless and cannot +# use the dashboards' `$node` template variable. +# B reduce (last) — collapse A's series to its most recent value. +# C threshold — the firing condition; `condition: C`. +# +# Thresholds are documented in docs/telemetry-runbook.md (Alerting section) +# and are intended to be tuned. + +apiVersion: 1 + +groups: + # ------------------------------------------------------------------ # + # Consensus / ledger health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-consensus + folder: xrpld + interval: 1m + rules: + # A closed ledger that later fails validation against the network — + # any sustained nonzero rate means this node built history the rest + # of the network rejects. + - uid: xrpld-ledger-history-mismatch + title: LedgerHistoryMismatch + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: critical + category: consensus + annotations: + summary: "Ledger history mismatch on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is recording ledger history + mismatches ({{ $values.B.Value }}/s over 5m). The node's built ledger + diverges from the validated network chain. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(ledger_history_mismatch_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0] + datasource: + type: __expr__ + uid: __expr__ + + # Healthy nodes close a ledger every ~3-5s. Zero closes for 3 minutes + # means consensus/ledger advancement is stuck. NoData (metric absent) + # also fires — a vanished series here means the node is down. + - uid: xrpld-ledger-close-stalled + title: LedgerCloseStalled + condition: C + for: 3m + noDataState: Alerting + execErrState: Error + labels: + severity: critical + category: consensus + annotations: + summary: "Ledger closing stalled on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has closed no ledgers for + several minutes (5m rate has decayed to zero). Consensus or ledger + advancement is stuck. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(ledgers_closed_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: lt + params: [0.001] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Validator health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-validator + folder: xrpld + interval: 1m + rules: + # This validator's own validations are not reaching / agreeing with + # the network. A sustained nonzero miss rate risks the validator being + # dropped from UNLs. + - uid: xrpld-validations-missed + title: ValidationsMissed + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: validator + annotations: + summary: "Validations missed on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is missing validations + ({{ $values.B.Value }}/s over 5m). Its validations are not agreeing with + the validated ledger. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(validation_missed_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0] + datasource: + type: __expr__ + uid: __expr__ + + # The node has stopped checking incoming validations. Zero checked + # validations means it is no longer processing the validation stream + # from peers. + - uid: xrpld-validations-not-checked + title: ValidationsNotChecked + condition: C + for: 5m + noDataState: Alerting + execErrState: Error + labels: + severity: warning + category: validator + annotations: + summary: "No validations checked on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has checked no incoming + validations for several minutes (5m rate has decayed to zero). The + validation stream from peers may have stopped. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(validations_checked_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: lt + params: [0.001] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Job queue / resource health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-jobqueue + folder: xrpld + interval: 1m + rules: + # Transactions are being dropped because the job queue is full — the + # node is shedding load it cannot process. + - uid: xrpld-jobqueue-tx-overflow + title: JobQueueTxOverflow + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: jobqueue + annotations: + summary: "Job queue transaction overflow on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is overflowing its transaction + job queue ({{ $values.B.Value }}/s over 5m). Transactions are being + dropped under load. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(jq_trans_overflow_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0] + datasource: + type: __expr__ + uid: __expr__ + + # p99 time a job waits in the queue before running. A sustained p99 + # above 1s means the node is saturated and work is backing up. Tune the + # threshold (in microseconds) to the deployment. + - uid: xrpld-jobqueue-latency-high + title: JobQueueLatencyHigh + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: jobqueue + annotations: + summary: "Job queue latency high on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has a p99 job-queue wait of + {{ $values.B.Value }}µs (>1s) over 5m. The node is saturated and jobs are + backing up. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m]))) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [1000000] + datasource: + type: __expr__ + uid: __expr__ diff --git a/docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml b/docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 00000000000..dec8dc87c0f --- /dev/null +++ b/docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: xrpld-telemetry + orgId: 1 + folder: xrpld + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/docker/telemetry/grafana/provisioning/datasources/loki.yaml b/docker/telemetry/grafana/provisioning/datasources/loki.yaml new file mode 100644 index 00000000000..0ae48fe7094 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/datasources/loki.yaml @@ -0,0 +1,35 @@ +# Grafana Loki data source provisioning for rippled log-trace correlation. +# +# Loki ingests rippled logs via OTel Collector's filelog receiver. +# The derivedFields config links trace_id values in log lines back to +# Tempo traces, enabling one-click log-to-trace navigation in Grafana. + +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: loki + jsonData: + derivedFields: + # Trace ID is an OTel-issued 32-hex value emitted by Logs::format() + # as `trace_id=...`. Grafana treats the captured value as a Tempo + # trace ID, opening the trace directly via Tempo's trace-by-id API. + - datasourceUid: tempo + matcherRegex: "trace_id=(\\w+)" + name: TraceID + url: "$${__value.raw}" + # 64-char uppercase hex tokens in log bodies are XRPL ledger + # hashes (or tx hashes). They are NOT OTel trace IDs and cannot + # be resolved via the trace-by-id endpoint. Build a Grafana + # Explore deep-link that runs TraceQL on the span attribute + # `xrpl.consensus.ledger_id` — set on `consensus.round` spans to + # the full prev_ledger hash. This finds the round span whose + # first 16 bytes were folded into the OTel trace_id by + # SpanGuard::hashSpan(). + - matcherRegex: "\\b([A-F0-9]{64})\\b" + name: ConsensusLedgerHash + urlDisplayLabel: "Search Tempo" + url: '/explore?schemaVersion=1&orgId=1&panes={"tempo":{"datasource":"tempo","queries":[{"refId":"A","queryType":"traceql","query":"{.xrpl.consensus.ledger_id=\"$${__value.raw}\"}"}],"range":{"from":"now-1h","to":"now"}}}' diff --git a/docker/telemetry/grafana/provisioning/datasources/prometheus.yaml b/docker/telemetry/grafana/provisioning/datasources/prometheus.yaml new file mode 100644 index 00000000000..af7492822a3 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/datasources/prometheus.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml new file mode 100644 index 00000000000..f89fa89eec5 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -0,0 +1,228 @@ +# Grafana datasource provisioning for Grafana Tempo. +# Auto-configures Tempo as a trace data source on Grafana startup. +# Access Grafana at http://localhost:3000, then use Explore -> Tempo +# to browse xrpld traces using TraceQL. +# +# Search filters provide pre-configured dropdowns in the Explore UI. +# Each phase adds filters for the span attributes it introduces. +# Base filters — node identity, service, span name, status. +# RPC command, status, role filters. +# Transaction hash, local/peer origin, status. +# Consensus mode, round, ledger sequence, close time. + +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + uid: tempo + jsonData: + nodeGraph: + enabled: true + # Service map and traces-to-metrics require a Prometheus datasource + # (not included in this stack). These features are inactive until a + # Prometheus service is added to docker-compose.yml. + serviceMap: + datasourceUid: prometheus + # Trace-to-log correlation — enables one-click navigation + # from a Tempo trace to the corresponding Loki log lines. Filters + # by trace_id so only logs from the same trace are shown. + tracesToLogs: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + tags: [] + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + search: + filters: + # --- Node identification filters --- + # service.name: logical service name (default: "xrpld"). + # Useful when running multiple service types in the same collector. + - id: service-name + tag: service.name + operator: "=" + scope: resource + type: static + # service.instance.id: unique node identifier — configurable via + # the service_instance_id setting in [telemetry], defaults to the + # node's public key. E.g. "Node-1" or "nHB1X37...". + - id: node-id + tag: service.instance.id + operator: "=" + scope: resource + type: dynamic + # service.version: xrpld build version (e.g., "2.4.0-b1"). + # Filter traces from specific software releases. + - id: node-version + tag: service.version + operator: "=" + scope: resource + type: dynamic + # xrpl.network.id: numeric network identifier + # (0 = mainnet, 1 = testnet, 2 = devnet, etc.). + # Derived from the [network_id] config section. + - id: network-id + tag: xrpl.network.id + operator: "=" + scope: resource + type: dynamic + # xrpl.network.type: human-readable network name derived from + # network ID ("mainnet", "testnet", "devnet", "unknown"). + - id: network-type + tag: xrpl.network.type + operator: "=" + scope: resource + type: dynamic + # --- Span intrinsic filters --- + # name: the span operation name (e.g., "rpc.command.server_info"). + # Use to find traces for a specific RPC command or subsystem. + - id: span-name + tag: name + operator: "=" + scope: intrinsic + type: dynamic + # status: span completion status ("ok", "error", "unset"). + # Filter for failed operations to diagnose errors. + - id: span-status + tag: status + operator: "=" + scope: intrinsic + type: dynamic + # duration: span wall-clock duration. Use with ">" operator + # to find slow operations (e.g., duration > 500ms). + - id: span-duration + tag: duration + operator: ">" + scope: intrinsic + type: dynamic + # RPC tracing filters + - id: rpc-command + tag: command + operator: "=" + scope: span + type: dynamic + - id: rpc-status + tag: rpc_status + operator: "=" + scope: span + type: dynamic + - id: rpc-role + tag: rpc_role + operator: "=" + scope: span + type: dynamic + # Transaction tracing filters + - id: tx-hash + tag: tx_hash + operator: "=" + scope: span + type: static + - id: tx-origin + tag: local + operator: "=" + scope: span + type: dynamic + - id: tx-status + tag: tx_status + operator: "=" + scope: span + type: dynamic + # tx_type: transaction type (e.g., "Payment", "OfferCreate"). + - id: tx-type + tag: tx_type + operator: "=" + scope: span + type: dynamic + # Consensus tracing filters + - id: consensus-mode + tag: consensus_mode + operator: "=" + scope: span + type: static + - id: consensus-round + tag: consensus_round + operator: "=" + scope: span + type: dynamic + - id: consensus-ledger-seq + tag: ledger_seq + operator: "=" + scope: span + type: static + # ledger_hash: ledger hash — scope all spans to a specific closed ledger. + - id: ledger-hash + tag: ledger_hash + operator: "=" + scope: span + type: static + - id: consensus-close-time-correct + tag: close_time_correct + operator: "=" + scope: span + type: static + - id: consensus-state + tag: consensus_state + operator: "=" + scope: span + type: static + - id: consensus-close-resolution + tag: close_resolution_ms + operator: "=" + scope: span + type: dynamic + - id: consensus-proposers + tag: proposers + operator: "=" + scope: span + type: dynamic + - id: consensus-result + tag: consensus_result + operator: "=" + scope: span + type: dynamic + - id: consensus-mode-old + tag: mode_old + operator: "=" + scope: span + type: dynamic + - id: consensus-mode-new + tag: mode_new + operator: "=" + scope: span + type: dynamic + - id: consensus-ledger-id + tag: consensus_ledger_id + operator: "=" + scope: span + type: static + # Additional transaction and queue filters + - id: tx-path + tag: path + operator: "=" + scope: span + type: dynamic + - id: tx-suppressed + tag: suppressed + operator: "=" + scope: span + type: dynamic + - id: peer-version + tag: peer_version + operator: "=" + scope: span + type: dynamic + - id: txq-status + tag: txq_status + operator: "=" + scope: span + type: dynamic + - id: txq-ter-code + tag: ter_code + operator: "=" + scope: span + type: dynamic diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh new file mode 100755 index 00000000000..b4401d59060 --- /dev/null +++ b/docker/telemetry/integration-test.sh @@ -0,0 +1,756 @@ +#!/usr/bin/env bash +# Integration test for rippled OpenTelemetry instrumentation. +# +# Launches a 6-node xrpld consensus network with telemetry enabled, +# exercises RPC / transaction / consensus code paths, then verifies +# that the expected spans and metrics appear in Tempo and Prometheus. +# +# Usage: +# bash docker/telemetry/integration-test.sh +# +# Prerequisites: +# - .build/xrpld built with telemetry=ON +# - docker compose (v2) +# - curl, jq +# +# The script leaves the observability stack and xrpld nodes running +# so you can manually inspect Tempo (localhost:3200) and Grafana +# (localhost:3000). Run with --cleanup to tear down instead. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +XRPLD="$REPO_ROOT/.build/xrpld" +COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml" +STANDALONE_CFG="$SCRIPT_DIR/xrpld-telemetry.cfg" +WORKDIR="${WORKDIR:-/tmp/xrpld-integration}" +NUM_NODES=6 +PEER_PORT_BASE=51235 +RPC_PORT_BASE=5005 +CONSENSUS_TIMEOUT=120 +GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" +GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb" +DEST_ACCOUNT="" # Generated dynamically via wallet_propose +TEMPO="http://localhost:3200" +PROM="http://localhost:9090" + +# Counters for pass/fail +PASS=0 +FAIL=0 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[INFO]\033[0m %s\n" "$*"; } +ok() { + printf "\033[1;32m[PASS]\033[0m %s\n" "$*" + PASS=$((PASS + 1)) +} +fail() { + printf "\033[1;31m[FAIL]\033[0m %s\n" "$*" + FAIL=$((FAIL + 1)) +} +die() { + printf "\033[1;31m[ERROR]\033[0m %s\n" "$*" >&2 + exit 1 +} + +check_span() { + local op="$1" + local count + count=$(curl -sf "$TEMPO/api/search" \ + --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ + --data-urlencode "limit=5" | + jq '.traces | length' 2>/dev/null || echo 0) + if [ "$count" -gt 0 ]; then + ok "$op ($count traces)" + else + fail "$op (0 traces)" + fi +} + +# Verify trace_id injection in xrpld log output. +# Greps all node debug.log files for the "trace_id= span_id=" +# pattern that Logs::format() injects when an active OTel span exists. +# Also cross-checks that a trace_id found in logs matches a trace in Tempo. +check_log_correlation() { + log "Checking log-trace correlation..." + + local total_matches=0 + local files_scanned=0 + local sample_trace_id="" + + for i in $(seq 1 "$NUM_NODES"); do + local logfile="$WORKDIR/node$i/debug.log" + if [ ! -f "$logfile" ]; then + continue + fi + files_scanned=$((files_scanned + 1)) + local matches + matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0 + total_matches=$((total_matches + matches)) + # Capture the first trace_id we find for cross-referencing with Tempo + if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then + sample_trace_id=$(grep -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | head -1 | cut -d= -f2) + fi + done + + if [ "$files_scanned" -eq 0 ]; then + fail "Log correlation: no debug.log files found in $WORKDIR/node*/" + return + fi + + if [ "$total_matches" -gt 0 ]; then + ok "Log correlation: found $total_matches log lines with trace_id ($files_scanned nodes scanned)" + else + fail "Log correlation: no trace_id found in any node debug.log ($files_scanned nodes scanned)" + fi + + # Cross-check: verify the sample trace_id exists in Tempo + if [ -n "$sample_trace_id" ]; then + local trace_found + # Tempo /api/traces/{id} returns OTLP shape: {"batches":[...]} + trace_found=$(curl -sf "$TEMPO/api/traces/$sample_trace_id" | + jq '.batches | length' 2>/dev/null) || trace_found=0 + if [ "$trace_found" -gt 0 ]; then + ok "Log-Tempo cross-check: trace_id=$sample_trace_id found in Tempo" + else + fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo" + fi + fi +} + +cleanup() { + log "Cleaning up..." + # Kill xrpld nodes + for i in $(seq 1 "$NUM_NODES"); do + local pidfile="$WORKDIR/node$i/xrpld.pid" + if [ -f "$pidfile" ]; then + kill "$(cat "$pidfile")" 2>/dev/null || true + rm -f "$pidfile" + fi + done + # Also kill any straggling xrpld processes from our workdir + pkill -f "$WORKDIR" 2>/dev/null || true + # Stop docker stack + docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true + # Remove workdir + rm -rf "$WORKDIR" + log "Cleanup complete." +} + +# Handle --cleanup flag +if [ "${1:-}" = "--cleanup" ]; then + cleanup + exit 0 +fi + +# --------------------------------------------------------------------------- +# Step 0: Prerequisites +# --------------------------------------------------------------------------- +log "Checking prerequisites..." + +command -v docker >/dev/null 2>&1 || die "docker not found" +docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found" +command -v curl >/dev/null 2>&1 || die "curl not found" +command -v jq >/dev/null 2>&1 || die "jq not found" +[ -x "$XRPLD" ] || die "xrpld binary not found at $XRPLD (build with telemetry=ON)" +[ -f "$COMPOSE_FILE" ] || die "docker-compose.yml not found at $COMPOSE_FILE" +[ -f "$STANDALONE_CFG" ] || die "xrpld-telemetry.cfg not found at $STANDALONE_CFG" + +log "All prerequisites met." + +# --------------------------------------------------------------------------- +# Step 1: Clean previous run +# --------------------------------------------------------------------------- +log "Cleaning previous run data..." +for i in $(seq 1 "$NUM_NODES"); do + pidfile="$WORKDIR/node$i/xrpld.pid" + if [ -f "$pidfile" ]; then + kill "$(cat "$pidfile")" 2>/dev/null || true + fi +done +pkill -f "$WORKDIR" 2>/dev/null || true +# Kill any xrpld using the standalone config (from key generation) +pkill -f "xrpld-telemetry.cfg" 2>/dev/null || true +sleep 2 +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" + +# --------------------------------------------------------------------------- +# Step 2: Start observability stack +# --------------------------------------------------------------------------- +log "Starting observability stack..." +# Point the collector's log mount at this test's workdir so it tails the +# per-node debug.log files this script generates. The compose default +# (./data/logs) is for user-run xrpld; the test owns its own log root. +XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d + +log "Waiting for otel-collector to be ready..." +for attempt in $(seq 1 30); do + # The OTLP HTTP endpoint returns 405 for GET (expects POST), which + # means it is listening. curl -sf would fail on 405, so we check + # the HTTP status code explicitly. + status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + if [ "$status" != "000" ]; then + log "otel-collector ready (attempt $attempt, HTTP $status)." + break + fi + if [ "$attempt" -eq 30 ]; then + die "otel-collector not ready after 30s" + fi + sleep 1 +done + +log "Waiting for Tempo to be ready..." +for attempt in $(seq 1 30); do + if curl -sf "$TEMPO/ready" >/dev/null 2>&1; then + log "Tempo ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 30 ]; then + die "Tempo not ready after 30s" + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 3: Generate validator keys +# --------------------------------------------------------------------------- +log "Generating $NUM_NODES validator key pairs..." + +# Start a temporary standalone xrpld for key generation +TEMP_DATA="$WORKDIR/temp-keygen" +mkdir -p "$TEMP_DATA" + +# Create a minimal temp config for key generation +TEMP_CFG="$TEMP_DATA/xrpld.cfg" +cat >"$TEMP_CFG" <"$TEMP_DATA/stdout.log" 2>&1 & +TEMP_PID=$! +log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..." + +for attempt in $(seq 1 30); do + if curl -sf http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then + log "Temporary xrpld RPC ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 30 ]; then + kill "$TEMP_PID" 2>/dev/null || true + die "Temporary xrpld RPC not ready after 30s" + fi + sleep 1 +done + +declare -a SEEDS +declare -a PUBKEYS + +for i in $(seq 1 "$NUM_NODES"); do + result=$(curl -sf http://localhost:5099 -d '{"method":"validation_create"}') + seed=$(echo "$result" | jq -r '.result.validation_seed') + pubkey=$(echo "$result" | jq -r '.result.validation_public_key') + if [ -z "$seed" ] || [ "$seed" = "null" ]; then + kill "$TEMP_PID" 2>/dev/null || true + die "Failed to generate key pair $i" + fi + SEEDS+=("$seed") + PUBKEYS+=("$pubkey") + log " Node $i: $pubkey" +done + +kill "$TEMP_PID" 2>/dev/null || true +wait "$TEMP_PID" 2>/dev/null || true +rm -rf "$TEMP_DATA" +log "Key generation complete." + +# --------------------------------------------------------------------------- +# Step 4: Generate node configs and validators.txt +# --------------------------------------------------------------------------- +log "Generating node configs..." + +# Create shared validators.txt +VALIDATORS_FILE="$WORKDIR/validators.txt" +{ + echo "[validators]" + for i in $(seq 0 $((NUM_NODES - 1))); do + echo "${PUBKEYS[$i]}" + done +} >"$VALIDATORS_FILE" + +# Create per-node configs +for i in $(seq 1 "$NUM_NODES"); do + NODE_DIR="$WORKDIR/node$i" + mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db" + + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + PEER_PORT=$((PEER_PORT_BASE + i - 1)) + SEED="${SEEDS[$((i - 1))]}" + + # Build ips_fixed list (all peers except self) + IPS_FIXED="" + for j in $(seq 1 "$NUM_NODES"); do + if [ "$j" -ne "$i" ]; then + IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1)) +" + fi + done + + cat >"$NODE_DIR/xrpld.cfg" <"$NODE_DIR/stdout.log" 2>&1 & + echo $! >"$NODE_DIR/xrpld.pid" + log " Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))" +done + +# Give nodes a moment to initialize +sleep 5 + +# --------------------------------------------------------------------------- +# Step 6: Wait for consensus +# --------------------------------------------------------------------------- +log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}s)..." + +start_time=$(date +%s) +nodes_ready=0 + +while [ "$nodes_ready" -lt "$NUM_NODES" ]; do + elapsed=$(($(date +%s) - start_time)) + if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then + fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)" + log "Continuing with partial consensus..." + break + fi + + nodes_ready=0 + for i in $(seq 1 "$NUM_NODES"); do + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + state=$(curl -sf "http://localhost:$RPC_PORT" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable") + if [ "$state" = "proposing" ]; then + nodes_ready=$((nodes_ready + 1)) + fi + done + printf "\r %d/%d nodes proposing (%ds elapsed)..." "$nodes_ready" "$NUM_NODES" "$elapsed" + if [ "$nodes_ready" -lt "$NUM_NODES" ]; then + sleep 3 + fi +done +echo "" + +if [ "$nodes_ready" -eq "$NUM_NODES" ]; then + ok "All $NUM_NODES nodes reached 'proposing' state" +else + fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state" +fi + +# --------------------------------------------------------------------------- +# Step 6b: Wait for validated ledger +# --------------------------------------------------------------------------- +log "Waiting for first validated ledger..." +for attempt in $(seq 1 60); do + val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$val_seq" -gt 2 ] 2>/dev/null; then + ok "First validated ledger: seq $val_seq" + break + fi + if [ "$attempt" -eq 60 ]; then + fail "No validated ledger after 60s" + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 7: Exercise RPC spans (Phase 2) +# --------------------------------------------------------------------------- +log "Exercising RPC spans..." + +curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_info"}' >/dev/null +curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_state"}' >/dev/null +curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null + +log "RPC commands sent. Waiting 5s for batch export..." +sleep 5 + +# --------------------------------------------------------------------------- +# Step 8: Submit transaction (Phase 3) +# --------------------------------------------------------------------------- +log "Submitting Payment transaction..." + +# Generate a destination wallet +log " Generating destination wallet..." +wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"wallet_propose"}') +DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null) +if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then + fail "Could not generate destination wallet" + DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback +fi +log " Destination: $DEST_ACCOUNT" + +# Get genesis account info +acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") +seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown") +log " Genesis account sequence: $seq_num" + +# Submit payment +submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") + +engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown") +tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown") + +if [ "$engine_result" = "tesSUCCESS" ] || [ "$engine_result" = "terQUEUED" ]; then + ok "Transaction submitted: $engine_result (hash: ${tx_hash:0:16}...)" +else + fail "Transaction submission: $engine_result" + log " Full response: $(echo "$submit_result" | jq -c .result 2>/dev/null)" +fi + +log "Waiting 15s for consensus round + batch export..." +sleep 15 + +# --------------------------------------------------------------------------- +# Step 9: Verify Tempo traces +# --------------------------------------------------------------------------- +log "Verifying spans in Tempo..." + +# Check service registration +services=$(curl -sf "$TEMPO/api/v2/search/tag/resource.service.name/values" | + jq -r '.tagValues[].value' 2>/dev/null || echo "") +if echo "$services" | grep -q "xrpld"; then + ok "Service 'xrpld' registered in Tempo" +else + fail "Service 'xrpld' NOT found in Tempo (found: $services)" +fi + +log "" +log "--- Phase 2: RPC Spans ---" +check_span "rpc.request" +check_span "rpc.process" +check_span "rpc.command.server_info" +check_span "rpc.command.server_state" +check_span "rpc.command.ledger" + +log "" +log "--- Phase 3: Transaction Spans ---" +check_span "tx.process" +check_span "tx.receive" +check_span "tx.apply" + +log "" +log "--- Phase 4: Consensus Spans ---" +check_span "consensus.proposal.send" +check_span "consensus.ledger_close" +check_span "consensus.accept" +check_span "consensus.validation.send" + +log "" +log "--- Phase 5: Ledger Spans ---" +check_span "ledger.build" +check_span "ledger.validate" +check_span "ledger.store" + +log "" +log "--- Phase 5: Peer Spans (trace_peer=1) ---" +check_span "peer.proposal.receive" +check_span "peer.validation.receive" + +# --------------------------------------------------------------------------- +# Step 9b: Verify log-trace correlation (Phase 8) +# --------------------------------------------------------------------------- +log "" +log "--- Log-Trace Correlation ---" +check_log_correlation + +# --------------------------------------------------------------------------- +# Step 10: Verify Prometheus spanmetrics +# --------------------------------------------------------------------------- +log "" +log "--- Phase 5: Spanmetrics ---" +log "Waiting 20s for Prometheus scrape cycle..." +sleep 20 + +calls_count=$(curl -sf "$PROM/api/v1/query?query=span_calls_total" | + jq '.data.result | length' 2>/dev/null || echo 0) +if [ "$calls_count" -gt 0 ]; then + ok "Prometheus: span_calls_total ($calls_count series)" +else + fail "Prometheus: span_calls_total (0 series)" +fi + +duration_count=$(curl -sf "$PROM/api/v1/query?query=span_duration_milliseconds_count" | + jq '.data.result | length' 2>/dev/null || echo 0) +if [ "$duration_count" -gt 0 ]; then + ok "Prometheus: duration histogram ($duration_count series)" +else + fail "Prometheus: duration histogram (0 series)" +fi + +# Check Grafana +if curl -sf http://localhost:3000/api/health >/dev/null 2>&1; then + ok "Grafana: healthy at localhost:3000" +else + fail "Grafana: not reachable at localhost:3000" +fi + +# --------------------------------------------------------------------------- +# Step 10b: Verify native OTel metrics in Prometheus (beast::insight) +# --------------------------------------------------------------------------- +log "" +log "--- Phase 7: Native OTel Metrics (beast::insight via OTLP) ---" +log "Waiting 20s for OTLP metric export + Prometheus scrape..." +sleep 20 + +check_otel_metric() { + local metric_name="$1" + local result + result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" | + jq '.data.result | length' 2>/dev/null || echo 0) + if [ "$result" -gt 0 ]; then + ok "OTel: $metric_name ($result series)" + else + fail "OTel: $metric_name (0 series)" + fi +} + +# Node health gauges (ObservableGauge — no _total suffix) +check_otel_metric "ledgermaster_validated_ledger_age" +check_otel_metric "ledgermaster_published_ledger_age" +check_otel_metric "job_count" + +# State accounting +check_otel_metric "state_accounting_full_duration" + +# Peer finder +check_otel_metric "peer_finder_active_inbound_peers" +check_otel_metric "peer_finder_active_outbound_peers" + +# RPC counters (Counter — Prometheus adds _total suffix automatically) +check_otel_metric "rpc_requests_total" + +# Overlay traffic +check_otel_metric "total_bytes_in" + +# Verify StatsD receiver is NOT required (no statsd receiver in pipeline) +log "" +log "--- Verify StatsD receiver is not required ---" +# StatsD listens on UDP 8125, so probe with a UDP-aware tool, not curl (TCP). +if command -v ss >/dev/null 2>&1; then + if ss -ulnp 2>/dev/null | grep -q ":8125"; then + fail "StatsD port 8125 appears to be listening (should not be needed)" + else + ok "StatsD port 8125 is not listening (not required)" + fi +else + log "ss not found -- skipping StatsD UDP port check" +fi + +# --------------------------------------------------------------------------- +# Step 10c: Verify Phase 9 OTel SDK Metrics +# --------------------------------------------------------------------------- +log "" +log "--- Phase 9: OTel SDK Metrics (MetricsRegistry) ---" +log "Waiting 15s for OTel metric export + Prometheus scrape..." +sleep 15 + +check_otel_metric() { + local metric_name="$1" + local result + result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" | + jq '.data.result | length' 2>/dev/null || echo 0) + if [ "$result" -gt 0 ]; then + ok "OTel: $metric_name ($result series)" + else + fail "OTel: $metric_name (0 series)" + fi +} + +# Task 9.1: NodeStore I/O +check_otel_metric 'nodestore_state{metric="node_reads_total"}' +check_otel_metric 'nodestore_state{metric="write_load"}' + +# Task 9.2: Cache hit rates +check_otel_metric 'cache_metrics{metric="SLE_hit_rate"}' +check_otel_metric 'cache_metrics{metric="treenode_cache_size"}' + +# Task 9.3: TxQ metrics +check_otel_metric 'txq_metrics{metric="txq_count"}' +check_otel_metric 'txq_metrics{metric="txq_reference_fee_level"}' + +# Task 9.4: Per-RPC metrics +check_otel_metric "rpc_method_started_total" +check_otel_metric "rpc_method_finished_total" + +# Task 9.5: Per-job metrics +check_otel_metric "job_queued_total" +check_otel_metric "job_finished_total" + +# Task 9.6: Counted object instances +check_otel_metric "object_count" + +# Task 9.7: Load factor breakdown +check_otel_metric 'load_factor_metrics{metric="load_factor"}' +check_otel_metric 'load_factor_metrics{metric="load_factor_server"}' + +# Task 7.15 / Phase 9: ValidationTracker rolling-window agreement gauge. +# MetricsRegistry::registerValidationAgreementGauge() publishes +# validation_agreement with a `metric` label for each window +# (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day +# window matches the external xrpl-validator-dashboard parity target. +check_otel_metric 'validation_agreement{metric="agreement_pct_1h"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_24h"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_7d"}' +check_otel_metric 'validation_agreement{metric="agreements_1h"}' +check_otel_metric 'validation_agreement{metric="missed_1h"}' +check_otel_metric 'validation_agreement{metric="agreements_24h"}' +check_otel_metric 'validation_agreement{metric="missed_24h"}' +check_otel_metric 'validation_agreement{metric="agreements_7d"}' +check_otel_metric 'validation_agreement{metric="missed_7d"}' + +# --------------------------------------------------------------------------- +# Step 11: Summary +# --------------------------------------------------------------------------- +echo "" +echo "===========================================================" +echo " INTEGRATION TEST RESULTS" +echo "===========================================================" +printf " \033[1;32mPASSED: %d\033[0m\n" "$PASS" +printf " \033[1;31mFAILED: %d\033[0m\n" "$FAIL" +echo "===========================================================" +echo "" +echo " Observability stack is running:" +echo "" +echo " Tempo: http://localhost:3200" +echo " Grafana: http://localhost:3000" +echo " Prometheus: http://localhost:9090" +echo " Loki: http://localhost:3100" +echo "" +echo " xrpld nodes (6) are running:" +for i in $(seq 1 "$NUM_NODES"); do + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + PEER_PORT=$((PEER_PORT_BASE + i - 1)) + echo " Node $i: RPC=localhost:$RPC_PORT Peer=:$PEER_PORT PID=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')" +done +echo "" +echo " To tear down:" +echo " bash docker/telemetry/integration-test.sh --cleanup" +echo "" +echo "===========================================================" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml new file mode 100644 index 00000000000..50efeb5d57a --- /dev/null +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -0,0 +1,237 @@ +# OpenTelemetry Collector configuration — Grafana Cloud dual-export variant. +# +# Identical to otel-collector-config.yaml (local Tempo/Prometheus/Loki) but +# ALSO ships all three signals to Grafana Cloud over a single OTLP/HTTP +# endpoint. Local backends are kept so the on-box stack still works as a +# fallback; remove the local exporters from the pipelines below if you want +# cloud-only. +# +# Selecting this config is a RUNTIME choice — no code or image changes: +# Local only (default): +# docker compose -f docker/telemetry/docker-compose.yml up -d +# Local + Grafana Cloud: +# docker compose -f docker/telemetry/docker-compose.yml \ +# -f docker/telemetry/docker-compose.grafanacloud.yaml up -d +# +# The override file mounts THIS config over the collector's config path and +# injects the three required secrets as environment variables: +# GRAFANA_CLOUD_OTLP_ENDPOINT e.g. https://otlp-gateway-.grafana.net/otlp +# GRAFANA_CLOUD_INSTANCE_ID numeric Grafana Cloud stack/instance id +# GRAFANA_CLOUD_API_TOKEN a Cloud Access Policy token with metrics/traces/logs:write +# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)". + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + # Basic-auth for the Grafana Cloud OTLP gateway: username = instance id, + # password = API token. Values come from the container environment so no + # secret is committed to the repo. + basicauth/grafanacloud: + client_auth: + username: ${env:GRAFANA_CLOUD_INSTANCE_ID} + password: ${env:GRAFANA_CLOUD_API_TOKEN} + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + filelog: + include: + - /var/log/xrpld/*/debug.log + operators: + - type: regex_parser + regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' + timestamp: + parse_from: attributes.timestamp + layout: "%Y-%b-%d %H:%M:%S.%f" + location: UTC + +processors: + batch: + timeout: 1s + send_batch_size: 100 + # Box-only (uncommitted): probabilistic 0.5% tail sampling on the + # trace-storage branch only. spanmetrics-derived metrics see 100% of spans + # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores + # ~1/200 of traces to keep long-running node cost bounded. + tail_sampling: + decision_wait: 10s + num_traces: 50000 + policies: + - name: half-percent + type: probabilistic + probabilistic: + sampling_percentage: 0.5 + resource/logs: + attributes: + - key: service.name + value: xrpld + action: upsert + - key: job + value: xrpld + action: upsert + # Deployment-tier tagging. Each collector serves ONE environment and ONE + # network, so it stamps both onto every signal it forwards to Grafana + # Cloud, letting one cloud stack hold data from many collectors and + # filter by tier. + # - deployment.environment: the collector IS the environment (local, ci, + # test, prod), so it is authoritative -> upsert (overwrite). + # - xrpl.network.type: the xrpld node knows its own chain and already + # stamps this, so the collector only fills it when absent -> insert. + # This keeps a node's real network (e.g. a local node on mainnet) + # from being overwritten by a collector's default. + # Replace the placeholder values per collector; see the "Deployment + # Tiers" section of docs/telemetry-runbook.md. + resource/tier: + attributes: + - key: deployment.environment + value: local + action: upsert + - key: xrpl.network.type + value: mainnet + action: insert + # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version). + # The OpenTelemetry SDK auto-adds these to every Resource; they carry no + # operational value, so drop them for every signal on every backend. + resource/stripsdk: + attributes: + - key: telemetry.sdk.language + action: delete + - key: telemetry.sdk.name + action: delete + - key: telemetry.sdk.version + action: delete + # Grafana Cloud ingests metrics via OTLP (no Prometheus scrape), so the + # tier/instance resource attributes never become series labels the way the + # local prometheus exporter promotes them (resource_to_telemetry_conversion + # is Prometheus-exporter-only). Copy them onto datapoint labels so the + # dashboards' $node / $deployment_environment / $xrpl_network_type filters + # resolve on Cloud exactly as they do locally. + # + # NOTE: service.name is intentionally NOT copied here. Cloud's OTLP ingest + # already promotes service.name to the `service_name` label, and adding a + # datapoint attribute with the same name causes Mimir to concatenate both + # values as `xrpld;xrpld`. service.instance.id does not exhibit the same + # doubling, so the copy for $node is kept. + transform/cloudlabels: + metric_statements: + - context: datapoint + statements: + - set(attributes["service_instance_id"], resource.attributes["service.instance.id"]) + - set(attributes["deployment_environment"], resource.attributes["deployment.environment"]) + - set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"]) + +connectors: + spanmetrics: + namespace: "span" + # Emit span-derived metrics every 15s (default is 60s). Grafana Cloud + # ingests these as one datapoint per flush, and dashboard rate() panels + # use [$__rate_interval], which at short ranges (<=1h) can be shorter than + # the flush interval. A 60s flush left <2 samples per rate window on Cloud, + # so those panels rendered "No data". 15s keeps >=2 samples per window at + # every dashboard range, at the cost of ~4x span-metric datapoints to Cloud. + metrics_flush_interval: 15s + # Resource attributes that define the resource-metrics grouping key. + # All resource attributes propagate to the output metrics regardless; + # label promotion is handled by resource_to_telemetry_conversion on the + # prometheus exporter. Listing the per-node id and tier keys keeps + # series from distinct nodes/tiers grouped separately. + resource_metrics_key_attributes: + - service.instance.id + - deployment.environment + - xrpl.network.type + histogram: + explicit: + buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s] + dimensions: + - name: command + - name: rpc_status + - name: consensus_mode + - name: close_time_correct + - name: consensus_state + - name: local + - name: suppressed + - name: proposal_trusted + - name: validation_trusted + - name: tx_type + - name: ter_result + - name: stage + - name: txq_status + - name: load_type + - name: is_batch + - name: mode_new + - name: consensus_stalled + - name: consensus_phase + - name: consensus_result + - name: method + - name: grpc_role + - name: grpc_status + - name: outcome + - name: acquire_reason + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + prometheus: + endpoint: 0.0.0.0:8889 + # Promote resource attributes (deployment.environment, xrpl.network.type, + # service.name, service.instance.id) to Prometheus labels so the local + # scrape surface can be filtered by tier. (Grafana Cloud ingests the + # attributes natively via OTLP, so this only affects the local export.) + resource_to_telemetry_conversion: + enabled: true + # Single OTLP/HTTP exporter to Grafana Cloud. The gateway fans the three + # signals out to hosted Tempo (traces), Mimir/Prometheus (metrics), and + # Loki (logs). Retry + queue guard against transient gateway errors. + otlphttp/grafanacloud: + endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT} + auth: + authenticator: basicauth/grafanacloud + retry_on_failure: + enabled: true + sending_queue: + enabled: true + +service: + extensions: [health_check, basicauth/grafanacloud] + pipelines: + # Each pipeline keeps its local exporter(s) AND adds Grafana Cloud. + # For cloud-only, drop debug/otlp/tempo, prometheus, and otlphttp/loki + # from the respective exporter lists. + # 100% of spans feed the spanmetrics connector so span-derived RED + # metrics stay exact. No tail sampling on this branch. + traces/metrics: + receivers: [otlp] + processors: [resource/tier, resource/stripsdk, batch] + exporters: [spanmetrics] + # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo + # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans. + traces/store: + receivers: [otlp] + processors: [tail_sampling, resource/tier, resource/stripsdk, batch] + exporters: [otlp/tempo, otlphttp/grafanacloud] + # The local Prometheus scrape promotes tier/instance resource attrs to + # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does + # not, so it runs a separate pipeline that copies them onto datapoint + # labels via transform/cloudlabels. Splitting avoids the local scrape and + # the transform both writing a service_instance_id label on the same series. + metrics/local: + receivers: [otlp, spanmetrics] + processors: [resource/tier, resource/stripsdk, batch] + exporters: [prometheus] + metrics/cloud: + receivers: [otlp, spanmetrics] + processors: + [resource/tier, resource/stripsdk, transform/cloudlabels, batch] + exporters: [otlphttp/grafanacloud] + logs: + receivers: [filelog] + processors: [resource/logs, resource/tier, resource/stripsdk, batch] + exporters: [otlphttp/loki, otlphttp/grafanacloud] diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml new file mode 100644 index 00000000000..6200318b561 --- /dev/null +++ b/docker/telemetry/otel-collector-config.yaml @@ -0,0 +1,196 @@ +# OpenTelemetry Collector configuration for xrpld development. +# +# Pipelines: +# traces: OTLP receiver -> batch processor -> debug + Tempo + spanmetrics +# metrics: OTLP receiver + spanmetrics connector -> Prometheus exporter +# logs: filelog receiver -> batch processor -> otlphttp/Loki (Phase 8) +# +# xrpld sends traces via OTLP/HTTP to port 4318. The collector batches +# them, forwards to Tempo, and derives RED metrics via the spanmetrics +# connector, which Prometheus scrapes on port 8889. +# +# xrpld sends beast::insight metrics natively via OTLP/HTTP to port 4318 +# (same endpoint as traces). The OTLP receiver feeds both the traces and +# metrics pipelines. Metrics are exported to Prometheus alongside +# span-derived metrics. +# +# The filelog receiver tails xrpld's debug.log files under +# /var/log/xrpld/ (mounted from the host). A regex_parser operator +# extracts timestamp, partition, severity, and optional trace_id/span_id +# fields injected by Logs::format(). Parsed logs are exported to Grafana +# Loki for log-trace correlation. + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + # Filelog receiver tails xrpld debug.log files for log-trace + # correlation. Extracts structured fields (timestamp, partition, severity, + # trace_id, span_id, message) via regex. The trace_id and span_id are + # optional — only present when the log was emitted within an active span. + filelog: + include: [/var/log/xrpld/*/debug.log] + operators: + # Log format emitted by Logs::format() is: + # YYYY-Mmm-DD HH:MM:SS.ffffff UTC : [trace_id=... span_id=...] + # The `partition:` prefix is omitted when partition is empty, so the + # capture group is non-capturing optional. Fractional seconds up to 6 + # digits are parsed via the `%f` strptime directive. + - type: regex_parser + regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' + timestamp: + parse_from: attributes.timestamp + layout: "%Y-%b-%d %H:%M:%S.%f" + location: UTC + +processors: + batch: + timeout: 1s + send_batch_size: 100 + resource/logs: + attributes: + - key: service.name + value: xrpld + action: upsert + # Loki 3.x OTLP ingestion converts `service.name` to the label + # `service_name`. The runbook and integration-test queries use the + # canonical Loki label `job` so operators can paste `{job="xrpld"}` + # without guessing the otel-to-loki naming convention. Upsert the + # `job` resource attribute here so it round-trips through OTLP + # into Loki as the `job` label. + - key: job + value: xrpld + action: upsert + # Deployment-tier tagging. Each collector serves ONE environment and ONE + # network, so it stamps both onto every signal it forwards. This lets a + # single Grafana stack hold data from many collectors and filter by tier. + # - deployment.environment: the collector IS the environment (local, ci, + # test, prod), so it is authoritative -> upsert (overwrite). + # - xrpl.network.type: the xrpld node knows its own chain and already + # stamps this, so the collector only fills it when absent -> insert. + # This keeps a node's real network (e.g. a local node on mainnet) + # from being overwritten by a collector's default. + # Replace the placeholder values per collector; see docker/telemetry + # tier examples. + resource/tier: + attributes: + - key: deployment.environment + value: local + action: upsert + - key: xrpl.network.type + value: mainnet + action: insert + # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version). + # The OpenTelemetry SDK auto-adds these to every Resource; they carry no + # operational value and clutter the attribute set on every backend, so drop + # them here for all signals. + resource/stripsdk: + attributes: + - key: telemetry.sdk.language + action: delete + - key: telemetry.sdk.name + action: delete + - key: telemetry.sdk.version + action: delete + # Defense-in-depth: hash path-finding account attributes. The xrpld SDK + # already hashes these before export, but a node that emitted raw values + # is caught here so raw addresses never reach the backend. + attributes/hash: + actions: + - key: pathfind_source_account + action: hash + - key: pathfind_dest_account + action: hash + +connectors: + spanmetrics: + namespace: "span" + # Resource attributes that define the resource-metrics grouping key. + # All resource attributes are propagated onto the output metrics + # regardless; promotion to Prometheus labels is handled by the + # prometheus exporter's resource_to_telemetry_conversion. Listing the + # per-node id and the deployment-tier keys keeps series from distinct + # nodes/tiers grouped separately. + resource_metrics_key_attributes: + - service.instance.id + - deployment.environment + - xrpl.network.type + histogram: + explicit: + buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s] + dimensions: + - name: command + - name: rpc_status + - name: consensus_mode + - name: close_time_correct + - name: consensus_state + - name: local + - name: suppressed + - name: proposal_trusted + - name: validation_trusted + - name: tx_type + - name: ter_result + # Apply-pipeline stage (preflight|preclaim|apply) — splits the + # tx.preflight/tx.preclaim/tx.transactor span RED metrics per stage. + - name: stage + - name: txq_status + - name: load_type + - name: is_batch + # Consensus lifecycle dimensions (low cardinality, bounded value sets). + - name: mode_new + - name: consensus_stalled + - name: consensus_phase + - name: consensus_result + # gRPC surface dimensions (bounded: method names, role, status). + - name: method + - name: grpc_role + - name: grpc_status + # ledger.acquire dimensions (bounded: outcome, acquire reason). + - name: outcome + - name: acquire_reason + +exporters: + debug: + verbosity: detailed + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + # Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports + # native OTLP ingestion on its /otlp endpoint, replacing the removed + # loki exporter (dropped in otel-collector-contrib v0.147.0). + otlphttp/loki: + endpoint: http://loki:3100/otlp + prometheus: + endpoint: 0.0.0.0:8889 + # Promote resource attributes (deployment.environment, xrpl.network.type, + # service.name, service.instance.id) to Prometheus labels. Without this + # the tier attributes stamped by resource/tier stay on the resource and + # never reach the metric series, so dashboards could not filter by them. + resource_to_telemetry_conversion: + enabled: true + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [resource/tier, resource/stripsdk, attributes/hash, batch] + exporters: [debug, otlp/tempo, spanmetrics] + metrics: + receivers: [otlp, spanmetrics] + processors: [resource/tier, resource/stripsdk, batch] + exporters: [prometheus] + # Log pipeline ingests xrpld debug.log via filelog receiver, + # batches entries, and exports to Loki for log-trace correlation. + logs: + receivers: [filelog] + processors: [resource/logs, resource/tier, resource/stripsdk, batch] + exporters: [otlphttp/loki] diff --git a/docker/telemetry/prometheus.yml b/docker/telemetry/prometheus.yml new file mode 100644 index 00000000000..d99d919a556 --- /dev/null +++ b/docker/telemetry/prometheus.yml @@ -0,0 +1,9 @@ +# Prometheus configuration for scraping spanmetrics from OTel Collector. +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8889"] diff --git a/docker/telemetry/tempo.yaml b/docker/telemetry/tempo.yaml new file mode 100644 index 00000000000..24c092ca95f --- /dev/null +++ b/docker/telemetry/tempo.yaml @@ -0,0 +1,84 @@ +# Grafana Tempo configuration for xrpld telemetry stack. +# +# Runs in single-binary mode for local development. +# Receives traces via OTLP/gRPC from the OTel Collector and stores +# them locally. Queryable via Grafana Explore using the Tempo datasource. +# +# Search filters are configured on the Grafana datasource side +# (grafana/provisioning/datasources/tempo.yaml). Tempo auto-indexes +# all span attributes for search in single-binary mode. +# +# For production, replace local storage with S3/GCS backend and adjust +# retention via the compactor settings. See: +# https://grafana.com/docs/tempo/latest/configuration/ + +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + +# Raise the TraceQL metrics query range limit. The default +# query_frontend.metrics.max_duration is 3h, so a dashboard set to a longer +# window (e.g. 6h/12h) fails with "range exceeds 3h0m0s". 168h matches the +# search max_duration and gives dashboards generous headroom. +query_frontend: + metrics: + max_duration: 168h + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 1h + +# Enable metrics generator for service graph, span metrics, and the +# local-blocks processor. Produces RED metrics (rate, errors, duration) per +# service/span for the service map, and keeps recent trace blocks queryable so +# TraceQL metrics queries (quantile_over_time, count_over_time, etc. via +# /api/metrics/query_range) work. +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + # Uncomment and add a Prometheus service to docker-compose.yml + # to enable remote_write for service graph metrics: + # remote_write: + # - url: http://prometheus:9090/api/v1/write + # Separate WAL the local-blocks processor flushes traces to for metrics + # queries. Required when flush_to_storage is true. + traces_storage: + path: /var/tempo/generator/traces + processor: + local_blocks: + # xrpld consensus/transaction spans are SPAN_KIND_INTERNAL. By default + # local-blocks keeps only server spans for TraceQL metrics, so attribute + # aggregations over internal spans return nothing. Keep all spans. + filter_server_spans: false + # Flush recent blocks to traces_storage so query_range can read them. + flush_to_storage: true + +overrides: + defaults: + metrics_generator: + processors: + - service-graphs + - span-metrics + - local-blocks + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/docker/telemetry/validators-devnet.txt b/docker/telemetry/validators-devnet.txt new file mode 100644 index 00000000000..644dac96f28 --- /dev/null +++ b/docker/telemetry/validators-devnet.txt @@ -0,0 +1,11 @@ +# Devnet validator list configuration. +# +# Uses the Devnet validator list publisher operated by Ripple. +# This fetches the current set of trusted validators for the +# XRP Ledger Devnet automatically. + +[validator_list_sites] +https://vl.devnet.rippletest.net + +[validator_list_keys] +EDBB54B0D9AEE071BB37784AF5A9E7CC49AC7A0EFCE868C54532BCB966B9CFC13B diff --git a/docker/telemetry/validators-mainnet.txt b/docker/telemetry/validators-mainnet.txt new file mode 100644 index 00000000000..ca908e237ac --- /dev/null +++ b/docker/telemetry/validators-mainnet.txt @@ -0,0 +1,15 @@ +# Mainnet validator list configuration. +# +# Uses the production XRPL validator list publishers operated by +# Ripple and the XRPL Foundation. These fetch the current set of +# trusted validators for XRPL Mainnet automatically. + +[validator_list_sites] +https://vl.ripple.com +https://unl.xrplf.org + +[validator_list_keys] +# vl.ripple.com +ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734 +# unl.xrplf.org +ED42AEC58B701EEBB77356FFFEC26F83C1F0407263530F068C7C73D392C7E06FD1 diff --git a/docker/telemetry/workload/README.md b/docker/telemetry/workload/README.md new file mode 100644 index 00000000000..eb926af6b9c --- /dev/null +++ b/docker/telemetry/workload/README.md @@ -0,0 +1,371 @@ +# Telemetry Workload Tools + +Synthetic workload generation and validation tools for xrpld's OpenTelemetry telemetry stack. These tools validate that all spans, metrics, dashboards, and log-trace correlation work end-to-end under controlled load. + +## Quick Start + +```bash +# Build xrpld with telemetry enabled +conan install . --build=missing -o telemetry=True +cmake --preset default -Dtelemetry=ON +cmake --build --preset default + +# Run full validation (starts everything, runs load, validates) +docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld + +# Cleanup when done +docker/telemetry/workload/run-full-validation.sh --cleanup +``` + +## Architecture + +The validation suite runs a multi-node xrpld cluster as local processes alongside +a Docker Compose telemetry stack. The cluster exercises consensus, peer-to-peer +spans (proposals, validations), and all metric pipelines. + +``` +run-full-validation.sh (shell orchestrator) + | + |-- docker-compose.workload.yaml + | |-- otel-collector (traces via OTLP + StatsD receiver) + | |-- tempo (trace backend + TraceQL search API) + | |-- prometheus (metrics scraping) + | |-- grafana (dashboards, provisioned automatically) + | + |-- generate-validator-keys.sh + | -> validator-keys.json, validators.txt + | + |-- Nx xrpld nodes (local processes, full telemetry) + | - Each node: [telemetry] enabled=1, trace_rpc/consensus/transactions + | - [signing_support] true (server-side signing for tx_submitter) + | - Peer discovery via [ips] (not [ips_fixed]) for active peer counts + | + |-- workload_orchestrator.py (phased load execution) + | |-- rpc_load_generator.py (WebSocket RPC traffic) + | |-- tx_submitter.py (transaction diversity) + | -> workload-report.json + per-phase reports + | + |-- validate_telemetry.py (pass/fail checks) + | -> validation-report.json + | + |-- benchmark.sh (baseline vs telemetry comparison) + -> benchmark-report-*.md +``` + +## Workload Profiles + +The workload orchestrator (`workload_orchestrator.py`) reads named profiles +from `workload-profiles.json` and executes sequential load phases. Within +each phase, the RPC generator and TX submitter run concurrently. + +### Available Profiles + +| Profile | Phases | Duration | Purpose | +| ----------------- | ------ | ---------------------------- | ----------------------------------------------------------- | +| `full-validation` | 6 | ~5 min + 1 min propagation | Full 18-dashboard coverage with burst/idle/plateau patterns | +| `quick-smoke` | 1 | ~30s + 30s propagation | Fast CI smoke test | +| `stress` | 3 | ~3.5 min + 1 min propagation | Heavy sustained load for benchmarking | + +### full-validation Phases + +| Phase | RPC Rate | TX TPS | Duration | Dashboard Coverage | +| ------------ | -------- | ------ | -------- | ----------------------------------------------- | +| warmup | 5 RPS | — | 30s | Node Health, Validator Health (baseline gauges) | +| steady-state | 30 RPS | 3 TPS | 60s | All dashboards (plateau data) | +| rpc-burst | 100 RPS | — | 30s | Job Queue, RPC Performance (latency spikes) | +| tx-flood | 5 RPS | 20 TPS | 30s | Fee Market & TxQ, Transaction Overview | +| mixed-peak | 50 RPS | 10 TPS | 60s | Consensus Health, Ledger Operations | +| cooldown | 5 RPS | — | 30s | Recovery patterns, state transitions | + +### Custom Profiles + +Add profiles to `workload-profiles.json`: + +```json +{ + "profiles": { + "my-custom": { + "description": "Custom profile for specific testing", + "phases": [ + { + "name": "phase-name", + "description": "What this phase exercises", + "duration_sec": 60, + "rpc": { "rate": 50, "weights": { "server_info": 80, "fee": 20 } }, + "tx": { "tps": 5, "weights": { "Payment": 100 } } + } + ], + "propagation_wait_sec": 30 + } + } +} +``` + +Set `"rpc"` or `"tx"` to `null` to skip that generator for a phase. +Custom `"weights"` override the default command/transaction distribution. + +## Tools Reference + +### run-full-validation.sh + +Orchestrates the complete validation pipeline. Starts the telemetry stack, starts a multi-node xrpld cluster, generates load, and validates the results. + +```bash +# Full validation with defaults (uses full-validation profile) +./run-full-validation.sh --xrpld /path/to/xrpld + +# Quick smoke test +./run-full-validation.sh --xrpld /path/to/xrpld --profile quick-smoke + +# Stress test with benchmarks +./run-full-validation.sh --xrpld /path/to/xrpld --profile stress --with-benchmark + +# Skip Loki checks (if Phase 8 not deployed) +./run-full-validation.sh --xrpld /path/to/xrpld --skip-loki +``` + +### workload_orchestrator.py + +Reads a named profile from `workload-profiles.json` and executes sequential +load phases. Within each phase, `rpc_load_generator.py` and `tx_submitter.py` +run as concurrent subprocesses. Produces per-phase reports and a combined +summary. + +```bash +# Run with a specific profile +python3 workload_orchestrator.py --profile full-validation + +# Multiple endpoints +python3 workload_orchestrator.py --profile full-validation \ + --endpoints ws://localhost:6006 ws://localhost:6007 + +# Save combined report +python3 workload_orchestrator.py --profile stress --report /tmp/report.json +``` + +### rpc_load_generator.py + +Generates RPC traffic matching realistic production distribution. Uses +xrpld's **native WebSocket command format** (`{"command": ...}`) with flat +parameters — the same format as `tx_submitter.py`. + +- 40% health checks (server_info, fee) +- 30% wallet queries (account_info, account_lines, account_objects) +- 15% explorer queries (ledger, ledger_data) +- 10% transaction lookups (tx, account_tx) +- 5% DEX queries (book_offers, amm_info) + +```bash +# Basic usage +python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120 + +# Multiple endpoints (round-robin) +python3 rpc_load_generator.py \ + --endpoints ws://localhost:6006 ws://localhost:6007 \ + --rate 100 --duration 300 + +# Custom weights +python3 rpc_load_generator.py --endpoints ws://localhost:6006 \ + --weights '{"server_info": 80, "account_info": 20}' +``` + +### tx_submitter.py + +Submits diverse transaction types to exercise the full span and metric surface. +Uses xrpld's **native WebSocket command format** (`{"command": ...}`) rather +than JSON-RPC format. The response payload is inside the `"result"` key, with +`"status"` at the top level. + +Supported transaction types: + +- Payment (XRP transfers) — exercises `tx.process`, `tx.receive`, `tx.apply` +- OfferCreate / OfferCancel (DEX activity) +- TrustSet (trust line creation) +- NFTokenMint / NFTokenCreateOffer (NFT activity) +- EscrowCreate / EscrowFinish (escrow lifecycle) +- AMMCreate / AMMDeposit (AMM pool operations) + +Requires `[signing_support] true` in the node config for server-side signing. + +```bash +# Basic usage +python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120 + +# Custom mix +python3 tx_submitter.py --endpoint ws://localhost:6006 \ + --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}' +``` + +### validate_telemetry.py + +Automated validation that all expected telemetry data exists. Every metric and span is required — if it doesn't fire, the validation fails. + +- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies +- **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, StatsD gauges/counters/histograms, Phase 9 OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale gauges. +- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki) +- **Dashboard validation**: All 10 Grafana dashboards load with panels + +```bash +# Run all validations +python3 validate_telemetry.py --report /tmp/report.json + +# Skip Loki checks +python3 validate_telemetry.py --skip-loki --report /tmp/report.json +``` + +### OTel Timings Regression Gate + +`capture_timings.py` + `compare_to_baseline.py` implement a regression gate +that compares OTel-derived per-span/per-RPC/per-job timings against a +committed baseline. Unlike `benchmark.sh` (which measures the overhead of +enabling telemetry on the current binary), this gate catches **xrpld +performance regressions over time** by diffing against a stored baseline +from a prior run. + +How it runs inside the validation pipeline: + +1. `run-full-validation.sh` executes the normal workload and validation suite. +2. After validation, `capture_timings.py` queries Prometheus for every + metric in `regression-metrics.json` and writes `reports/timings.json`. +3. `compare_to_baseline.py` reads `timings.json`, + `baselines/baseline-timings.json`, and `regression-thresholds.json`, + then either: + - Prints the paste-me JSON block (when the baseline is a placeholder + or empty) and exits 0. + - Prints a delta table, writes `reports/regression-report.json`, and + exits non-zero if any metric breached both the percentage AND + absolute bound. + +Bootstrapping a baseline: + +1. Push the branch. The `Telemetry Validation` CI run prints the full + timings JSON under "Paste into `baselines/baseline-timings.json`" in + the workflow Step Summary. +2. Open a PR copying that JSON block verbatim into + `baselines/baseline-timings.json`. Reviewer approval is the audit gate. +3. Subsequent runs compare against it; the gate fails on regression. + +Per-run tuning: + +- `--skip-regression` disables the gate (local exploration only). +- `REGRESSION_WINDOW` env var overrides the default Prometheus `rate()` + window (`3m`). Keep close to the workload duration. +- Metric surface lives in `regression-metrics.json`; thresholds in + `regression-thresholds.json`; both are reviewed changes. + +See [`baselines/README.md`](./baselines/README.md) for the baseline +lifecycle and refresh process. + +### benchmark.sh + +Compares baseline (no telemetry) vs telemetry-enabled performance: + +```bash +./benchmark.sh --xrpld /path/to/xrpld --duration 300 +``` + +Thresholds (configurable via environment): + +| Metric | Threshold | Env Variable | +| ----------------- | --------- | --------------------------- | +| CPU overhead | < 3% | BENCH_CPU_OVERHEAD_PCT | +| Memory overhead | < 5MB | BENCH_MEM_OVERHEAD_MB | +| RPC p99 latency | < 2ms | BENCH_RPC_LATENCY_IMPACT_MS | +| Throughput impact | < 5% | BENCH_TPS_IMPACT_PCT | +| Consensus impact | < 1% | BENCH_CONSENSUS_IMPACT_PCT | + +## Reading Validation Reports + +The validation report (`validation-report.json`) is structured as: + +```json +{ + "summary": { + "total": 45, + "passed": 42, + "failed": 3, + "all_passed": false + }, + "checks": [ + { + "name": "span.rpc.ws_message", + "category": "span", + "passed": true, + "message": "rpc.ws_message: 15 traces found", + "details": { "trace_count": 15 } + } + ] +} +``` + +Categories: + +- **span**: Span type existence and attribute validation +- **metric**: Prometheus metric existence +- **log**: Log-trace correlation checks +- **dashboard**: Grafana dashboard accessibility + +## CI Integration + +The validation runs as a GitHub Actions workflow (`.github/workflows/telemetry-validation.yml`): + +- Triggered manually or on pushes to telemetry branches +- Builds xrpld, starts the full stack, runs load, validates +- Uploads reports as artifacts +- Posts summary to PR + +## Configuration Files + +| File | Purpose | +| --------------------------------- | ------------------------------------------------------------- | +| `workload-profiles.json` | Named load profiles with phase definitions | +| `expected_spans.json` | Span inventory (names, attributes, hierarchies, config flags) | +| `expected_metrics.json` | Metric inventory — every listed metric must be present | +| `test_accounts.json` | Test account roles (keys generated at runtime) | +| `regression-metrics.json` | Metric surface for the OTel regression gate | +| `regression-thresholds.json` | Per-metric regression bounds (pct AND abs) | +| `baselines/baseline-timings.json` | Committed baseline — populated from first CI run | +| `requirements.txt` | Python dependencies | + +### expected_metrics.json Format + +```json +{ + "category_name": { + "description": "Human-readable description.", + "metrics": ["metric_1", "metric_2"] + } +} +``` + +Every metric listed must produce > 0 Prometheus series during the validation run. If a metric doesn't fire, the workload generators need to produce enough load to trigger it. + +### expected_spans.json Format + +Each span entry defines its name, category, parent (for hierarchy validation), +required attributes, and the `config_flag` that must be enabled: + +```json +{ + "name": "rpc.command.*", + "category": "rpc", + "parent": "rpc.process", + "required_attributes": ["command", "version", "rpc_role", "rpc_status"], + "config_flag": "trace_rpc" +} +``` + +## Node Configuration Notes + +The orchestrator (`run-full-validation.sh`) generates node configs with: + +- `[telemetry] enabled=1` with all trace categories (`trace_rpc`, `trace_consensus`, `trace_transactions`) +- `[signing_support] true` — required for `tx_submitter.py` to submit signed transactions via WebSocket +- `[ips]` (not `[ips_fixed]`) — ensures peer connections are counted in `Peer_Finder_Active_Inbound/Outbound_Peers` metrics (fixed peers are excluded from these counters by design) + +## StatsD Gauge Behaviour + +Beast::insight StatsD gauges only emit when their value _changes_ from the previous sample. This can cause two problems in the validation environment: + +1. **Initial-zero gauges** — if a gauge value is 0 from startup and never changes, the gauge would never emit. To address this, `StatsDGaugeImpl` initializes `m_dirty = true`, ensuring the first flush always emits the initial value. +2. **Stale gauges** — once a gauge stabilizes (e.g., peer count stays at 1), it stops emitting new data points. Prometheus marks it stale after ~5 minutes. The validation script uses the Prometheus `/api/v1/series` endpoint instead of instant queries to catch such gauges. diff --git a/docker/telemetry/workload/baselines/README.md b/docker/telemetry/workload/baselines/README.md new file mode 100644 index 00000000000..515a3f561fe --- /dev/null +++ b/docker/telemetry/workload/baselines/README.md @@ -0,0 +1,72 @@ +# Performance Baselines + +This directory holds the committed baseline file used by the OTel-driven regression gate. + +## How the gate works + +After the validation suite runs, `capture_timings.py` queries Prometheus for the timings +declared in [`../regression-metrics.json`](../regression-metrics.json) and writes a +`timings.json`. Then `compare_to_baseline.py` reads [`baseline-timings.json`](./baseline-timings.json), +[`../regression-thresholds.json`](../regression-thresholds.json), and the captured +`timings.json`. The comparator picks one of two modes automatically: + +- **Placeholder baseline** (`"placeholder": true` or empty `metrics`): the comparator + prints the captured timings JSON in exactly the format expected for this file, then + exits 0 without gating. This is how we bootstrap the baseline. +- **Populated baseline**: the comparator diffs per-metric, enforces the thresholds + (regression = current exceeds baseline on BOTH the percentage AND absolute bound), + and exits non-zero on any regression. + +The regression gate runs against whatever workload profile `run-full-validation.sh` +was invoked with. Capture and comparison are profile-agnostic — they only read +Prometheus — so all existing profiles (`full-validation`, `quick-smoke`, `stress`) +continue to work unchanged. + +## Bootstrapping the baseline + +1. Merge a CI run with a `"placeholder": true` baseline. The telemetry-validation + workflow runs, fails no gate, and prints the captured timings block to the workflow + Step Summary under the heading `### Paste into baselines/baseline-timings.json`. +2. Open a new PR. Copy the full JSON block from the Step Summary (or download the + `timings.json` artifact) into this file, replacing the placeholder contents. The + JSON is emitted in the exact byte-for-byte format this file expects — sorted keys, + 2-space indent, trailing newline. +3. The committed baseline PR needs reviewer approval just like any other code change. + This is the primary audit point for "who moved the performance bar." + +## Refreshing the baseline + +Refresh when a legitimate performance change lands on `develop` (for example, a +deliberate rewrite that changes a span's structure). The process is identical to +bootstrapping: run CI with the current baseline, inspect the delta, and if the +new numbers should become the norm, open a PR pasting the fresh timings into +`baseline-timings.json`. The reviewer decides whether the new baseline is acceptable. + +Do **not** edit `baseline-timings.json` by hand outside of this process — every entry +should trace back to a real CI run so variance characteristics are preserved. + +## Schema + +```json +{ + "schema_version": 1, + "captured_at": "2026-04-24T17:30:00Z", + "window": "3m", + "git_sha": "", + "profile": "", + "metrics": { + "span.tx.process.p99": { "value": 12.4, "unit": "ms" }, + "rpc.server_info.p95": { "value": 850.0, "unit": "us" }, + "job.transaction.queued.p95": { "value": 1500.0, "unit": "us" } + } +} +``` + +Placeholder baselines additionally include `"placeholder": true`. The comparator +detects this field (or an empty `metrics` object) to switch into "populate" mode +instead of enforcing thresholds. Remove the `placeholder` key when pasting real +captured timings. + +Missing metrics (value `null`) in a captured run do not count as regressions — they +are reported separately in `regression-report.json` under `missing_in_current`. +This keeps the gate robust when a profile doesn't exercise every span on every run. diff --git a/docker/telemetry/workload/baselines/baseline-timings.json b/docker/telemetry/workload/baselines/baseline-timings.json new file mode 100644 index 00000000000..cafd683e3c3 --- /dev/null +++ b/docker/telemetry/workload/baselines/baseline-timings.json @@ -0,0 +1,133 @@ +{ + "captured_at": "2026-06-05T18:41:52Z", + "git_sha": "fd1c8c6060f7a15cc9e65b16f99629d9ab7ac7dc", + "metrics": { + "job.acceptLedger.queued.p95": { + "unit": "us", + "value": 96.78571428571428 + }, + "job.acceptLedger.running.p95": { + "unit": "us", + "value": 10562.499999999945 + }, + "job.transaction.queued.p95": { + "unit": "us", + "value": 478.96551724137925 + }, + "job.transaction.running.p95": { + "unit": "us", + "value": 494.1361256544502 + }, + "span.consensus.accept.p50": { + "unit": "ms", + "value": 1.059405940594059 + }, + "span.consensus.accept.p95": { + "unit": "ms", + "value": 9.749999999999996 + }, + "span.consensus.accept.p99": { + "unit": "ms", + "value": 23.704545454545432 + }, + "span.consensus.ledger_close.p50": { + "unit": "ms", + "value": 0.5284697508896797 + }, + "span.consensus.ledger_close.p95": { + "unit": "ms", + "value": 1.511111111111103 + }, + "span.consensus.ledger_close.p99": { + "unit": "ms", + "value": 7.878571428571429 + }, + "span.ledger.build.p50": { + "unit": "ms", + "value": 0.7412060301507538 + }, + "span.ledger.build.p95": { + "unit": "ms", + "value": 4.611111111111112 + }, + "span.ledger.build.p99": { + "unit": "ms", + "value": 7.541666666666674 + }, + "span.ledger.store.p50": { + "unit": "ms", + "value": 0.5 + }, + "span.ledger.store.p95": { + "unit": "ms", + "value": 0.95 + }, + "span.ledger.store.p99": { + "unit": "ms", + "value": 0.9900000000000001 + }, + "span.ledger.validate.p50": { + "unit": "ms", + "value": 0.5283687943262412 + }, + "span.ledger.validate.p95": { + "unit": "ms", + "value": 1.3666666666666627 + }, + "span.ledger.validate.p99": { + "unit": "ms", + "value": 6.699999999999978 + }, + "span.rpc.process.p50": { + "unit": "ms", + "value": null + }, + "span.rpc.process.p95": { + "unit": "ms", + "value": null + }, + "span.rpc.process.p99": { + "unit": "ms", + "value": null + }, + "span.rpc.ws_message.p50": { + "unit": "ms", + "value": 0.5026522773001647 + }, + "span.rpc.ws_message.p95": { + "unit": "ms", + "value": 0.9550393268703128 + }, + "span.rpc.ws_message.p99": { + "unit": "ms", + "value": 0.9952515090543261 + }, + "span.tx.apply.p50": { + "unit": "ms", + "value": 0.6330472103004292 + }, + "span.tx.apply.p95": { + "unit": "ms", + "value": 4.203389830508474 + }, + "span.tx.apply.p99": { + "unit": "ms", + "value": 5.083333333333319 + }, + "span.tx.process.p50": { + "unit": "ms", + "value": 0.5042801992591597 + }, + "span.tx.process.p95": { + "unit": "ms", + "value": 0.9581323781882418 + }, + "span.tx.process.p99": { + "unit": "ms", + "value": 0.998474791584883 + } + }, + "profile": "full-validation", + "schema_version": 1, + "window": "3m" +} diff --git a/docker/telemetry/workload/benchmark-results/.gitkeep b/docker/telemetry/workload/benchmark-results/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docker/telemetry/workload/benchmark.sh b/docker/telemetry/workload/benchmark.sh new file mode 100755 index 00000000000..954322f7143 --- /dev/null +++ b/docker/telemetry/workload/benchmark.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +# benchmark.sh — Performance benchmark for rippled telemetry overhead. +# +# Runs two identical workloads against a rippled cluster: +# 1. Baseline: telemetry disabled ([telemetry] enabled=0) +# 2. Telemetry: full telemetry enabled (traces + StatsD + all categories) +# +# Compares CPU, memory, RPC latency, TPS, and consensus round time. +# Outputs a Markdown table with pass/fail against configured thresholds. +# +# Usage: +# ./benchmark.sh --xrpld /path/to/xrpld --duration 300 +# +# Thresholds (configurable via environment variables): +# BENCH_CPU_OVERHEAD_PCT=3 CPU overhead < 3% +# BENCH_MEM_OVERHEAD_MB=5 Memory overhead < 5MB +# BENCH_RPC_LATENCY_IMPACT_MS=2 RPC p99 latency impact < 2ms +# BENCH_TPS_IMPACT_PCT=5 Throughput impact < 5% +# BENCH_CONSENSUS_IMPACT_PCT=1 Consensus round time impact < 1% + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[BENCH]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[BENCH]\033[0m %s\n" "$*"; } +warn() { printf "\033[1;33m[BENCH]\033[0m %s\n" "$*"; } +fail() { printf "\033[1;31m[BENCH]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[BENCH]\033[0m %s\n" "$*" >&2 + exit 1 +} + +# --------------------------------------------------------------------------- +# Defaults and thresholds +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +# Configurable thresholds via environment variables. +CPU_THRESHOLD="${BENCH_CPU_OVERHEAD_PCT:-3}" +MEM_THRESHOLD="${BENCH_MEM_OVERHEAD_MB:-5}" +RPC_THRESHOLD="${BENCH_RPC_LATENCY_IMPACT_MS:-2}" +TPS_THRESHOLD="${BENCH_TPS_IMPACT_PCT:-5}" +CONSENSUS_THRESHOLD="${BENCH_CONSENSUS_IMPACT_PCT:-1}" + +XRPLD="${BENCH_XRPLD:-$REPO_ROOT/.build/xrpld}" +DURATION=300 +NUM_NODES=3 +WORKDIR="/tmp/xrpld-benchmark" +RESULTS_DIR="$SCRIPT_DIR/benchmark-results" +RPC_PORT_BASE=5020 +PEER_PORT_BASE=51250 + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --xrpld PATH Path to xrpld binary (default: \$REPO_ROOT/.build/xrpld)" + echo " --duration SECS Benchmark duration per run (default: 300)" + echo " --nodes NUM Number of validator nodes (default: 3)" + echo " --output DIR Results output directory" + echo " -h, --help Show this help" + exit 0 +} + +while [ $# -gt 0 ]; do + case "$1" in + --xrpld) + XRPLD="$2" + shift 2 + ;; + --duration) + DURATION="$2" + shift 2 + ;; + --nodes) + NUM_NODES="$2" + shift 2 + ;; + --output) + RESULTS_DIR="$2" + shift 2 + ;; + -h | --help) usage ;; + *) die "Unknown option: $1" ;; + esac +done + +# Validate prerequisites. +[ -x "$XRPLD" ] || die "xrpld not found at $XRPLD" +command -v jq >/dev/null 2>&1 || die "jq not found" +command -v bc >/dev/null 2>&1 || die "bc not found" +command -v curl >/dev/null 2>&1 || die "curl not found" + +mkdir -p "$RESULTS_DIR" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) + +# --------------------------------------------------------------------------- +# Node cluster management +# --------------------------------------------------------------------------- +start_cluster() { + local telemetry_enabled="$1" + local label="$2" + + log "Starting $NUM_NODES-node cluster ($label, telemetry=$telemetry_enabled)..." + + rm -rf "$WORKDIR" + mkdir -p "$WORKDIR" + + # Generate keys using first node. + bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" + + # Build per-node configs. + for i in $(seq 1 "$NUM_NODES"); do + local node_dir="$WORKDIR/node$i" + mkdir -p "$node_dir/nudb" "$node_dir/db" + + local rpc_port + rpc_port=$((RPC_PORT_BASE + i - 1)) + local peer_port + peer_port=$((PEER_PORT_BASE + i - 1)) + local seed + seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") + + # Build ips_fixed list. + local ips_fixed="" + for j in $(seq 1 "$NUM_NODES"); do + if [ "$j" -ne "$i" ]; then + ips_fixed="${ips_fixed}127.0.0.1 $((PEER_PORT_BASE + j - 1)) +" + fi + done + + # Build telemetry section. + local telemetry_section="" + if [ "$telemetry_enabled" = "1" ]; then + telemetry_section=" +[telemetry] +enabled=1 +service_instance_id=bench-node-${i} +endpoint=http://localhost:4318/v1/traces +exporter=otlp_http +batch_size=512 +batch_delay_ms=2000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld" + else + telemetry_section=" +[telemetry] +enabled=0" + fi + + cat >"$node_dir/xrpld.cfg" <"$node_dir/stdout.log" 2>&1 & + echo $! >"$node_dir/xrpld.pid" + done + + # Wait for consensus. + log "Waiting for consensus..." + for attempt in $(seq 1 120); do + local ready=0 + for i in $(seq 1 "$NUM_NODES"); do + local port + port=$((RPC_PORT_BASE + i - 1)) + local state + state=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.server_state' 2>/dev/null || echo "") + if [ "$state" = "proposing" ]; then + ready=$((ready + 1)) + fi + done + if [ "$ready" -ge "$NUM_NODES" ]; then + ok "All $NUM_NODES nodes proposing (attempt $attempt)" + break + fi + if [ "$attempt" -eq 120 ]; then + warn "Consensus timeout — $ready/$NUM_NODES nodes ready" + fi + sleep 1 + done + + # Let the cluster stabilize. + sleep 5 +} + +stop_cluster() { + log "Stopping cluster..." + for i in $(seq 1 "$NUM_NODES"); do + local pidfile="$WORKDIR/node$i/xrpld.pid" + if [ -f "$pidfile" ]; then + kill "$(cat "$pidfile")" 2>/dev/null || true + fi + done + pkill -f "$WORKDIR" 2>/dev/null || true + sleep 3 +} + +# Build RPC ports CSV string. +rpc_ports_csv() { + local ports="" + for i in $(seq 1 "$NUM_NODES"); do + [ -n "$ports" ] && ports="$ports," + ports="$ports$((RPC_PORT_BASE + i - 1))" + done + echo "$ports" +} + +# --------------------------------------------------------------------------- +# Run benchmark +# --------------------------------------------------------------------------- +log "=" +log " rippled Telemetry Performance Benchmark" +log " Nodes: $NUM_NODES | Duration: ${DURATION}s | Binary: $XRPLD" +log "=" + +# --- Baseline run --- +BASELINE_FILE="$RESULTS_DIR/baseline-${TIMESTAMP}.json" +start_cluster "0" "baseline" +bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$BASELINE_FILE" +stop_cluster + +# --- Telemetry run --- +TELEMETRY_FILE="$RESULTS_DIR/telemetry-${TIMESTAMP}.json" +start_cluster "1" "telemetry" +bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$TELEMETRY_FILE" +stop_cluster + +# --------------------------------------------------------------------------- +# Compare results +# --------------------------------------------------------------------------- +log "Comparing results..." + +read_metric() { + local file="$1" + local key="$2" + jq -r ".$key // 0" "$file" +} + +BASE_CPU=$(read_metric "$BASELINE_FILE" "cpu_pct_avg") +TELE_CPU=$(read_metric "$TELEMETRY_FILE" "cpu_pct_avg") +CPU_DELTA=$(echo "scale=2; $TELE_CPU - $BASE_CPU" | bc 2>/dev/null || echo "0") + +BASE_MEM=$(read_metric "$BASELINE_FILE" "memory_rss_mb_peak") +TELE_MEM=$(read_metric "$TELEMETRY_FILE" "memory_rss_mb_peak") +MEM_DELTA=$(echo "scale=2; $TELE_MEM - $BASE_MEM" | bc 2>/dev/null || echo "0") + +BASE_RPC=$(read_metric "$BASELINE_FILE" "rpc_p99_ms") +TELE_RPC=$(read_metric "$TELEMETRY_FILE" "rpc_p99_ms") +RPC_DELTA=$(echo "scale=2; $TELE_RPC - $BASE_RPC" | bc 2>/dev/null || echo "0") + +BASE_TPS=$(read_metric "$BASELINE_FILE" "tps") +TELE_TPS=$(read_metric "$TELEMETRY_FILE" "tps") +if [[ "$(echo "$BASE_TPS > 0" | bc 2>/dev/null)" = "1" ]]; then + TPS_IMPACT=$(echo "scale=2; ($BASE_TPS - $TELE_TPS) / $BASE_TPS * 100" | bc 2>/dev/null || echo "0") +else + TPS_IMPACT="0" +fi + +BASE_CONS=$(read_metric "$BASELINE_FILE" "consensus_round_p95_ms") +TELE_CONS=$(read_metric "$TELEMETRY_FILE" "consensus_round_p95_ms") +if [[ "$(echo "$BASE_CONS > 0" | bc 2>/dev/null)" = "1" ]]; then + CONS_IMPACT=$(echo "scale=2; ($TELE_CONS - $BASE_CONS) / $BASE_CONS * 100" | bc 2>/dev/null || echo "0") +else + CONS_IMPACT="0" +fi + +# --------------------------------------------------------------------------- +# Pass/fail checks +# --------------------------------------------------------------------------- +PASS_COUNT=0 +FAIL_COUNT=0 + +check_threshold() { + local name="$1" + local actual="$2" + local threshold="$3" + local unit="$4" + + # Compare: actual <= threshold + if [[ "$(echo "$actual <= $threshold" | bc 2>/dev/null)" = "1" ]]; then + ok "$name: ${actual}${unit} <= ${threshold}${unit} PASS" + PASS_COUNT=$((PASS_COUNT + 1)) + echo "PASS" + else + fail "$name: ${actual}${unit} > ${threshold}${unit} FAIL" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "FAIL" + fi +} + +CPU_RESULT=$(check_threshold "CPU overhead" "$CPU_DELTA" "$CPU_THRESHOLD" "%") +MEM_RESULT=$(check_threshold "Memory overhead" "$MEM_DELTA" "$MEM_THRESHOLD" "MB") +RPC_RESULT=$(check_threshold "RPC p99 impact" "$RPC_DELTA" "$RPC_THRESHOLD" "ms") +TPS_RESULT=$(check_threshold "TPS impact" "$TPS_IMPACT" "$TPS_THRESHOLD" "%") +CONS_RESULT=$(check_threshold "Consensus impact" "$CONS_IMPACT" "$CONSENSUS_THRESHOLD" "%") + +# --------------------------------------------------------------------------- +# Output Markdown table +# --------------------------------------------------------------------------- +REPORT_FILE="$RESULTS_DIR/benchmark-report-${TIMESTAMP}.md" + +cat >"$REPORT_FILE" <", + "profile": "regression", + "metrics": { + "span.tx.process.p99": {"value": 12.4, "unit": "ms"}, + "rpc.server_info.p95": {"value": 850.0, "unit": "us"}, + ... + } + } + +Usage:: + + python3 capture_timings.py \\ + --prometheus http://localhost:9090 \\ + --metrics regression-metrics.json \\ + --output /tmp/timings.json \\ + --window 3m \\ + --profile regression +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import aiohttp + +from prom_queries import build_query_plan, run_query_plan + +logger = logging.getLogger("capture_timings") + +SCHEMA_VERSION = 1 + + +async def capture( + prom_url: str, + metrics_path: Path, + window: str, + profile: str, +) -> dict: + """Build and execute the query plan, return the full report dict.""" + plan = build_query_plan(metrics_path, window=window) + logger.info("Capturing %d metrics from %s (window=%s)", len(plan), prom_url, window) + + async with aiohttp.ClientSession() as session: + metrics = await run_query_plan(session, prom_url, plan) + + return { + "schema_version": SCHEMA_VERSION, + "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "window": window, + "git_sha": _detect_git_sha(), + "profile": profile, + "metrics": dict(sorted(metrics.items())), + } + + +def _detect_git_sha() -> str: + """Return the current commit SHA from env or git, else ``"unknown"``. + + Prefers ``GITHUB_SHA`` (set in Actions), falls back to ``git rev-parse``. + Silent fallback is fine here — a missing SHA only affects the captured + metadata, not the comparison logic. + """ + env_sha = os.environ.get("GITHUB_SHA") + if env_sha: + return env_sha + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--prometheus", + default="http://localhost:9090", + help="Prometheus base URL (default: http://localhost:9090)", + ) + parser.add_argument( + "--metrics", + type=Path, + default=Path(__file__).parent / "regression-metrics.json", + help="Path to regression-metrics.json", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Where to write the captured timings JSON", + ) + parser.add_argument( + "--window", + default="3m", + help="Prometheus rate() window (default: 3m)", + ) + parser.add_argument( + "--profile", + default="regression", + help="Workload profile used during capture (metadata only)", + ) + parser.add_argument( + "--min-capture-ratio", + type=float, + default=0.5, + help="Fail if fewer than this fraction of metrics are captured (default: 0.5)", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + + report = asyncio.run( + capture( + prom_url=args.prometheus, + metrics_path=args.metrics, + window=args.window, + profile=args.profile, + ) + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + json.dump(report, f, indent=2, sort_keys=True) + f.write("\n") + + captured = sum(1 for v in report["metrics"].values() if v["value"] is not None) + total = len(report["metrics"]) + logger.info("Wrote %s (%d/%d metrics captured)", args.output, captured, total) + + if total > 0 and (captured / total) < args.min_capture_ratio: + logger.error( + "Only %d/%d (%.0f%%) metrics captured — below the %.0f%% minimum. " + "Is Prometheus reachable at %s?", + captured, + total, + captured / total * 100, + args.min_capture_ratio * 100, + args.prometheus, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/telemetry/workload/collect_system_metrics.sh b/docker/telemetry/workload/collect_system_metrics.sh new file mode 100755 index 00000000000..58c1c7b8961 --- /dev/null +++ b/docker/telemetry/workload/collect_system_metrics.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# collect_system_metrics.sh — Collect CPU, memory, and RPC latency metrics +# from running xrpld nodes for benchmark comparison. +# +# Samples system metrics at regular intervals and writes a JSON summary. +# Used by benchmark.sh for baseline vs telemetry comparison. +# +# Usage: +# ./collect_system_metrics.sh +# +# Example: +# ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json +# +# Output JSON format: +# { +# "cpu_pct_avg": 12.5, +# "memory_rss_mb_peak": 450.2, +# "rpc_p99_ms": 15.3, +# "tps": 4.8, +# "consensus_round_p95_ms": 3200, +# "samples": 60 +# } + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[METRICS]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[METRICS]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[METRICS]\033[0m %s\n" "$*" >&2 + exit 1 +} + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " rpc_ports_csv Comma-separated RPC ports (e.g., 5005,5006,5007)" + echo " duration_seconds How long to collect metrics" + echo " output_file Path to write JSON results" + exit 1 +} + +if [ $# -lt 3 ]; then + usage +fi + +RPC_PORTS_CSV="$1" +DURATION="$2" +OUTPUT_FILE="$3" + +IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV" +SAMPLE_INTERVAL=5 +SAMPLES=$((DURATION / SAMPLE_INTERVAL)) + +log "Collecting metrics for ${DURATION}s (${SAMPLES} samples, ${#RPC_PORTS[@]} nodes)..." + +# --------------------------------------------------------------------------- +# Temporary files for aggregation +# --------------------------------------------------------------------------- +TMPDIR_METRICS="$(mktemp -d)" +CPU_FILE="$TMPDIR_METRICS/cpu.txt" +MEM_FILE="$TMPDIR_METRICS/mem.txt" +RPC_FILE="$TMPDIR_METRICS/rpc.txt" +LEDGER_FILE="$TMPDIR_METRICS/ledger.txt" + +touch "$CPU_FILE" "$MEM_FILE" "$RPC_FILE" "$LEDGER_FILE" + +cleanup() { + rm -rf "$TMPDIR_METRICS" +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Get initial ledger sequence for TPS calculation +# --------------------------------------------------------------------------- +INITIAL_SEQ=0 +INITIAL_TIME=$(date +%s) +for port in "${RPC_PORTS[@]}"; do + seq=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$seq" -gt "$INITIAL_SEQ" ]; then + INITIAL_SEQ=$seq + fi +done +log "Initial validated ledger seq: $INITIAL_SEQ" + +# --------------------------------------------------------------------------- +# Sampling loop +# --------------------------------------------------------------------------- +for sample in $(seq 1 "$SAMPLES"); do + # Collect CPU usage for xrpld processes. + # Uses ps to find all xrpld processes and average their CPU%. + cpu_sum=0 + cpu_count=0 + while IFS= read -r line; do + cpu_val=$(echo "$line" | awk '{print $1}') + if [ -n "$cpu_val" ] && [ "$cpu_val" != "0.0" ]; then + cpu_sum=$(echo "$cpu_sum + $cpu_val" | bc 2>/dev/null || echo "$cpu_sum") + cpu_count=$((cpu_count + 1)) + fi + done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $3}') + + if [ "$cpu_count" -gt 0 ]; then + cpu_avg=$(echo "scale=2; $cpu_sum / $cpu_count" | bc 2>/dev/null || echo "0") + echo "$cpu_avg" >>"$CPU_FILE" + fi + + # Collect memory RSS for xrpld processes. + while IFS= read -r line; do + rss_kb=$(echo "$line" | awk '{print $1}') + if [ -n "$rss_kb" ] && [ "$rss_kb" != "0" ]; then + rss_mb=$(echo "scale=2; $rss_kb / 1024" | bc 2>/dev/null || echo "0") + echo "$rss_mb" >>"$MEM_FILE" + fi + done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $6}') + + # Collect RPC latency from each node. + for port in "${RPC_PORTS[@]}"; do + start_ms=$(date +%s%N) + curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' >/dev/null 2>&1 || true + end_ms=$(date +%s%N) + latency_ms=$(((end_ms - start_ms) / 1000000)) + echo "$latency_ms" >>"$RPC_FILE" + done + + # Record current validated ledger seq. + for port in "${RPC_PORTS[@]}"; do + seq=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + echo "$seq" >>"$LEDGER_FILE" + break # Only need one node's seq per sample. + done + + # Progress indicator. + if [ $((sample % 10)) -eq 0 ]; then + log " Sample $sample/$SAMPLES..." + fi + + sleep "$SAMPLE_INTERVAL" +done + +# --------------------------------------------------------------------------- +# Compute aggregated metrics +# --------------------------------------------------------------------------- +log "Computing aggregated metrics..." + +# CPU average. +if [ -s "$CPU_FILE" ]; then + CPU_AVG=$(awk '{ sum += $1; n++ } END { if (n>0) printf "%.2f", sum/n; else print "0" }' "$CPU_FILE") +else + CPU_AVG="0" +fi + +# Memory peak RSS (MB). +if [ -s "$MEM_FILE" ]; then + MEM_PEAK=$(sort -n "$MEM_FILE" | tail -1) +else + MEM_PEAK="0" +fi + +# RPC latency p99 (ms). +if [ -s "$RPC_FILE" ]; then + RPC_COUNT=$(wc -l <"$RPC_FILE") + P99_INDEX=$(echo "scale=0; $RPC_COUNT * 99 / 100" | bc) + RPC_P99=$(sort -n "$RPC_FILE" | sed -n "${P99_INDEX}p") + [ -z "$RPC_P99" ] && RPC_P99="0" +else + RPC_P99="0" +fi + +# TPS calculation from ledger sequence advancement. +FINAL_SEQ=0 +for port in "${RPC_PORTS[@]}"; do + seq=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$seq" -gt "$FINAL_SEQ" ]; then + FINAL_SEQ=$seq + fi +done +FINAL_TIME=$(date +%s) +ELAPSED=$((FINAL_TIME - INITIAL_TIME)) +LEDGER_ADVANCE=$((FINAL_SEQ - INITIAL_SEQ)) +if [ "$ELAPSED" -gt 0 ] && [ "$LEDGER_ADVANCE" -gt 0 ]; then + # Rough TPS: assume ~avg_txs_per_ledger * ledgers / elapsed. + # Without tx count, use ledger close rate as proxy. + TPS=$(echo "scale=2; $LEDGER_ADVANCE / $ELAPSED" | bc 2>/dev/null || echo "0") +else + TPS="0" +fi + +# Consensus round time p95 (from ledger close interval). +# Approximate by looking at ledger sequence progression intervals. +if [ -s "$LEDGER_FILE" ]; then + # Calculate intervals between consecutive ledger sequences. + LEDGER_COUNT=$(wc -l <"$LEDGER_FILE") + # Rough estimate: DURATION / number_of_distinct_ledgers * 1000 ms + UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l) + if [ "$UNIQUE_LEDGERS" -gt 1 ]; then + CONSENSUS_P95=$(echo "scale=0; $DURATION * 1000 / ($UNIQUE_LEDGERS - 1)" | bc 2>/dev/null || echo "0") + else + CONSENSUS_P95="0" + fi +else + CONSENSUS_P95="0" +fi + +# --------------------------------------------------------------------------- +# Write output JSON +# --------------------------------------------------------------------------- +cat >"$OUTPUT_FILE" < dict: + with open(path) as f: + return json.load(f) + + +def is_placeholder(baseline: dict) -> bool: + """A baseline is a placeholder if explicitly marked OR metrics are empty.""" + if baseline.get("placeholder") is True: + return True + return not baseline.get("metrics") + + +def print_paste_me(timings: dict) -> None: + """Print captured timings in the exact baseline-timings.json format. + + The output between the two banner lines is the file contents to paste, + byte-for-byte — sorted keys, 2-space indent, trailing newline. + """ + banner = "=" * 72 + print(banner, file=sys.stderr) + print( + " NO BASELINE FOUND — paste the JSON below into", + file=sys.stderr, + ) + print( + " docker/telemetry/workload/baselines/baseline-timings.json", + file=sys.stderr, + ) + print(banner, file=sys.stderr) + + print(json.dumps(timings, indent=2, sort_keys=True)) + + print(banner, file=sys.stderr) + print( + " (End of paste-me JSON. Gate did NOT run — baseline is empty.)", + file=sys.stderr, + ) + print(banner, file=sys.stderr) + + +def resolve_thresholds( + key: str, + thresholds: dict, +) -> tuple[float | None, float | None]: + """Return ``(pct_threshold, abs_threshold)`` for a metric key. + + Per-metric overrides win over defaults. Returns ``(None, None)`` if no + threshold is defined for this category/quantile — such metrics are + captured but never gate the build. + """ + parts = key.split(".") + if len(parts) < 3: + return (None, None) + category_key = parts[0] + quantile_key = parts[-1] + + category_map = { + "span": "span", + "rpc": "rpc_method", + "job": "job_queue", + } + cat = category_map.get(category_key) + if cat is None: + return (None, None) + + override_key = f"{category_key}.{'.'.join(parts[1:-1])}" + overrides = thresholds.get("overrides", {}) + defaults = thresholds.get("defaults", {}).get(cat, {}) + + rule = overrides.get(override_key, {}).get(quantile_key) + if rule is None: + rule = defaults.get(quantile_key) + if rule is None: + return (None, None) + + pct = rule.get("max_pct_increase") + abs_bound = rule.get("max_abs_increase_ms") + if abs_bound is None: + abs_bound = rule.get("max_abs_increase_us") + return (pct, abs_bound) + + +def _skip_delta( + key: str, + baseline: float | None, + current: float | None, + unit: str, + thresholds: dict, + note: str, +) -> MetricDelta: + """Build a MetricDelta for cases where comparison is not possible.""" + pct_threshold, abs_threshold = resolve_thresholds(key, thresholds) + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=None, + pct_change=None, + unit=unit, + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=False, + note=note, + ) + + +def compute_delta( + key: str, + baseline_entry: dict | None, + current_entry: dict | None, + thresholds: dict, +) -> MetricDelta: + """Compute a MetricDelta for one metric key. + + A regression requires BOTH bounds to be breached simultaneously. This + tolerates small-value noise: a 100% increase on a 0.5 ms metric + (to 1.0 ms) is not a regression under a 5 ms absolute bound. + """ + baseline = baseline_entry.get("value") if baseline_entry else None + current = current_entry.get("value") if current_entry else None + unit = (baseline_entry or current_entry or {}).get("unit", "") + + if baseline is None and current is None: + return _skip_delta( + key, None, None, unit, thresholds, "no data (neither baseline nor current)" + ) + + if baseline is None: + return _skip_delta( + key, None, current, unit, thresholds, "new metric (not in baseline)" + ) + + if current is None: + return _skip_delta( + key, baseline, None, unit, thresholds, "not captured in current run" + ) + + pct_threshold, abs_threshold = resolve_thresholds(key, thresholds) + delta = current - baseline + pct_change = (delta / baseline * 100.0) if baseline > 0 else None + + if pct_threshold is None or abs_threshold is None: + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=delta, + pct_change=pct_change, + unit=unit, + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=False, + note="no threshold configured", + ) + + pct_breach = pct_change is not None and pct_change > pct_threshold + abs_breach = delta > abs_threshold + regressed = pct_breach and abs_breach + + if regressed: + note = "REGRESSION" + elif delta < 0: + note = "improved" + else: + note = "within bounds" + + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=delta, + pct_change=pct_change, + unit=unit, + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=regressed, + note=note, + ) + + +def print_summary(deltas: list[MetricDelta]) -> None: + """Print a sorted, human-readable table of per-metric results.""" + regressions = [d for d in deltas if d.regressed] + improvements = [ + d + for d in deltas + if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0) + ] + improvements.sort(key=lambda d: d.pct_change or 0) + regressions.sort(key=lambda d: -(d.pct_change or 0)) + + print("=" * 72) + print(f" Regression check: {len(regressions)} regression(s) detected") + print("=" * 72) + + if regressions: + print("\nRegressions (breached BOTH pct AND absolute bounds):") + _print_table(regressions) + + if improvements: + top = improvements[:5] + print("\nTop improvements:") + _print_table(top) + + missing = [d for d in deltas if d.note == "not captured in current run"] + if missing: + print(f"\n{len(missing)} baseline metric(s) not captured in current run:") + for d in missing: + print(f" {d.key}") + + +def _print_table(rows: list[MetricDelta]) -> None: + """Print a fixed-width table for a list of deltas.""" + header = f" {'METRIC':<45} {'BASE':>10} {'CUR':>10} {'Δ':>10} {'%':>8} UNIT" + print(header) + print(" " + "-" * (len(header) - 2)) + for d in rows: + base = f"{d.baseline:.2f}" if d.baseline is not None else "-" + cur = f"{d.current:.2f}" if d.current is not None else "-" + delta = f"{d.delta:+.2f}" if d.delta is not None else "-" + pct = f"{d.pct_change:+.1f}%" if d.pct_change is not None else "-" + print(f" {d.key:<45} {base:>10} {cur:>10} {delta:>10} {pct:>8} {d.unit}") + + +def write_report( + deltas: list[MetricDelta], + report_path: Path, + baseline: dict, + timings: dict, +) -> None: + """Write regression-report.json — machine-readable artifact for CI.""" + regressions = [d for d in deltas if d.regressed] + payload = { + "schema_version": 1, + "baseline_captured_at": baseline.get("captured_at"), + "baseline_git_sha": baseline.get("git_sha"), + "current_captured_at": timings.get("captured_at"), + "current_git_sha": timings.get("git_sha"), + "window": timings.get("window"), + "profile": timings.get("profile"), + "summary": { + "total": len(deltas), + "regressions": len(regressions), + "improvements": sum( + 1 + for d in deltas + if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0) + ), + "missing_in_current": sum( + 1 for d in deltas if d.note == "not captured in current run" + ), + }, + "metrics": [asdict(d) for d in deltas], + } + report_path.parent.mkdir(parents=True, exist_ok=True) + with open(report_path, "w") as f: + json.dump(payload, f, indent=2, sort_keys=True) + f.write("\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--timings", + type=Path, + required=True, + help="Captured timings JSON (from capture_timings.py)", + ) + parser.add_argument( + "--baseline", + type=Path, + required=True, + help="Committed baseline-timings.json", + ) + parser.add_argument( + "--thresholds", + type=Path, + default=Path(__file__).parent / "regression-thresholds.json", + help="Threshold policy JSON", + ) + parser.add_argument( + "--report", + type=Path, + default=None, + help="Where to write regression-report.json (optional)", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + + try: + timings = load_json(args.timings) + baseline = load_json(args.baseline) + thresholds = load_json(args.thresholds) + except (OSError, json.JSONDecodeError) as exc: + logger.error("failed to load inputs: %s", exc) + return 2 + + if is_placeholder(baseline): + print_paste_me(timings) + return 0 + + baseline_metrics = baseline.get("metrics", {}) + current_metrics = timings.get("metrics", {}) + + all_keys = sorted(set(baseline_metrics) | set(current_metrics)) + deltas = [ + compute_delta( + key, + baseline_metrics.get(key), + current_metrics.get(key), + thresholds, + ) + for key in all_keys + ] + + print_summary(deltas) + + if args.report: + write_report(deltas, args.report, baseline, timings) + logger.info("wrote %s", args.report) + + return 1 if any(d.regressed for d in deltas) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json new file mode 100644 index 00000000000..b305dff39b7 --- /dev/null +++ b/docker/telemetry/workload/expected_metrics.json @@ -0,0 +1,145 @@ +{ + "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.", + "spanmetrics": { + "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.", + "metrics": [ + "span_calls_total", + "span_duration_milliseconds_bucket", + "span_duration_milliseconds_count", + "span_duration_milliseconds_sum" + ], + "required_labels": [ + "span_name", + "status_code", + "service_name", + "span_kind" + ], + "dimension_labels": [ + "command", + "rpc_status", + "consensus_mode", + "local", + "proposal_trusted", + "validation_trusted", + "tx_type", + "ter_result", + "stage", + "txq_status", + "close_time_correct", + "consensus_state", + "suppressed" + ], + "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)." + }, + "statsd_gauges": { + "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).", + "metrics": [ + "ledgermaster_validated_ledger_age", + "ledgermaster_published_ledger_age", + "state_accounting_full_duration", + "peer_finder_active_inbound_peers", + "peer_finder_active_outbound_peers", + "jobq_job_count" + ] + }, + "statsd_counters": { + "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.", + "metrics": ["rpc_requests_total", "ledger_fetches_total"] + }, + "overlay_traffic": { + "description": "Overlay traffic metrics (subset — full list has 45+ categories).", + "metrics": [ + "total_bytes_in", + "total_bytes_out", + "total_messages_in", + "total_messages_out" + ] + }, + "phase9_nodestore": { + "description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.", + "metrics": ["nodestore_state"] + }, + "phase9_cache": { + "description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", + "metrics": ["cache_metrics"] + }, + "phase9_txq": { + "description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", + "metrics": ["txq_metrics"] + }, + "phase9_rpc_method": { + "description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).", + "metrics": ["rpc_method_started_total"] + }, + "phase9_objects": { + "description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).", + "metrics": ["object_count"] + }, + "phase9_load": { + "description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).", + "metrics": ["load_factor_metrics"] + }, + "parity_validation_agreement": { + "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).", + "metrics": [ + "validation_agreement{metric=\"agreement_pct_1h\"}", + "validation_agreement{metric=\"agreement_pct_24h\"}" + ] + }, + "parity_validator_health": { + "description": "External dashboard parity: validator health indicators (MetricsRegistry).", + "metrics": [ + "validator_health{metric=\"amendment_blocked\"}", + "validator_health{metric=\"unl_expiry_days\"}" + ] + }, + "parity_peer_quality": { + "description": "External dashboard parity: peer quality metrics (MetricsRegistry).", + "metrics": [ + "peer_quality{metric=\"peer_latency_p90_ms\"}", + "peer_quality{metric=\"peers_insane_count\"}" + ] + }, + "parity_ledger_economy": { + "description": "External dashboard parity: ledger economy metrics (MetricsRegistry).", + "metrics": [ + "ledger_economy{metric=\"base_fee_xrp\"}", + "ledger_economy{metric=\"transaction_rate\"}" + ] + }, + "parity_state_tracking": { + "description": "External dashboard parity: server state tracking (MetricsRegistry).", + "metrics": ["state_tracking{metric=\"state_value\"}"] + }, + "parity_counters": { + "description": "External dashboard parity: monotonic counters (MetricsRegistry).", + "metrics": [ + "ledgers_closed_total", + "validations_sent_total", + "state_changes_total" + ] + }, + "parity_storage": { + "description": "External dashboard parity: storage detail metrics (MetricsRegistry).", + "metrics": ["storage_detail{metric=\"nudb_bytes\"}"] + }, + "grafana_dashboards": { + "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", + "uids": [ + "rpc-performance", + "transaction-overview", + "consensus-health", + "ledger-operations", + "peer-network", + "peer-quality", + "fee-market", + "job-queue", + "validator-health", + "node-health", + "network-traffic", + "rpc-pathfinding", + "overlay-traffic-detail", + "ledger-data-sync" + ] + } +} diff --git a/docker/telemetry/workload/expected_spans.json b/docker/telemetry/workload/expected_spans.json new file mode 100644 index 00000000000..f663f303cf3 --- /dev/null +++ b/docker/telemetry/workload/expected_spans.json @@ -0,0 +1,419 @@ +{ + "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers. Spans marked \"optional\": true are conditional — they only fire under traffic the harness may not produce (e.g. gRPC client, missing-ledger fetch, mode transitions) and are not failed when absent.", + "spans": [ + { + "name": "rpc.ws_message", + "category": "rpc", + "parent": null, + "required_attributes": ["command"], + "config_flag": "trace_rpc", + "note": "WebSocket RPC root span. The load generator uses WS, so this is the RPC entry span (not rpc.http_request, which needs an HTTP/JSON-RPC client)." + }, + { + "name": "rpc.process", + "category": "rpc", + "parent": "rpc.ws_message", + "required_attributes": [], + "config_flag": "trace_rpc" + }, + { + "name": "rpc.command.*", + "category": "rpc", + "parent": "rpc.process", + "required_attributes": ["command", "version", "rpc_role", "rpc_status"], + "config_flag": "trace_rpc", + "note": "Wildcard — matches rpc.command.server_info, rpc.command.ledger, etc." + }, + { + "name": "rpc.http_request", + "category": "rpc", + "parent": null, + "required_attributes": ["request_payload_size"], + "config_flag": "trace_rpc", + "optional": true, + "note": "HTTP/JSON-RPC root span. The harness load generator is WebSocket-only, so this does not fire." + }, + { + "name": "tx.process", + "category": "transaction", + "parent": null, + "required_attributes": ["tx_hash", "local", "path"], + "config_flag": "trace_transactions" + }, + { + "name": "tx.receive", + "category": "transaction", + "parent": null, + "required_attributes": ["tx_hash", "peer_id", "suppressed"], + "config_flag": "trace_transactions", + "note": "Cross-node span: parent context propagated from the sender's tx.process via protobuf. Also carries tx_type and peer_version. tx_status is only set when a tx is suppressed/known-bad, so it is not a required attribute on every tx.receive." + }, + { + "name": "tx.apply", + "category": "transaction", + "parent": "ledger.build", + "required_attributes": ["tx_count", "tx_failed"], + "config_flag": "trace_transactions", + "note": "Apply-step span inside BuildLedger. Carries tx_count/tx_failed (ledger_seq lives on the parent ledger.build span)." + }, + { + "name": "tx.preflight", + "category": "transaction", + "parent": null, + "required_attributes": ["stage", "tx_type", "ter_result"], + "config_flag": "trace_transactions", + "note": "Apply-pipeline stage span (stage=preflight). Shares a deterministic trace_id (txID[0:16]) with tx.preclaim/tx.transactor." + }, + { + "name": "tx.preclaim", + "category": "transaction", + "parent": null, + "required_attributes": ["stage", "tx_type", "ter_result"], + "config_flag": "trace_transactions", + "note": "Apply-pipeline stage span (stage=preclaim)." + }, + { + "name": "tx.transactor", + "category": "transaction", + "parent": null, + "required_attributes": ["stage", "tx_type"], + "config_flag": "trace_transactions", + "note": "Apply-pipeline stage span (stage=apply). Also carries applied." + }, + { + "name": "txq.enqueue", + "category": "transaction", + "parent": "tx.process", + "required_attributes": ["tx_hash", "tx_type", "txq_status"], + "config_flag": "trace_transactions", + "optional": true, + "note": "Only fires when a tx is queued (fee below open-ledger level). Requires fee escalation — driven by the txq-burst workload phase. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required." + }, + { + "name": "txq.apply_direct", + "category": "transaction", + "parent": "txq.enqueue", + "required_attributes": [], + "config_flag": "trace_transactions", + "optional": true, + "note": "Child of txq.enqueue when the tx applies directly without queueing." + }, + { + "name": "txq.batch_clear", + "category": "transaction", + "parent": "txq.enqueue", + "required_attributes": ["num_cleared"], + "config_flag": "trace_transactions", + "optional": true + }, + { + "name": "txq.accept", + "category": "transaction", + "parent": null, + "required_attributes": ["queue_size", "ledger_changed"], + "config_flag": "trace_transactions", + "optional": true, + "note": "Ledger-close accept loop. Fires on the consensus thread; only meaningful when the queue is non-empty." + }, + { + "name": "txq.accept_tx", + "category": "transaction", + "parent": "txq.accept", + "required_attributes": [ + "tx_hash", + "ter_code", + "retries_remaining", + "txq_status" + ], + "config_flag": "trace_transactions", + "optional": true + }, + { + "name": "txq.cleanup", + "category": "transaction", + "parent": null, + "required_attributes": ["ledger_seq", "expired_count"], + "config_flag": "trace_transactions", + "optional": true + }, + { + "name": "consensus.round", + "category": "consensus", + "parent": null, + "required_attributes": [ + "consensus_ledger_id", + "ledger_seq", + "consensus_mode", + "consensus_round_id", + "consensus_phase" + ], + "config_flag": "trace_consensus", + "note": "Root consensus span created per round. Also carries trace_strategy, previous_ledger_seq, previous_proposers, previous_round_time_ms." + }, + { + "name": "consensus.phase.open", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.proposal.send", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": ["consensus_round"], + "config_flag": "trace_consensus", + "note": "Also carries is_bow_out." + }, + { + "name": "consensus.ledger_close", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": ["ledger_seq", "consensus_mode"], + "config_flag": "trace_consensus", + "note": "Also carries tx_count_open, close_time_resolution_ms." + }, + { + "name": "consensus.establish", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [ + "converge_percent", + "establish_count", + "proposers", + "disputes_count" + ], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.update_positions", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [ + "converge_percent", + "proposers", + "disputes_count" + ], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.check", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [ + "agree_count", + "disagree_count", + "threshold_percent", + "consensus_result" + ], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.accept", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": ["proposers", "round_time_ms", "quorum"], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.accept.apply", + "category": "consensus", + "parent": "consensus.accept", + "required_attributes": [ + "ledger_seq", + "close_time", + "parent_close_time", + "close_time_self", + "close_time_vote_bins", + "resolution_direction" + ], + "config_flag": "trace_consensus", + "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count." + }, + { + "name": "consensus.validation.send", + "category": "consensus", + "parent": null, + "required_attributes": [ + "ledger_seq", + "proposing", + "ledger_hash", + "full_validation" + ], + "config_flag": "trace_consensus", + "note": "follows-from consensus.accept. ledger_hash and full_validation are shared with peer.validation.receive (same keys, told apart by span name). Also carries validation_sign_time." + }, + { + "name": "consensus.proposal.receive", + "category": "consensus", + "parent": null, + "required_attributes": [], + "config_flag": "trace_consensus", + "note": "Context-propagated from the sending peer. No required local attributes." + }, + { + "name": "consensus.validation.receive", + "category": "consensus", + "parent": null, + "required_attributes": [], + "config_flag": "trace_consensus", + "note": "Context-propagated from the sending peer. No required local attributes." + }, + { + "name": "consensus.mode_change", + "category": "consensus", + "parent": null, + "required_attributes": ["mode_old", "mode_new"], + "config_flag": "trace_consensus", + "optional": true, + "note": "Only fires on an operating-mode transition; a steady cluster rarely changes mode after warmup." + }, + { + "name": "ledger.build", + "category": "ledger", + "parent": null, + "required_attributes": [ + "ledger_seq", + "close_time", + "close_time_correct", + "close_resolution_ms" + ], + "config_flag": "trace_ledger", + "note": "tx_count/tx_failed live on the child tx.apply span, not here." + }, + { + "name": "ledger.validate", + "category": "ledger", + "parent": null, + "required_attributes": ["ledger_seq", "validations"], + "config_flag": "trace_ledger" + }, + { + "name": "ledger.store", + "category": "ledger", + "parent": null, + "required_attributes": ["ledger_seq"], + "config_flag": "trace_ledger" + }, + { + "name": "ledger.acquire", + "category": "ledger", + "parent": null, + "required_attributes": [ + "ledger_seq", + "acquire_reason", + "timeouts", + "peer_count", + "outcome" + ], + "config_flag": "trace_ledger", + "optional": true, + "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history." + }, + { + "name": "peer.proposal.receive", + "category": "peer", + "parent": null, + "required_attributes": ["peer_id", "proposal_trusted"], + "config_flag": "trace_peer" + }, + { + "name": "peer.validation.receive", + "category": "peer", + "parent": null, + "required_attributes": [ + "peer_id", + "validation_trusted", + "ledger_hash", + "full_validation" + ], + "config_flag": "trace_peer", + "note": "ledger_hash and full_validation are shared with consensus.validation.send (same keys, told apart by span name)." + }, + { + "name": "pathfind.request", + "category": "pathfind", + "parent": null, + "required_attributes": [ + "pathfind_source_account", + "pathfind_dest_account" + ], + "config_flag": "trace_rpc", + "note": "Fires on ripple_path_find / path_find RPC. Driven by the ripple_path_find load in rpc_load_generator.py." + }, + { + "name": "pathfind.compute", + "category": "pathfind", + "parent": "pathfind.request", + "required_attributes": ["pathfind_fast"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Only fires when PathRequest::doUpdate runs a computation; the self-to-self XRP probe from the load generator returns early without computing paths in a fresh cluster with no liquidity." + }, + { + "name": "pathfind.discover", + "category": "pathfind", + "parent": "pathfind.compute", + "required_attributes": ["pathfind_search_level", "pathfind_num_paths"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Graph exploration; only fires under pathfind.compute, which needs real path liquidity not present in the fresh test cluster." + }, + { + "name": "pathfind.update_all", + "category": "pathfind", + "parent": null, + "required_attributes": ["pathfind_ledger_index", "pathfind_num_requests"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Async recomputation at ledger close; only fires when there are active path_find subscriptions (the one-shot ripple_path_find load does not register one)." + }, + { + "name": "grpc.*", + "category": "grpc", + "parent": null, + "required_attributes": ["method", "grpc_role", "grpc_status"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Wildcard — grpc.. The harness has no gRPC client, so these do not fire. Tracked for completeness." + } + ], + "parent_child_relationships": [ + { + "parent": "rpc.ws_message", + "child": "rpc.process", + "description": "WebSocket message contains processing span", + "skip": true, + "skip_reason": "rpc.ws_message and rpc.process run on different threads (the WS handler posts a coroutine to JobQueue for processing). Span context is not propagated across the thread boundary. Requires a C++ fix to capture and forward the span context through the coroutine lambda." + }, + { + "parent": "rpc.process", + "child": "rpc.command.*", + "description": "Processing span contains per-command span" + }, + { + "parent": "ledger.build", + "child": "tx.apply", + "description": "Ledger build contains transaction application" + }, + { + "parent": "consensus.round", + "child": "consensus.accept", + "description": "Consensus round contains the accept sub-span" + }, + { + "parent": "consensus.accept", + "child": "consensus.accept.apply", + "description": "Accept contains the ledger-apply sub-span" + }, + { + "parent": "pathfind.request", + "child": "pathfind.compute", + "description": "Pathfind request contains the compute sub-span", + "skip": true, + "skip_reason": "pathfind.compute only fires when a path computation actually runs; the self-to-self XRP probe in a fresh cluster with no liquidity returns before computing, so the child is not emitted under the harness workload." + } + ], + "total_span_types": 40, + "total_unique_attributes": 58 +} diff --git a/docker/telemetry/workload/generate-validator-keys.sh b/docker/telemetry/workload/generate-validator-keys.sh new file mode 100755 index 00000000000..7324a8bf61e --- /dev/null +++ b/docker/telemetry/workload/generate-validator-keys.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# generate-validator-keys.sh — Generate validator key pairs for the workload harness. +# +# Uses a temporary standalone xrpld instance to call `validation_create` RPC +# for each node. Outputs a JSON file mapping node index to seed + public key. +# +# Usage: +# ./generate-validator-keys.sh +# +# Output: +# /validator-keys.json — JSON array of {index, seed, public_key} +# /validators.txt — [validators] section for xrpld.cfg + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[KEYGEN]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[KEYGEN]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[KEYGEN]\033[0m %s\n" "$*" >&2 + exit 1 +} + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " xrpld_binary Path to xrpld binary (built with telemetry=ON)" + echo " num_nodes Number of validator key pairs to generate (1-20)" + echo " output_dir Directory to write validator-keys.json and validators.txt" + exit 1 +} + +if [ $# -lt 3 ]; then + usage +fi + +XRPLD="$1" +NUM_NODES="$2" +OUTPUT_DIR="$3" + +# Validate arguments +[ -x "$XRPLD" ] || die "xrpld binary not found or not executable: $XRPLD" +[[ "$NUM_NODES" =~ ^[0-9]+$ ]] || die "num_nodes must be a positive integer" +[ "$NUM_NODES" -ge 1 ] && [ "$NUM_NODES" -le 20 ] || die "num_nodes must be between 1 and 20" + +mkdir -p "$OUTPUT_DIR" + +# --------------------------------------------------------------------------- +# Start a temporary standalone xrpld for key generation +# --------------------------------------------------------------------------- +TEMP_DIR="$(mktemp -d)" +TEMP_PORT=5099 +TEMP_CFG="$TEMP_DIR/xrpld.cfg" + +log "Starting temporary xrpld for key generation (port $TEMP_PORT)..." + +cat >"$TEMP_CFG" <"$TEMP_DIR/stdout.log" 2>&1 & +TEMP_PID=$! + +# Ensure cleanup on exit +cleanup_temp() { + kill "$TEMP_PID" 2>/dev/null || true + wait "$TEMP_PID" 2>/dev/null || true + rm -rf "$TEMP_DIR" +} +trap cleanup_temp EXIT + +# Wait for RPC to become available +for attempt in $(seq 1 30); do + if curl -sf "http://localhost:$TEMP_PORT" \ + -d '{"method":"server_info"}' >/dev/null 2>&1; then + log "Temporary xrpld RPC ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 30 ]; then + die "Temporary xrpld RPC not ready after 30s" + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Generate key pairs +# --------------------------------------------------------------------------- +log "Generating $NUM_NODES validator key pairs..." + +KEYS_JSON="[" +VALIDATORS_TXT="[validators]" + +for i in $(seq 1 "$NUM_NODES"); do + result=$(curl -sf "http://localhost:$TEMP_PORT" \ + -d '{"method":"validation_create"}') + seed=$(echo "$result" | jq -r '.result.validation_seed') + pubkey=$(echo "$result" | jq -r '.result.validation_public_key') + + if [ -z "$seed" ] || [ "$seed" = "null" ]; then + die "Failed to generate key pair for node $i" + fi + + log " Node $i: ${pubkey:0:20}..." + + # Build JSON entry + entry="{\"index\": $i, \"seed\": \"$seed\", \"public_key\": \"$pubkey\"}" + if [ "$i" -gt 1 ]; then + KEYS_JSON="$KEYS_JSON," + fi + KEYS_JSON="$KEYS_JSON$entry" + + VALIDATORS_TXT="$VALIDATORS_TXT +$pubkey" +done + +KEYS_JSON="$KEYS_JSON]" + +# --------------------------------------------------------------------------- +# Write output files +# --------------------------------------------------------------------------- +echo "$KEYS_JSON" | jq '.' >"$OUTPUT_DIR/validator-keys.json" +echo "$VALIDATORS_TXT" >"$OUTPUT_DIR/validators.txt" + +ok "Generated $NUM_NODES key pairs:" +ok " Keys: $OUTPUT_DIR/validator-keys.json" +ok " Validators: $OUTPUT_DIR/validators.txt" diff --git a/docker/telemetry/workload/prom_queries.py b/docker/telemetry/workload/prom_queries.py new file mode 100644 index 00000000000..b257c61241e --- /dev/null +++ b/docker/telemetry/workload/prom_queries.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Shared Prometheus query helpers for the regression gate. + +Single source of truth for how regression metrics are computed. Both +``capture_timings.py`` and any future tooling consume this module so metric +name → PromQL expression stays consistent. + +Design: +- Every captured metric has a key in the form ``{category}.{name}.p{quantile}`` + (e.g. ``span.tx.process.p99``). Keys are flat strings so JSON diffing is + trivial. +- Quantile queries go through ``histogram_quantile`` over the standard + ``_bucket`` series. The rate window is a parameter (defaults to the + capture window, not Prometheus's default 5m) so short CI runs are usable. +- The catalogue of what to capture lives in ``regression-metrics.json`` — + this module only knows how to translate that JSON into HTTP queries. + +Usage:: + + import asyncio, aiohttp + from prom_queries import build_query_plan, run_query_plan + + plan = build_query_plan("regression-metrics.json", window="3m") + async with aiohttp.ClientSession() as s: + timings = await run_query_plan(s, "http://localhost:9090", plan) + # timings = {"span.tx.process.p99": {"value": 12.4, "unit": "ms"}, ...} +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import aiohttp + +logger = logging.getLogger("prom_queries") + + +@dataclass(frozen=True) +class QueryEntry: + """One metric to capture from Prometheus. + + Attributes: + key: Flat output key, e.g. ``span.tx.process.p99``. + promql: The PromQL expression to send to /api/v1/query. + unit: Unit of the returned value, e.g. ``ms`` or ``us``. + Baseline JSON preserves this so the comparator can + sanity-check unit drift. + """ + + key: str + promql: str + unit: str + + +def _build_simple_entries( + cfg: dict, + prefix: str, + window: str, +) -> list[QueryEntry]: + """Build QueryEntry list for a single-template category (spans, rpc).""" + tmpl = cfg.get("_query_template", "") + unit = cfg.get("_unit", "ms") + entries: list[QueryEntry] = [] + for name in cfg.get("names", []): + for q in cfg.get("_quantiles", []): + expr = ( + tmpl.replace("{quantile}", _format_quantile(q)) + .replace("{name}", name) + .replace("{window}", window) + ) + entries.append( + QueryEntry( + key=f"{prefix}.{name}.p{_quantile_label(q)}", + promql=expr, + unit=unit, + ) + ) + return entries + + +def _build_job_entries(cfg: dict, window: str) -> list[QueryEntry]: + """Build QueryEntry list for the job_queue category (multi-phase).""" + unit = cfg.get("_unit", "us") + phases = cfg.get("_phases", ["queued", "running"]) + tmpl_map = { + "queued": cfg.get("_queued_template", ""), + "running": cfg.get("_running_template", ""), + } + entries: list[QueryEntry] = [] + for name in cfg.get("names", []): + for phase in phases: + tmpl = tmpl_map.get(phase, "") + if not tmpl: + continue + for q in cfg.get("_quantiles", []): + expr = ( + tmpl.replace("{quantile}", _format_quantile(q)) + .replace("{name}", name) + .replace("{window}", window) + ) + entries.append( + QueryEntry( + key=f"job.{name}.{phase}.p{_quantile_label(q)}", + promql=expr, + unit=unit, + ) + ) + return entries + + +def build_query_plan(metrics_path: str | Path, window: str = "3m") -> list[QueryEntry]: + """Translate regression-metrics.json into a list of PromQL queries. + + Args: + metrics_path: Path to ``regression-metrics.json``. + window: Rate window passed to ``rate()``. For short CI runs + keep this close to the test duration so the bucket + counts are meaningful. Default 3m matches the + ``regression`` workload profile. + + Returns: + A list of ``QueryEntry`` values, one per (metric × quantile). + """ + with open(metrics_path) as f: + cfg = json.load(f) + + plan: list[QueryEntry] = [] + plan.extend(_build_simple_entries(cfg.get("spans", {}), "span", window)) + plan.extend(_build_simple_entries(cfg.get("rpc_methods", {}), "rpc", window)) + plan.extend(_build_job_entries(cfg.get("job_queue", {}), window)) + return plan + + +async def run_query_plan( + session: aiohttp.ClientSession, + prom_url: str, + plan: list[QueryEntry], +) -> dict[str, dict[str, Any]]: + """Execute a query plan and return a flat ``key → {value, unit}`` map. + + Queries that return no data (NaN, empty result) are still included in + the output with ``value: null`` — the comparator treats missing values + as "not yet observed" rather than as a regression. This keeps the + baseline schema stable across runs with different load levels. + + Args: + session: Shared aiohttp session. + prom_url: Base URL of Prometheus (e.g. ``http://localhost:9090``). + plan: Output of :func:`build_query_plan`. + + Returns: + Mapping from metric key to ``{"value": float|None, "unit": str}``. + """ + results: dict[str, dict[str, Any]] = {} + for entry in plan: + value = await _instant_query(session, prom_url, entry.promql) + results[entry.key] = {"value": value, "unit": entry.unit} + return results + + +async def _instant_query( + session: aiohttp.ClientSession, + prom_url: str, + promql: str, +) -> float | None: + """POST an instant query to Prometheus; return the scalar value or None. + + None is returned for NaN, empty results, or HTTP errors — every call + site treats None identically ("no data captured"). + """ + url = f"{prom_url.rstrip('/')}/api/v1/query" + try: + async with session.post( + url, data={"query": promql}, timeout=aiohttp.ClientTimeout(total=30) + ) as resp: + if resp.status != 200: + logger.warning("query HTTP %d: %s", resp.status, promql) + return None + body = await resp.json() + except (aiohttp.ClientError, TimeoutError) as exc: + logger.warning("query failed: %s — %s", promql, exc) + return None + + if body.get("status") != "success": + logger.warning("query status=%s: %s", body.get("status"), promql) + return None + + result = body.get("data", {}).get("result", []) + if not result: + return None + + raw = result[0].get("value", [None, None])[1] + if raw is None or raw in ("NaN", "+Inf", "-Inf"): + return None + try: + return float(raw) + except (TypeError, ValueError): + return None + + +def _format_quantile(q: float) -> str: + """Format a quantile for PromQL (``0.99`` → ``"0.99"``).""" + return f"{q:g}" + + +def _quantile_label(q: float) -> str: + """Format a quantile for the output key (``0.95`` → ``"95"``).""" + return str(int(round(q * 100))) diff --git a/docker/telemetry/workload/regression-metrics.json b/docker/telemetry/workload/regression-metrics.json new file mode 100644 index 00000000000..f475d86fd2f --- /dev/null +++ b/docker/telemetry/workload/regression-metrics.json @@ -0,0 +1,28 @@ +{ + "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.", + "_key_format": "{category}.{name}.p{quantile} (e.g. span.tx.process.p99, rpc.server_info.p95, job.transaction.queued.p95)", + "spans": { + "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))", + "_unit": "ms", + "_quantiles": [0.5, 0.95, 0.99], + "names": [ + "rpc.ws_message", + "rpc.process", + "tx.process", + "tx.apply", + "ledger.build", + "ledger.validate", + "ledger.store", + "consensus.ledger_close", + "consensus.accept" + ] + }, + "job_queue": { + "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))", + "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))", + "_unit": "us", + "_quantiles": [0.95], + "_phases": ["queued", "running"], + "names": ["transaction", "acceptLedger"] + } +} diff --git a/docker/telemetry/workload/regression-thresholds.json b/docker/telemetry/workload/regression-thresholds.json new file mode 100644 index 00000000000..e0eabfb95a2 --- /dev/null +++ b/docker/telemetry/workload/regression-thresholds.json @@ -0,0 +1,26 @@ +{ + "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR — this tolerates small-value noise). Defaults apply unless a per-metric override exists.", + "_bucket_note": "SpanMetrics latency histograms use explicit buckets [1,5,10,25,50,100,250,500,1000,5000]ms. A quantile sitting near a low-end boundary can jump a full bucket (e.g. 1ms->5ms) between runs with no real change, so absolute span bounds are set to ~2 low-end bucket widths (10ms) to tolerate that quantization noise while still catching genuine multi-bucket regressions. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load.", + "defaults": { + "span": { + "p50": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 }, + "p95": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 }, + "p99": { "max_pct_increase": 50.0, "max_abs_increase_ms": 15.0 } + }, + "job_queue": { + "p95": { "max_pct_increase": 50.0, "max_abs_increase_us": 20000.0 } + } + }, + "overrides": { + "span.consensus.ledger_close": { + "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 }, + "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 }, + "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 } + }, + "span.consensus.accept": { + "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 }, + "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 }, + "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 } + } + } +} diff --git a/docker/telemetry/workload/requirements.txt b/docker/telemetry/workload/requirements.txt new file mode 100644 index 00000000000..f115de082b0 --- /dev/null +++ b/docker/telemetry/workload/requirements.txt @@ -0,0 +1,6 @@ +# Python dependencies for Phase 10 workload tools. +# +# Install: pip install -r requirements.txt + +websockets>=12.0 +aiohttp>=3.9.0 diff --git a/docker/telemetry/workload/rpc_load_generator.py b/docker/telemetry/workload/rpc_load_generator.py new file mode 100644 index 00000000000..7b5e22f6318 --- /dev/null +++ b/docker/telemetry/workload/rpc_load_generator.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""RPC Load Generator for rippled telemetry validation. + +Connects to one or more rippled WebSocket endpoints and fires all traced +RPC commands at configurable rates with realistic production-like +distribution. + +Command distribution (default weights): + 40% Health checks: server_info, fee + 30% Wallet queries: account_info, account_lines, account_objects + 15% Explorer: ledger, ledger_data + 10% TX lookups: tx, account_tx + 5% DEX queries: book_offers, amm_info + 3% Pathfinding: ripple_path_find + +Usage: + python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120 + + # Multiple endpoints (round-robin): + python3 rpc_load_generator.py \\ + --endpoints ws://localhost:6006 ws://localhost:6007 \\ + --rate 100 --duration 300 + + # Custom weights: + python3 rpc_load_generator.py --endpoints ws://localhost:6006 \\ + --weights '{"server_info":60,"account_info":30,"ledger":10}' +""" + +import argparse +import asyncio +import json +import logging +import random +import sys +import time +import uuid +from dataclasses import dataclass, field +from typing import Any + +import websockets + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +# Default command distribution matching realistic production ratios. +# Keys are RPC command names; values are relative weights. +DEFAULT_WEIGHTS: dict[str, int] = { + # 40% health checks + "server_info": 25, + "fee": 15, + # 30% wallet queries + "account_info": 15, + "account_lines": 8, + "account_objects": 7, + # 15% explorer + "ledger": 10, + "ledger_data": 5, + # 10% tx lookups + "tx": 5, + "account_tx": 5, + # 5% DEX queries + "book_offers": 3, + "amm_info": 2, + # Pathfinding — exercises the pathfind.request/compute/discover spans. + # ripple_path_find is the synchronous (one-shot) variant that fits this + # fire-one-request WS client; path_find is a streaming subscription. + "ripple_path_find": 3, +} + +# Well-known genesis account for queries that require an account parameter. +GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" + +logger = logging.getLogger("rpc_load_generator") + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class LoadStats: + """Tracks request counts and latencies during a load run. + + Attributes: + total_sent: Total RPC requests dispatched. + total_success: Requests that returned a valid result. + total_errors: Requests that returned an error or timed out. + latencies: Per-command list of round-trip times in seconds. + command_counts: Per-command request count. + """ + + total_sent: int = 0 + total_success: int = 0 + total_errors: int = 0 + latencies: dict[str, list[float]] = field(default_factory=dict) + command_counts: dict[str, int] = field(default_factory=dict) + + def record(self, command: str, latency: float, success: bool) -> None: + """Record the outcome of a single RPC call.""" + self.total_sent += 1 + if success: + self.total_success += 1 + else: + self.total_errors += 1 + self.latencies.setdefault(command, []).append(latency) + self.command_counts[command] = self.command_counts.get(command, 0) + 1 + + def summary(self) -> dict[str, Any]: + """Return a summary dict suitable for JSON serialization.""" + per_command: dict[str, Any] = {} + for cmd, lats in self.latencies.items(): + sorted_lats = sorted(lats) + n = len(sorted_lats) + per_command[cmd] = { + "count": self.command_counts.get(cmd, 0), + "p50_ms": round(sorted_lats[n // 2] * 1000, 2) if n else 0, + "p95_ms": (round(sorted_lats[int(n * 0.95)] * 1000, 2) if n else 0), + "p99_ms": (round(sorted_lats[int(n * 0.99)] * 1000, 2) if n else 0), + } + return { + "total_sent": self.total_sent, + "total_success": self.total_success, + "total_errors": self.total_errors, + "error_rate_pct": ( + round(self.total_errors / self.total_sent * 100, 2) + if self.total_sent + else 0 + ), + "per_command": per_command, + } + + +# --------------------------------------------------------------------------- +# RPC command builders +# --------------------------------------------------------------------------- + + +def build_rpc_request(command: str) -> dict[str, Any]: + """Build a native WebSocket command request for the given command. + + Uses rippled's native WS format (``{"command": ...}``) with flat + parameters, NOT the JSON-RPC format (``{"method": ..., "params": [...]}``). + + Args: + command: The rippled RPC command name. + + Returns: + A dict representing the native WebSocket request body. + """ + req: dict[str, Any] = {"command": command} + + if command in ("server_info", "fee"): + pass # No params needed. + elif command == "account_info": + req["account"] = GENESIS_ACCOUNT + elif command == "account_lines": + req["account"] = GENESIS_ACCOUNT + elif command == "account_objects": + req["account"] = GENESIS_ACCOUNT + req["limit"] = 10 + elif command == "ledger": + req["ledger_index"] = "validated" + elif command == "ledger_data": + req["ledger_index"] = "validated" + req["limit"] = 5 + elif command == "tx": + # Use a dummy hash — returns "txnNotFound" error but still exercises + # the full RPC span pipeline (rpc.ws_message -> rpc.process -> rpc.command.tx). + req["transaction"] = "0" * 64 + req["binary"] = False + elif command == "account_tx": + req["account"] = GENESIS_ACCOUNT + req["ledger_index_min"] = -1 + req["ledger_index_max"] = -1 + req["limit"] = 5 + elif command == "book_offers": + req["taker_pays"] = {"currency": "XRP"} + req["taker_gets"] = { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + } + req["limit"] = 5 + elif command == "amm_info": + # AMM may not exist — the span is still created on the server side. + req["asset"] = {"currency": "XRP"} + req["asset2"] = { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + } + elif command == "ripple_path_find": + # Self-to-self XRP path search. It returns no usable paths, but the + # server still runs the full pathfinding pipeline (pathfind.request -> + # pathfind.compute -> pathfind.discover), which is what we trace. + req["source_account"] = GENESIS_ACCOUNT + req["destination_account"] = GENESIS_ACCOUNT + req["destination_amount"] = "1000000" # 1 XRP in drops + + return req + + +def choose_command(weights: dict[str, int]) -> str: + """Select a random RPC command based on configured weights. + + Args: + weights: Mapping of command name to relative weight. + + Returns: + A command name string. + """ + commands = list(weights.keys()) + w = [weights[c] for c in commands] + return random.choices(commands, weights=w, k=1)[0] + + +# --------------------------------------------------------------------------- +# WebSocket RPC client +# --------------------------------------------------------------------------- + + +async def send_rpc( + ws: websockets.WebSocketClientProtocol, + command: str, + stats: LoadStats, + inject_traceparent: bool = True, +) -> None: + """Send a single RPC request over WebSocket and record the result. + + Args: + ws: Open WebSocket connection. + command: RPC command name. + stats: LoadStats instance to record results. + inject_traceparent: If True, add a W3C traceparent header field + to the request for context propagation testing. + """ + request = build_rpc_request(command) + + # Inject W3C traceparent for context propagation testing. + # The rippled WebSocket handler extracts this from the JSON body + # when present (Phase 2 context propagation). + if inject_traceparent: + trace_id = uuid.uuid4().hex + span_id = uuid.uuid4().hex[:16] + request["traceparent"] = f"00-{trace_id}-{span_id}-01" + + t0 = time.monotonic() + try: + await ws.send(json.dumps(request)) + raw = await asyncio.wait_for(ws.recv(), timeout=10.0) + latency = time.monotonic() - t0 + response = json.loads(raw) + # Native WS responses have {"status": "success", "result": {...}} + # or {"status": "error", "error": "...", "error_message": "..."}. + success = response.get("status") == "success" + stats.record(command, latency, success) + except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as exc: + latency = time.monotonic() - t0 + stats.record(command, latency, False) + logger.debug("RPC %s failed: %s", command, exc) + + +async def run_load( + endpoints: list[str], + rate: float, + duration: float, + weights: dict[str, int], + inject_traceparent: bool, +) -> LoadStats: + """Run the RPC load generator against the given endpoints. + + Distributes requests round-robin across endpoints at the specified + rate (requests per second) for the given duration. + + Args: + endpoints: List of WebSocket URLs (ws://host:port). + rate: Target requests per second. + duration: Total run time in seconds. + weights: Command distribution weights. + inject_traceparent: Whether to inject W3C traceparent headers. + + Returns: + LoadStats with aggregated results. + """ + stats = LoadStats() + interval = 1.0 / rate if rate > 0 else 0.1 + + # Open persistent connections to all endpoints. + connections: list[websockets.WebSocketClientProtocol] = [] + for ep in endpoints: + try: + ws = await websockets.connect(ep, ping_interval=20, ping_timeout=10) + connections.append(ws) + logger.info("Connected to %s", ep) + except Exception as exc: + logger.error("Failed to connect to %s: %s", ep, exc) + + if not connections: + logger.error("No connections established. Aborting.") + return stats + + logger.info( + "Starting load: rate=%s RPS, duration=%ss, endpoints=%d", + rate, + duration, + len(connections), + ) + + start = time.monotonic() + conn_idx = 0 + + try: + while (time.monotonic() - start) < duration: + command = choose_command(weights) + ws = connections[conn_idx % len(connections)] + conn_idx += 1 + + # Fire-and-forget style with bounded concurrency via sleep. + asyncio.create_task(send_rpc(ws, command, stats, inject_traceparent)) + await asyncio.sleep(interval) + + # Periodic progress log. + elapsed = time.monotonic() - start + if stats.total_sent % 100 == 0 and stats.total_sent > 0: + actual_rps = stats.total_sent / elapsed if elapsed > 0 else 0 + logger.info( + "Progress: %d sent, %d errors, %.1f RPS (%.0fs elapsed)", + stats.total_sent, + stats.total_errors, + actual_rps, + elapsed, + ) + except asyncio.CancelledError: + logger.info("Load generation cancelled.") + finally: + # Allow in-flight requests to complete. + await asyncio.sleep(2) + for ws in connections: + await ws.close() + + elapsed = time.monotonic() - start + logger.info( + "Load complete: %d sent, %d success, %d errors in %.1fs (%.1f RPS)", + stats.total_sent, + stats.total_success, + stats.total_errors, + elapsed, + stats.total_sent / elapsed if elapsed > 0 else 0, + ) + + return stats + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="RPC Load Generator for rippled telemetry validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic usage (50 RPS for 2 minutes): + python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120 + + # Multiple endpoints with custom weights: + python3 rpc_load_generator.py \\ + --endpoints ws://localhost:6006 ws://localhost:6007 \\ + --rate 100 --duration 300 \\ + --weights '{"server_info": 80, "account_info": 20}' + """, + ) + parser.add_argument( + "--endpoints", + nargs="+", + default=["ws://localhost:6006"], + help="WebSocket endpoints (default: ws://localhost:6006)", + ) + parser.add_argument( + "--rate", + type=float, + default=50.0, + help="Target requests per second (default: 50)", + ) + parser.add_argument( + "--duration", + type=float, + default=120.0, + help="Run duration in seconds (default: 120)", + ) + parser.add_argument( + "--weights", + type=str, + default=None, + help="JSON string of command weights (overrides defaults)", + ) + parser.add_argument( + "--no-traceparent", + action="store_true", + help="Disable W3C traceparent injection", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Write JSON summary to this file path", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the RPC load generator.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + # Parse custom weights if provided. + weights = DEFAULT_WEIGHTS.copy() + if args.weights: + try: + custom = json.loads(args.weights) + weights = {k: int(v) for k, v in custom.items()} + logger.info("Using custom weights: %s", weights) + except (json.JSONDecodeError, ValueError) as exc: + logger.error("Invalid --weights JSON: %s", exc) + sys.exit(1) + + # Run the load generator. + stats = asyncio.run( + run_load( + endpoints=args.endpoints, + rate=args.rate, + duration=args.duration, + weights=weights, + inject_traceparent=not args.no_traceparent, + ) + ) + + summary = stats.summary() + print(json.dumps(summary, indent=2)) + + if args.output: + with open(args.output, "w") as f: + json.dump(summary, f, indent=2) + logger.info("Summary written to %s", args.output) + + # Exit with error if error rate exceeds 50%. + if summary["error_rate_pct"] > 50: + logger.error("High error rate: %.1f%%", summary["error_rate_pct"]) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/run-full-validation.sh b/docker/telemetry/workload/run-full-validation.sh new file mode 100755 index 00000000000..c6c292410b1 --- /dev/null +++ b/docker/telemetry/workload/run-full-validation.sh @@ -0,0 +1,500 @@ +#!/usr/bin/env bash +# run-full-validation.sh — Orchestrates the full telemetry validation pipeline. +# +# Sequence: +# 1. Start the observability stack (OTel Collector, Tempo, Prometheus, Loki, Grafana) +# 2. Start a multi-node rippled cluster with full telemetry enabled +# 3. Wait for consensus +# 4. Run workload orchestrator (RPC load, TX submission, propagation wait) +# 5. Run the telemetry validation suite +# 6. Capture OTel timings and compare against committed baseline +# 7. (Optional) Run the performance overhead benchmark +# +# Usage: +# ./run-full-validation.sh --xrpld /path/to/xrpld +# ./run-full-validation.sh --xrpld /path/to/xrpld --with-benchmark +# ./run-full-validation.sh --xrpld /path/to/xrpld --skip-regression +# ./run-full-validation.sh --cleanup +# +# Exit codes: +# 0 — All validation checks and the regression gate passed +# 1 — Validation checks failed OR the regression gate detected a regression +# 2 — Infrastructure error (cluster/stack failed to start, timing capture failed) + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[VALIDATE]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[VALIDATE]\033[0m %s\n" "$*"; } +warn() { printf "\033[1;33m[VALIDATE]\033[0m %s\n" "$*"; } +fail() { printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*" >&2 + exit 2 +} + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TELEMETRY_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$TELEMETRY_DIR/../.." && pwd)" +COMPOSE_FILE="$TELEMETRY_DIR/docker-compose.workload.yaml" +WORKDIR="/tmp/xrpld-validation" + +XRPLD="${XRPLD:-$REPO_ROOT/.build/xrpld}" +NUM_NODES=5 +RPC_PORT_BASE=5005 +WS_PORT_BASE=6006 +PEER_PORT_BASE=51235 +RPC_RATE=50 +RPC_DURATION=120 +TX_TPS=5 +TX_DURATION=120 +WITH_BENCHMARK=false +SKIP_LOKI=false +SKIP_REGRESSION=false +WORKLOAD_PROFILE="full-validation" +REPORT_DIR="$WORKDIR/reports" +# Rate window handed to Prometheus `rate()` when capturing timings. Keep +# this close to the active workload duration so histogram buckets cover +# the measurement window; longer windows dilute short-lived regressions. +REGRESSION_WINDOW="${REGRESSION_WINDOW:-3m}" +BASELINE_FILE="${BASELINE_FILE:-$SCRIPT_DIR/baselines/baseline-timings.json}" +THRESHOLDS_FILE="${THRESHOLDS_FILE:-$SCRIPT_DIR/regression-thresholds.json}" +METRICS_FILE="${METRICS_FILE:-$SCRIPT_DIR/regression-metrics.json}" + +GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" +GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --xrpld PATH Path to xrpld binary" + echo " --nodes NUM Number of validator nodes (default: 5)" + echo " --rpc-rate RPS RPC load rate (default: 50)" + echo " --rpc-duration SECS RPC load duration (default: 120)" + echo " --tx-tps TPS Transaction submit rate (default: 5)" + echo " --tx-duration SECS Transaction submit duration (default: 120)" + echo " --profile NAME Workload profile (default: full-validation)" + echo " --with-benchmark Also run performance overhead benchmark (telemetry off vs on)" + echo " --skip-loki Skip Loki log-trace correlation checks" + echo " --skip-regression Skip the OTel-baseline regression gate" + echo " --cleanup Tear down everything and exit" + echo " -h, --help Show this help" + exit 0 +} + +while [ $# -gt 0 ]; do + case "$1" in + --xrpld) + XRPLD="$2" + shift 2 + ;; + --nodes) + NUM_NODES="$2" + shift 2 + ;; + --rpc-rate) + RPC_RATE="$2" + shift 2 + ;; + --rpc-duration) + RPC_DURATION="$2" + shift 2 + ;; + --tx-tps) + TX_TPS="$2" + shift 2 + ;; + --tx-duration) + TX_DURATION="$2" + shift 2 + ;; + --profile) + WORKLOAD_PROFILE="$2" + shift 2 + ;; + --with-benchmark) + WITH_BENCHMARK=true + shift + ;; + --skip-loki) + SKIP_LOKI=true + shift + ;; + --skip-regression) + SKIP_REGRESSION=true + shift + ;; + --cleanup) # Cleanup mode + log "Cleaning up..." + pkill -f "$WORKDIR" 2>/dev/null || true + docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true + rm -rf "$WORKDIR" + ok "Cleanup complete." + exit 0 + ;; + -h | --help) usage ;; + *) die "Unknown option: $1" ;; + esac +done + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- +log "Checking prerequisites..." +[ -x "$XRPLD" ] || die "xrpld binary not found: $XRPLD" +command -v docker >/dev/null 2>&1 || die "docker not found" +docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found" +command -v python3 >/dev/null 2>&1 || die "python3 not found" +command -v curl >/dev/null 2>&1 || die "curl not found" +command -v jq >/dev/null 2>&1 || die "jq not found" +[ -f "$COMPOSE_FILE" ] || die "docker-compose.workload.yaml not found" + +# Install Python dependencies. +log "Installing Python dependencies..." +pip3 install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null || + pip install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null || + warn "Could not install Python dependencies — they may already be present" + +ok "Prerequisites verified." + +# --------------------------------------------------------------------------- +# Cleanup previous run +# --------------------------------------------------------------------------- +log "Cleaning up previous run..." +pkill -f "$WORKDIR" 2>/dev/null || true +sleep 2 +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" "$REPORT_DIR" + +# --------------------------------------------------------------------------- +# Step 1: Start observability stack +# --------------------------------------------------------------------------- +log "Step 1: Starting observability stack..." +# Point the collector's log mount at this run's workdir so the filelog +# receiver tails the per-node debug.log files generated below. +XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d + +log "Waiting for OTel Collector..." +for attempt in $(seq 1 30); do + status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + if [ "$status" != "000" ]; then + ok "OTel Collector ready (attempt $attempt)" + break + fi + [ "$attempt" -eq 30 ] && die "OTel Collector not ready after 30s" + sleep 1 +done + +log "Waiting for Tempo..." +for attempt in $(seq 1 30); do + if curl -sf "http://localhost:3200/ready" >/dev/null 2>&1; then + ok "Tempo ready (attempt $attempt)" + break + fi + [ "$attempt" -eq 30 ] && die "Tempo not ready after 30s" + sleep 1 +done + +log "Waiting for Prometheus..." +for attempt in $(seq 1 30); do + if curl -sf "http://localhost:9090/-/healthy" >/dev/null 2>&1; then + ok "Prometheus ready (attempt $attempt)" + break + fi + [ "$attempt" -eq 30 ] && die "Prometheus not ready after 30s" + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 2: Generate validator keys and start cluster +# --------------------------------------------------------------------------- +log "Step 2: Starting $NUM_NODES-node validator cluster..." + +bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" + +for i in $(seq 1 "$NUM_NODES"); do + NODE_DIR="$WORKDIR/node$i" + mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db" + + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + WS_PORT=$((WS_PORT_BASE + i - 1)) + PEER_PORT=$((PEER_PORT_BASE + i - 1)) + SEED=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") + + # Build ips_fixed. + IPS_FIXED="" + for j in $(seq 1 "$NUM_NODES"); do + if [ "$j" -ne "$i" ]; then + IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1)) +" + fi + done + + cat >"$NODE_DIR/xrpld.cfg" <"$NODE_DIR/stdout.log" 2>&1 & + echo $! >"$NODE_DIR/xrpld.pid" + log " Node $i: RPC=$RPC_PORT WS=$WS_PORT Peer=$PEER_PORT PID=$!" +done + +# --------------------------------------------------------------------------- +# Step 3: Wait for consensus +# --------------------------------------------------------------------------- +log "Step 3: Waiting for consensus..." +for attempt in $(seq 1 120); do + ready=0 + for i in $(seq 1 "$NUM_NODES"); do + port=$((RPC_PORT_BASE + i - 1)) + state=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.server_state' 2>/dev/null || echo "") + if [ "$state" = "proposing" ]; then + ready=$((ready + 1)) + fi + done + if [ "$ready" -ge "$NUM_NODES" ]; then + ok "All $NUM_NODES nodes proposing (attempt $attempt)" + break + fi + if [ "$attempt" -eq 120 ]; then + warn "Consensus timeout — $ready/$NUM_NODES nodes ready" + fi + printf "\r %d/%d nodes proposing..." "$ready" "$NUM_NODES" + sleep 1 +done +echo "" + +# Wait for first validated ledger. +log "Waiting for validated ledger..." +for attempt in $(seq 1 60); do + val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$val_seq" -gt 2 ] 2>/dev/null; then + ok "Validated ledger: seq $val_seq" + break + fi + [ "$attempt" -eq 60 ] && warn "No validated ledger after 60s" + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 4: Run workload orchestrator +# --------------------------------------------------------------------------- +log "Step 4: Running workload orchestrator (profile: $WORKLOAD_PROFILE)..." + +WS_ENDPOINTS="" +for i in $(seq 1 "$NUM_NODES"); do + WS_ENDPOINTS="$WS_ENDPOINTS ws://localhost:$((WS_PORT_BASE + i - 1))" +done + +python3 "$SCRIPT_DIR/workload_orchestrator.py" \ + --profile "$WORKLOAD_PROFILE" \ + --endpoints $WS_ENDPOINTS \ + --report "$REPORT_DIR/workload-report.json" \ + --report-dir "$REPORT_DIR" || + warn "Workload orchestrator returned non-zero exit" + +ok "Workload orchestration complete." + +# --------------------------------------------------------------------------- +# Step 5: Run telemetry validation suite +# --------------------------------------------------------------------------- +log "Step 5: Running telemetry validation suite..." + +VALIDATION_ARGS="--report $REPORT_DIR/validation-report.json" +if [ "$SKIP_LOKI" = true ]; then + VALIDATION_ARGS="$VALIDATION_ARGS --skip-loki" +fi + +VALIDATION_EXIT=0 +python3 "$SCRIPT_DIR/validate_telemetry.py" $VALIDATION_ARGS || VALIDATION_EXIT=$? + +if [ "$VALIDATION_EXIT" -eq 0 ]; then + ok "All telemetry validation checks passed!" +else + fail "Some telemetry validation checks failed (exit $VALIDATION_EXIT)" +fi + +# --------------------------------------------------------------------------- +# Step 6: Capture OTel timings and run the regression comparison +# --------------------------------------------------------------------------- +# This step ALWAYS captures timings (so CI always has an artifact from which +# to bootstrap/refresh the committed baseline). The comparator then either: +# - prints the paste-me JSON when the baseline is a placeholder, or +# - enforces thresholds and fails the run on regression. +# Use --skip-regression to opt out (e.g. for ad-hoc local exploration). +TIMINGS_FILE="$REPORT_DIR/timings.json" +REGRESSION_REPORT="$REPORT_DIR/regression-report.json" +REGRESSION_EXIT=0 + +if [ "$SKIP_REGRESSION" != true ]; then + log "Step 6: Capturing OTel timings from Prometheus..." + if python3 "$SCRIPT_DIR/capture_timings.py" \ + --prometheus "http://localhost:9090" \ + --metrics "$METRICS_FILE" \ + --output "$TIMINGS_FILE" \ + --window "$REGRESSION_WINDOW" \ + --profile "$WORKLOAD_PROFILE"; then + ok "Timings captured: $TIMINGS_FILE" + else + fail "Failed to capture timings — skipping regression comparison." + REGRESSION_EXIT=2 + SKIP_REGRESSION=true + fi +fi + +if [ "$SKIP_REGRESSION" != true ]; then + log "Comparing against baseline $BASELINE_FILE..." + python3 "$SCRIPT_DIR/compare_to_baseline.py" \ + --timings "$TIMINGS_FILE" \ + --baseline "$BASELINE_FILE" \ + --thresholds "$THRESHOLDS_FILE" \ + --report "$REGRESSION_REPORT" || REGRESSION_EXIT=$? + if [ "$REGRESSION_EXIT" -eq 0 ]; then + ok "Regression gate passed (or baseline placeholder — paste JSON printed above)." + elif [ "$REGRESSION_EXIT" -eq 1 ]; then + fail "Regression detected — see $REGRESSION_REPORT" + else + fail "Regression comparator internal error (exit $REGRESSION_EXIT)" + fi +else + warn "Regression gate skipped." +fi + +# --------------------------------------------------------------------------- +# Step 7: (Optional) Run overhead benchmark +# --------------------------------------------------------------------------- +if [ "$WITH_BENCHMARK" = true ]; then + log "Step 7: Running performance benchmark..." + bash "$SCRIPT_DIR/benchmark.sh" \ + --xrpld "$XRPLD" \ + --duration 120 \ + --nodes 3 \ + --output "$REPORT_DIR" || + warn "Benchmark returned non-zero exit" +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo "===========================================================" +echo " FULL VALIDATION RESULTS" +echo "===========================================================" +echo "" +echo " Reports directory: $REPORT_DIR" +echo "" +ls -la "$REPORT_DIR/" 2>/dev/null || true +echo "" +echo " Observability stack is running:" +echo " Tempo: http://localhost:3200" +echo " Grafana: http://localhost:3000" +echo " Prometheus: http://localhost:9090" +echo "" +echo " xrpld nodes ($NUM_NODES) are running:" +for i in $(seq 1 "$NUM_NODES"); do + rpc=$((RPC_PORT_BASE + i - 1)) + ws=$((WS_PORT_BASE + i - 1)) + pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown') + echo " Node $i: RPC=$rpc WS=$ws PID=$pid" +done +echo "" +echo " To tear down:" +echo " $0 --cleanup" +echo "" +echo "===========================================================" + +# Fail the run if EITHER validation or the regression gate failed. The +# `[ "$VAR" -gt N ]` comparison works here because exit codes are numeric. +FINAL_EXIT=0 +if [ "$VALIDATION_EXIT" -ne 0 ]; then + FINAL_EXIT="$VALIDATION_EXIT" +fi +if [ "$REGRESSION_EXIT" -ne 0 ] && [ "$FINAL_EXIT" -eq 0 ]; then + FINAL_EXIT="$REGRESSION_EXIT" +fi +exit "$FINAL_EXIT" diff --git a/docker/telemetry/workload/test_accounts.json b/docker/telemetry/workload/test_accounts.json new file mode 100644 index 00000000000..cb85670f52e --- /dev/null +++ b/docker/telemetry/workload/test_accounts.json @@ -0,0 +1,42 @@ +{ + "genesis": { + "account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + "seed": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", + "description": "Genesis account with all XRP. Used to fund test accounts." + }, + "test_accounts": [ + { + "name": "alice", + "description": "Primary sender for Payment and OfferCreate transactions." + }, + { + "name": "bob", + "description": "Primary receiver for Payment transactions." + }, + { + "name": "carol", + "description": "TrustSet and issued currency counterparty." + }, + { + "name": "dave", + "description": "NFToken operations (mint, offer, accept)." + }, + { + "name": "eve", + "description": "Escrow operations (create, finish)." + }, + { + "name": "frank", + "description": "AMM pool operations (create, deposit, withdraw)." + }, + { + "name": "grace", + "description": "Additional sender for parallel transaction submission." + }, + { + "name": "heidi", + "description": "Additional receiver for payment diversity." + } + ], + "note": "Test account keypairs are generated dynamically at runtime via wallet_propose RPC. This file defines the logical roles. Actual keys are stored in the workdir during execution." +} diff --git a/docker/telemetry/workload/tx_submitter.py b/docker/telemetry/workload/tx_submitter.py new file mode 100644 index 00000000000..4ff015f40f1 --- /dev/null +++ b/docker/telemetry/workload/tx_submitter.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +"""Transaction Submitter for rippled telemetry validation. + +Generates diverse transaction types against a rippled cluster to exercise +the full span and metric surface: tx.process, tx.apply, ledger.build, +consensus.*, and all associated attributes. + +Pre-funds test accounts from the genesis account, then submits a +configurable mix of transaction types at a target TPS. + +Supported transaction types: + - Payment (XRP and issued currencies) + - OfferCreate / OfferCancel (DEX activity) + - TrustSet (trust line creation) + - NFTokenMint / NFTokenCreateOffer / NFTokenAcceptOffer + - EscrowCreate / EscrowFinish + - AMMCreate / AMMDeposit / AMMWithdraw (if amendment enabled) + +Usage: + python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120 + + # Custom transaction mix: + python3 tx_submitter.py --endpoint ws://localhost:6006 \\ + --weights '{"Payment":50,"OfferCreate":20,"TrustSet":10,"NFTokenMint":10,"EscrowCreate":10}' +""" + +import argparse +import asyncio +import json +import logging +import random +import sys +import time +from dataclasses import dataclass, field +from typing import Any + +import websockets + +logger = logging.getLogger("tx_submitter") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" +GENESIS_SEED = "snoPBrXtMeMyMHUVTgbuqAfg1SUTb" + +# Amount to fund each test account (100,000 XRP in drops). +FUND_AMOUNT = "100000000000" + +# Default transaction mix weights (relative). +DEFAULT_TX_WEIGHTS: dict[str, int] = { + "Payment": 40, + "OfferCreate": 15, + "OfferCancel": 5, + "TrustSet": 10, + "NFTokenMint": 10, + "NFTokenCreateOffer": 5, + "EscrowCreate": 5, + "EscrowFinish": 5, + "AMMCreate": 3, + "AMMDeposit": 2, +} + +# Number of test accounts to create. +NUM_TEST_ACCOUNTS = 8 + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class Account: + """Represents a funded XRPL test account. + + Attributes: + name: Human-readable name (e.g., "alice"). + account: Classic address (rXXX...). + seed: Secret seed for signing. + sequence: Next available sequence number. + """ + + name: str + account: str + seed: str + sequence: int = 0 + + +@dataclass +class TxStats: + """Tracks transaction submission results. + + Attributes: + total_submitted: Total transactions sent to the network. + total_success: Transactions that returned tesSUCCESS or terQUEUED. + total_errors: Transactions that returned an error engine_result. + by_type: Per-transaction-type count of submissions. + errors_by_type: Per-transaction-type count of errors. + """ + + total_submitted: int = 0 + total_success: int = 0 + total_errors: int = 0 + by_type: dict[str, int] = field(default_factory=dict) + errors_by_type: dict[str, int] = field(default_factory=dict) + + def record(self, tx_type: str, success: bool) -> None: + """Record the result of a transaction submission.""" + self.total_submitted += 1 + self.by_type[tx_type] = self.by_type.get(tx_type, 0) + 1 + if success: + self.total_success += 1 + else: + self.total_errors += 1 + self.errors_by_type[tx_type] = self.errors_by_type.get(tx_type, 0) + 1 + + def summary(self) -> dict[str, Any]: + """Return a summary dict suitable for JSON serialization.""" + return { + "total_submitted": self.total_submitted, + "total_success": self.total_success, + "total_errors": self.total_errors, + "success_rate_pct": ( + round(self.total_success / self.total_submitted * 100, 2) + if self.total_submitted + else 0 + ), + "by_type": self.by_type, + "errors_by_type": self.errors_by_type, + } + + +# --------------------------------------------------------------------------- +# WebSocket RPC helpers +# --------------------------------------------------------------------------- + + +async def ws_request( + ws: websockets.WebSocketClientProtocol, + command: str, + params: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Send a native WebSocket command and return the result payload. + + Uses rippled's native WebSocket format (``command`` key with flat + parameters). The response has ``status`` at the top level and the + actual data payload inside ``result``. This helper unwraps the + ``result`` dict so callers can read fields directly. + + Args: + ws: Open WebSocket connection. + command: RPC command name (e.g., ``account_info``, ``submit``). + params: Optional flat parameter dict merged into the request. + + Returns: + The inner ``result`` dict from the response. + + Raises: + RuntimeError: If the request fails or times out. + """ + request: dict[str, Any] = {"command": command} + if params: + request.update(params) + await ws.send(json.dumps(request)) + raw = await asyncio.wait_for(ws.recv(), timeout=30.0) + resp = json.loads(raw) + + # WS command format: {"status": "success", "result": {...}, "type": "response"} + # On error: {"status": "error", "error": "...", "error_message": "..."} + if resp.get("status") == "error": + logger.warning( + "%s error: %s — %s", + command, + resp.get("error", "unknown"), + resp.get("error_message", ""), + ) + return resp.get("result", resp) + + +async def create_account(ws: websockets.WebSocketClientProtocol, name: str) -> Account: + """Create a new account via wallet_propose RPC. + + Args: + ws: Open WebSocket connection. + name: Human-readable name for the account. + + Returns: + An Account instance with the generated keypair. + """ + result = await ws_request(ws, "wallet_propose") + if "account_id" not in result: + raise RuntimeError( + f"wallet_propose failed: {json.dumps(result, indent=None)[:300]}" + ) + return Account( + name=name, + account=result["account_id"], + seed=result["master_seed"], + ) + + +async def fund_account( + ws: websockets.WebSocketClientProtocol, + dest: Account, + genesis_seq: int, +) -> tuple[bool, int]: + """Fund a test account from genesis. + + Args: + ws: Open WebSocket connection. + dest: Destination account to fund. + genesis_seq: Current genesis account sequence number. + + Returns: + Tuple of (success: bool, next_sequence: int). + """ + resp = await ws_request( + ws, + "submit", + { + "secret": GENESIS_SEED, + "tx_json": { + "TransactionType": "Payment", + "Account": GENESIS_ACCOUNT, + "Destination": dest.account, + "Amount": FUND_AMOUNT, + "Sequence": genesis_seq, + }, + }, + ) + engine_result = resp.get("engine_result", "unknown") + success = engine_result in ("tesSUCCESS", "terQUEUED") + if not success: + # Log the full response to help diagnose submit failures in CI. + logger.warning( + "Fund %s failed: engine_result=%s, full response: %s", + dest.name, + engine_result, + json.dumps(resp, indent=None)[:500], + ) + return success, genesis_seq + 1 + + +async def get_account_sequence( + ws: websockets.WebSocketClientProtocol, account: str +) -> int: + """Get the current sequence number for an account. + + Args: + ws: Open WebSocket connection. + account: Classic address. + + Returns: + Current sequence number. + """ + resp = await ws_request(ws, "account_info", {"account": account}) + if "account_data" not in resp: + # Log full response to diagnose WS API format issues. + logger.warning( + "account_info for %s: no account_data, full response: %s", + account[:12], + json.dumps(resp, indent=None)[:500], + ) + return 0 + return resp["account_data"].get("Sequence", 0) + + +# --------------------------------------------------------------------------- +# Transaction builders +# --------------------------------------------------------------------------- + + +def build_payment(sender: Account, receiver: Account) -> dict[str, Any]: + """Build an XRP Payment transaction. + + Args: + sender: Source account. + receiver: Destination account. + + Returns: + Transaction JSON and signing secret. + """ + amount = str(random.randint(1000, 1000000)) # 0.001 - 1 XRP + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "Payment", + "Account": sender.account, + "Destination": receiver.account, + "Amount": amount, + "Sequence": sender.sequence, + }, + } + + +def build_offer_create(sender: Account) -> dict[str, Any]: + """Build an OfferCreate transaction (XRP/USD pair). + + Args: + sender: Account placing the offer. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "OfferCreate", + "Account": sender.account, + "TakerPays": str(random.randint(100000, 10000000)), + "TakerGets": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + "value": str(round(random.uniform(0.1, 100.0), 2)), + }, + "Sequence": sender.sequence, + }, + } + + +def build_offer_cancel(sender: Account) -> dict[str, Any]: + """Build an OfferCancel transaction. + + Uses a non-existent offer sequence — will fail gracefully but still + exercises the tx.process span pipeline. + + Args: + sender: Account cancelling the offer. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "OfferCancel", + "Account": sender.account, + "OfferSequence": max(1, sender.sequence - 1), + "Sequence": sender.sequence, + }, + } + + +def build_trust_set(sender: Account) -> dict[str, Any]: + """Build a TrustSet transaction for a USD trust line. + + Args: + sender: Account setting the trust line. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "TrustSet", + "Account": sender.account, + "LimitAmount": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + "value": "1000000", + }, + "Sequence": sender.sequence, + }, + } + + +def build_nftoken_mint(sender: Account) -> dict[str, Any]: + """Build an NFTokenMint transaction. + + Args: + sender: Account minting the NFT. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "NFTokenMint", + "Account": sender.account, + "NFTokenTaxon": random.randint(0, 100), + "Flags": 8, # tfTransferable + "Sequence": sender.sequence, + }, + } + + +def build_nftoken_create_offer(sender: Account) -> dict[str, Any]: + """Build an NFTokenCreateOffer transaction. + + Uses a dummy NFTokenID — will fail but exercises the span pipeline. + + Args: + sender: Account creating the NFT offer. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "NFTokenCreateOffer", + "Account": sender.account, + "NFTokenID": "0" * 64, + "Amount": str(random.randint(100000, 1000000)), + "Flags": 1, # tfSellNFToken + "Sequence": sender.sequence, + }, + } + + +def build_escrow_create(sender: Account, receiver: Account) -> dict[str, Any]: + """Build an EscrowCreate transaction. + + Creates a time-based escrow that finishes 10 seconds from now. + + Args: + sender: Account creating the escrow. + receiver: Destination account for escrow funds. + + Returns: + Transaction JSON and signing secret. + """ + # Ripple epoch offset: 946684800 seconds from Unix epoch + ripple_time = int(time.time()) - 946684800 + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "EscrowCreate", + "Account": sender.account, + "Destination": receiver.account, + "Amount": str(random.randint(100000, 1000000)), + "FinishAfter": ripple_time + 10, + "Sequence": sender.sequence, + }, + } + + +def build_escrow_finish(sender: Account, owner: Account) -> dict[str, Any]: + """Build an EscrowFinish transaction. + + Uses a dummy offer sequence — will likely fail but exercises spans. + + Args: + sender: Account finishing the escrow. + owner: Account that created the escrow. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "EscrowFinish", + "Account": sender.account, + "Owner": owner.account, + "OfferSequence": max(1, owner.sequence - 2), + "Sequence": sender.sequence, + }, + } + + +def build_amm_create(sender: Account) -> dict[str, Any]: + """Build an AMMCreate transaction (XRP/USD pool). + + Requires the AMM amendment to be enabled on the network. + + Args: + sender: Account creating the AMM pool. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "AMMCreate", + "Account": sender.account, + "Amount": str(random.randint(10000000, 100000000)), + "Amount2": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + "value": str(round(random.uniform(10.0, 1000.0), 2)), + }, + "TradingFee": 500, # 0.5% + "Sequence": sender.sequence, + }, + } + + +def build_amm_deposit(sender: Account) -> dict[str, Any]: + """Build an AMMDeposit transaction. + + Args: + sender: Account depositing into the AMM pool. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "AMMDeposit", + "Account": sender.account, + "Asset": {"currency": "XRP"}, + "Asset2": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + }, + "Amount": str(random.randint(1000000, 10000000)), + "Flags": 0x00080000, # tfSingleAsset + "Sequence": sender.sequence, + }, + } + + +# Transaction type -> builder function mapping. +# Each builder takes (accounts: list[Account]) and returns submit params. +TX_BUILDERS: dict[str, Any] = { + "Payment": lambda accts: build_payment(accts[0], accts[1]), + "OfferCreate": lambda accts: build_offer_create(accts[0]), + "OfferCancel": lambda accts: build_offer_cancel(accts[0]), + "TrustSet": lambda accts: build_trust_set(accts[2]), + "NFTokenMint": lambda accts: build_nftoken_mint(accts[3]), + "NFTokenCreateOffer": lambda accts: build_nftoken_create_offer(accts[3]), + "EscrowCreate": lambda accts: build_escrow_create(accts[4], accts[1]), + "EscrowFinish": lambda accts: build_escrow_finish(accts[4], accts[4]), + "AMMCreate": lambda accts: build_amm_create(accts[5]), + "AMMDeposit": lambda accts: build_amm_deposit(accts[5]), +} + + +# --------------------------------------------------------------------------- +# Main submission loop +# --------------------------------------------------------------------------- + + +async def setup_accounts( + ws: websockets.WebSocketClientProtocol, +) -> list[Account]: + """Create and fund test accounts from genesis. + + Generates NUM_TEST_ACCOUNTS accounts via wallet_propose, then funds + each with FUND_AMOUNT XRP from genesis. + + Args: + ws: Open WebSocket connection to a rippled node. + + Returns: + List of funded Account instances. + """ + account_names = ["alice", "bob", "carol", "dave", "eve", "frank", "grace", "heidi"] + + logger.info("Creating %d test accounts...", NUM_TEST_ACCOUNTS) + accounts: list[Account] = [] + for name in account_names[:NUM_TEST_ACCOUNTS]: + acct = await create_account(ws, name) + accounts.append(acct) + logger.info(" Created %s: %s", name, acct.account) + + # Get genesis sequence. + genesis_seq = await get_account_sequence(ws, GENESIS_ACCOUNT) + logger.info("Genesis sequence: %d", genesis_seq) + + # Fund all accounts. + logger.info("Funding test accounts...") + for acct in accounts: + success, genesis_seq = await fund_account(ws, acct, genesis_seq) + if success: + logger.info(" Funded %s", acct.name) + else: + logger.warning(" Failed to fund %s", acct.name) + + # Wait for funding transactions to be validated. + logger.info("Waiting 10s for funding transactions to validate...") + await asyncio.sleep(10) + + # Refresh sequence numbers for all accounts. + for acct in accounts: + try: + acct.sequence = await get_account_sequence(ws, acct.account) + logger.info(" %s sequence: %d", acct.name, acct.sequence) + except Exception as exc: + logger.warning(" Failed to get sequence for %s: %s", acct.name, exc) + + return accounts + + +async def submit_transaction( + ws: websockets.WebSocketClientProtocol, + tx_type: str, + accounts: list[Account], + stats: TxStats, +) -> None: + """Submit a single transaction of the given type. + + Selects the appropriate builder, constructs the transaction, submits + it via the submit RPC, and records the result. + + Args: + ws: Open WebSocket connection. + tx_type: Transaction type name (e.g., "Payment"). + accounts: List of funded test accounts. + stats: TxStats instance to record results. + """ + builder = TX_BUILDERS.get(tx_type) + if not builder: + logger.warning("Unknown transaction type: %s", tx_type) + return + + try: + params = builder(accounts) + # Identify which account is the sender to bump its sequence. + sender_addr = params["tx_json"]["Account"] + sender = next((a for a in accounts if a.account == sender_addr), None) + + resp = await ws_request(ws, "submit", params) + engine_result = resp.get("engine_result", "unknown") + success = engine_result in ( + "tesSUCCESS", + "terQUEUED", + "tecUNFUNDED_OFFER", + "tecNO_DST_INSUF_XRP", + ) + stats.record(tx_type, success) + + if sender: + sender.sequence += 1 + + if not success: + logger.debug( + "%s result: %s (%s)", + tx_type, + engine_result, + resp.get("engine_result_message", ""), + ) + except Exception as exc: + stats.record(tx_type, False) + logger.debug("%s error: %s", tx_type, exc) + + +async def _refresh_sequences( + ws: websockets.WebSocketClientProtocol, + accounts: list[Account], +) -> None: + """Re-sync account sequences from the validated ledger. + + In a consensus network, other nodes' transactions advance sequences + beyond the submitter's local tracking. Refreshing every ~10 s keeps + the local counter close to the ledger and prevents tefPAST_SEQ storms. + """ + for acct in accounts: + try: + seq = await get_account_sequence(ws, acct.account) + if seq > acct.sequence: + acct.sequence = seq + except Exception: + pass + + +async def run_submitter( + endpoint: str, + tps: float, + duration: float, + weights: dict[str, int], +) -> TxStats: + """Run the transaction submitter against a single endpoint. + + Args: + endpoint: WebSocket URL (ws://host:port). + tps: Target transactions per second. + duration: Total run time in seconds. + weights: Transaction type distribution weights. + + Returns: + TxStats with aggregated results. + """ + stats = TxStats() + interval = 1.0 / tps if tps > 0 else 0.5 + + ws = await websockets.connect(endpoint, ping_interval=20, ping_timeout=10) + logger.info("Connected to %s", endpoint) + + try: + # Setup test accounts. + accounts = await setup_accounts(ws) + if len(accounts) < 6: + logger.error("Need at least 6 funded accounts, got %d", len(accounts)) + return stats + + # Build weighted command list. + tx_types = list(weights.keys()) + tx_weights = [weights[t] for t in tx_types] + + logger.info( + "Starting TX submission: tps=%s, duration=%ss, types=%d", + tps, + duration, + len(tx_types), + ) + + start = time.monotonic() + last_seq_refresh = start + seq_refresh_interval = 10.0 + while (time.monotonic() - start) < duration: + # Periodically re-sync account sequences from the ledger so + # locally-tracked sequences don't drift behind consensus. + if (time.monotonic() - last_seq_refresh) >= seq_refresh_interval: + await _refresh_sequences(ws, accounts) + last_seq_refresh = time.monotonic() + + tx_type = random.choices(tx_types, weights=tx_weights, k=1)[0] + await submit_transaction(ws, tx_type, accounts, stats) + await asyncio.sleep(interval) + + # Progress logging every 50 transactions. + if stats.total_submitted % 50 == 0 and stats.total_submitted > 0: + elapsed = time.monotonic() - start + actual_tps = stats.total_submitted / elapsed if elapsed > 0 else 0 + logger.info( + "Progress: %d submitted, %d success, %d errors, " + "%.1f TPS (%.0fs elapsed)", + stats.total_submitted, + stats.total_success, + stats.total_errors, + actual_tps, + elapsed, + ) + + finally: + await ws.close() + + elapsed = time.monotonic() - start + logger.info( + "Submission complete: %d submitted, %d success, %d errors " + "in %.1fs (%.1f TPS)", + stats.total_submitted, + stats.total_success, + stats.total_errors, + elapsed, + stats.total_submitted / elapsed if elapsed > 0 else 0, + ) + + return stats + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Transaction Submitter for rippled telemetry validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic usage (5 TPS for 2 minutes): + python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120 + + # Custom transaction mix: + python3 tx_submitter.py --endpoint ws://localhost:6006 \\ + --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}' + """, + ) + parser.add_argument( + "--endpoint", + type=str, + default="ws://localhost:6006", + help="WebSocket endpoint (default: ws://localhost:6006)", + ) + parser.add_argument( + "--tps", + type=float, + default=5.0, + help="Target transactions per second (default: 5)", + ) + parser.add_argument( + "--duration", + type=float, + default=120.0, + help="Run duration in seconds (default: 120)", + ) + parser.add_argument( + "--weights", + type=str, + default=None, + help="JSON string of transaction type weights (overrides defaults)", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Write JSON summary to this file path", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the transaction submitter.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + # Parse custom weights if provided. + weights = DEFAULT_TX_WEIGHTS.copy() + if args.weights: + try: + custom = json.loads(args.weights) + weights = {k: int(v) for k, v in custom.items()} + logger.info("Using custom weights: %s", weights) + except (json.JSONDecodeError, ValueError) as exc: + logger.error("Invalid --weights JSON: %s", exc) + sys.exit(1) + + # Run the submitter. + stats = asyncio.run( + run_submitter( + endpoint=args.endpoint, + tps=args.tps, + duration=args.duration, + weights=weights, + ) + ) + + summary = stats.summary() + print(json.dumps(summary, indent=2)) + + if args.output: + with open(args.output, "w") as f: + json.dump(summary, f, indent=2) + logger.info("Summary written to %s", args.output) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py new file mode 100644 index 00000000000..40f7e507a82 --- /dev/null +++ b/docker/telemetry/workload/validate_telemetry.py @@ -0,0 +1,1321 @@ +#!/usr/bin/env python3 +"""Telemetry Validation Suite for xrpld. + +Validates that the full telemetry stack is emitting expected data after +a workload run. Queries Tempo (spans), Prometheus (metrics), Loki (logs), +and Grafana (dashboards) APIs to produce a pass/fail report. + +Validation categories: + 1. Span validation — All 16+ span types present with required attributes + 2. Metric validation — SpanMetrics, StatsD, and Phase 9 metrics are non-zero + 3. Log-trace correlation — Loki logs contain trace_id/span_id fields + 4. Dashboard validation — All 14 Grafana dashboards render data + 5. External parity — Span attrs, metric existence, and value sanity for + external dashboard parity (validator-health, + peer-quality, node-health) + +Usage: + python3 validate_telemetry.py --report /tmp/validation-report.json + + # Custom API endpoints: + python3 validate_telemetry.py \\ + --tempo http://localhost:3200 \\ + --prometheus http://localhost:9090 \\ + --loki http://localhost:3100 \\ + --grafana http://localhost:3000 +""" + +import argparse +import asyncio +import json +import logging +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import aiohttp + +logger = logging.getLogger("validate_telemetry") + +# --------------------------------------------------------------------------- +# Configuration defaults +# --------------------------------------------------------------------------- + +DEFAULT_TEMPO = "http://localhost:3200" +DEFAULT_PROMETHEUS = "http://localhost:9090" +DEFAULT_LOKI = "http://localhost:3100" +DEFAULT_GRAFANA = "http://localhost:3000" + +SCRIPT_DIR = Path(__file__).parent +EXPECTED_SPANS_FILE = SCRIPT_DIR / "expected_spans.json" +EXPECTED_METRICS_FILE = SCRIPT_DIR / "expected_metrics.json" + +# Some beast::insight gauges/counters (ledger-age, peer-finder, overlay +# traffic) only populate after the node validates ledgers and sustains peer +# traffic, then travel a 1s periodic OTLP export + a 15s Prometheus scrape +# before they are queryable. On a slow CI runner the fixed post-workload wait +# can end before that pipeline settles, so a single query races and reports 0 +# series. Poll each missing metric for up to this long (covering two scrape +# cycles) before failing, so the check is robust to runner speed. +METRIC_POLL_TIMEOUT_SEC = 45.0 +METRIC_POLL_INTERVAL_SEC = 5.0 + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class CheckResult: + """Result of a single validation check. + + Attributes: + name: Check identifier (e.g., "span.rpc.ws_message"). + category: Validation category (span, metric, log, dashboard). + passed: Whether the check passed. + message: Human-readable description of the result. + details: Optional additional data (counts, values, etc.). + """ + + name: str + category: str + passed: bool + message: str + details: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + """Serialize to a JSON-compatible dict.""" + return { + "name": self.name, + "category": self.category, + "passed": self.passed, + "message": self.message, + "details": self.details, + } + + +@dataclass +class ValidationReport: + """Aggregated validation report. + + Attributes: + checks: List of all individual check results. + start_time: ISO timestamp when validation started. + end_time: ISO timestamp when validation completed. + """ + + checks: list[CheckResult] = field(default_factory=list) + start_time: str = "" + end_time: str = "" + + @property + def total_checks(self) -> int: + """Total number of checks executed.""" + return len(self.checks) + + @property + def passed(self) -> int: + """Number of checks that passed.""" + return sum(1 for c in self.checks if c.passed) + + @property + def failed(self) -> int: + """Number of checks that failed.""" + return sum(1 for c in self.checks if not c.passed) + + @property + def all_passed(self) -> bool: + """Whether all checks passed.""" + return self.failed == 0 + + def add(self, check: CheckResult) -> None: + """Add a check result to the report.""" + self.checks.append(check) + status = "PASS" if check.passed else "FAIL" + logger.info("[%s] %s: %s", status, check.name, check.message) + + def to_dict(self) -> dict[str, Any]: + """Serialize to a JSON-compatible dict.""" + return { + "summary": { + "total": self.total_checks, + "passed": self.passed, + "failed": self.failed, + "all_passed": self.all_passed, + }, + "start_time": self.start_time, + "end_time": self.end_time, + "checks": [c.to_dict() for c in self.checks], + } + + +# --------------------------------------------------------------------------- +# Tempo API helpers +# --------------------------------------------------------------------------- + + +async def _tempo_search( + session: aiohttp.ClientSession, + tempo_url: str, + query: str, + limit: int = 20, +) -> list[dict[str, Any]]: + """Search traces in Tempo using TraceQL. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API (e.g., http://localhost:3200). + query: TraceQL query string. + limit: Maximum number of traces to return. + + Returns: + List of trace summary dicts from Tempo search results. + """ + params = {"q": query, "limit": str(limit)} + async with session.get(f"{tempo_url}/api/search", params=params) as resp: + data = await resp.json() + return data.get("traces", []) + + +async def _tempo_get_trace( + session: aiohttp.ClientSession, + tempo_url: str, + trace_id: str, +) -> list[dict[str, Any]]: + """Fetch a full trace from Tempo by trace ID. + + Returns the list of spans extracted from the OTLP-format response. + + Args: + session: aiohttp client session. + tempo_url: Tempo API base URL. + trace_id: Hex trace ID string. + + Returns: + Flat list of span dicts with 'name' and 'attributes' keys. + """ + async with session.get(f"{tempo_url}/api/traces/{trace_id}") as resp: + data = await resp.json() + spans: list[dict[str, Any]] = [] + for batch in data.get("batches", []): + for scope_spans in batch.get("scopeSpans", []): + spans.extend(scope_spans.get("spans", [])) + return spans + + +def _otlp_span_attr_keys(span: dict[str, Any]) -> set[str]: + """Extract all attribute key names from an OTLP span. + + Args: + span: OTLP span dict with an 'attributes' list. + + Returns: + Set of attribute key strings. + """ + return {a["key"] for a in span.get("attributes", []) if "key" in a} + + +# --------------------------------------------------------------------------- +# Span Validation (Tempo API) +# --------------------------------------------------------------------------- + + +async def validate_spans( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate that all expected spans appear in Tempo. + + Queries the Tempo TraceQL API for each expected span name and checks + that traces exist. Also validates required attributes on spans and + parent-child relationships. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API (e.g., http://localhost:3200). + report: ValidationReport to accumulate results. + """ + logger.info("--- Span Validation (Tempo) ---") + + # Load expected spans. + with open(EXPECTED_SPANS_FILE) as f: + expected = json.load(f) + + # Check service registration. + try: + async with session.get( + f"{tempo_url}/api/v2/search/tag/resource.service.name/values" + ) as resp: + data = await resp.json() + tag_values = data.get("tagValues", []) + services = [tv.get("value", "") for tv in tag_values] + has_xrpld = "xrpld" in services + report.add( + CheckResult( + name="span.service_registration", + category="span", + passed=has_xrpld, + message=( + f"Service 'xrpld' registered (found: {services})" + if has_xrpld + else f"Service 'xrpld' NOT found (found: {services})" + ), + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="span.service_registration", + category="span", + passed=False, + message=f"Tempo API unreachable: {exc}", + ) + ) + return + + # Diagnostic: list all available operations (span names) for the xrpld + # service. This output appears in CI logs and helps debug missing-span + # failures without needing to reproduce the full stack locally. + try: + async with session.get( + f"{tempo_url}/api/v2/search/tag/span.name/values" + ) as resp: + ops_data = await resp.json() + tag_values = ops_data.get("tagValues", []) + operations = [tv.get("value", "") for tv in tag_values] + logger.info( + "Tempo operations (%d total): %s", + len(operations), + operations, + ) + except Exception as exc: + logger.warning("Failed to fetch Tempo operations: %s", exc) + + # Concrete probe names for wildcard span entries. Exact-match TraceQL can't + # match a literal "*", so a representative operation name is substituted. + # Wildcards without a known concrete example (e.g. grpc. when no + # gRPC client runs) are skipped when marked optional. + wildcard_probes = {"rpc.command.*": "rpc.command.server_info"} + + # Check each expected span. + for span_def in expected["spans"]: + span_name = span_def["name"] + is_optional = span_def.get("optional", False) + check_name = f"span.{span_name}" + + if "*" in span_name: + operation = wildcard_probes.get(span_name) + if operation is None: + # No concrete probe. Optional wildcards (e.g. grpc.*) are skipped; + # a required one would be a config error worth surfacing. + if is_optional: + logger.info( + "[SKIP] %s: optional wildcard span with no concrete " + "probe (not exercised by the workload)", + check_name, + ) + continue + report.add( + CheckResult( + name=check_name, + category="span", + passed=False, + message=f"{span_name}: required wildcard has no probe name", + ) + ) + continue + else: + operation = span_name + + try: + query = '{resource.service.name="xrpld" && name="' + operation + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=5) + count = len(traces) + # Optional spans only fire under specific traffic (mode changes, + # missing-ledger fetch, fee escalation). Absence is not a failure — + # mirror the parent-child "skip" handling so CI stays green. + if count == 0 and is_optional: + logger.info( + "[SKIP] %s: optional span not emitted under this workload", + check_name, + ) + report.add( + CheckResult( + name=check_name, + category="span", + passed=True, + message=f"{span_name}: optional, not emitted (skipped)", + details={"trace_count": 0, "optional": True}, + ) + ) + continue + report.add( + CheckResult( + name=check_name, + category="span", + passed=count > 0, + message=( + f"{span_name}: {count} traces found" + if count > 0 + else f"{span_name}: 0 traces (expected > 0)" + ), + details={"trace_count": count}, + ) + ) + + # Validate required attributes on first trace. + if count > 0 and span_def.get("required_attributes"): + trace_id = traces[0].get("traceID", "") + if trace_id: + spans = await _tempo_get_trace(session, tempo_url, trace_id) + await _validate_span_attributes_otlp(spans, span_def, report) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="span", + passed=False, + message=f"{span_name}: query failed ({exc})", + ) + ) + + # Validate parent-child relationships. + for rel in expected.get("parent_child_relationships", []): + # Skip relationships marked with "skip: true" (e.g., cross-thread + # parent-child that requires a C++ fix to propagate span context). + if rel.get("skip", False): + reason = rel.get("skip_reason", "marked skip in expected_spans.json") + logger.info( + "[SKIP] span.hierarchy.%s->%s: %s", + rel["parent"], + rel["child"], + reason, + ) + continue + await _validate_parent_child(session, tempo_url, rel, report) + + +async def _validate_span_attributes_otlp( + spans: list[dict[str, Any]], + span_def: dict[str, Any], + report: ValidationReport, +) -> None: + """Check that OTLP spans contain expected attributes. + + Args: + spans: List of OTLP span dicts from Tempo. + span_def: Span definition from expected_spans.json. + report: ValidationReport to accumulate results. + """ + required_attrs = span_def.get("required_attributes", []) + if not required_attrs: + return + + span_name = span_def["name"] + # Collect all attribute keys from all spans. + found_attrs: set[str] = set() + for span in spans: + found_attrs.update(_otlp_span_attr_keys(span)) + + missing = [a for a in required_attrs if a not in found_attrs] + report.add( + CheckResult( + name=f"span.attrs.{span_name}", + category="span", + passed=len(missing) == 0, + message=( + f"{span_name}: all {len(required_attrs)} attributes present" + if not missing + else f"{span_name}: missing attributes: {missing}" + ), + details={ + "required": required_attrs, + "found": list(found_attrs), + "missing": missing, + }, + ) + ) + + +async def _validate_parent_child( + session: aiohttp.ClientSession, + tempo_url: str, + relationship: dict[str, Any], + report: ValidationReport, +) -> None: + """Validate a parent-child span relationship in Tempo traces. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API. + relationship: Dict with 'parent' and 'child' span names. + report: ValidationReport to accumulate results. + """ + parent_name = relationship["parent"] + child_name = relationship["child"] + + try: + # Query traces for the parent span. + query = '{resource.service.name="xrpld" && name="' + parent_name + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=3) + + if not traces: + report.add( + CheckResult( + name=f"span.hierarchy.{parent_name}->{child_name}", + category="span", + passed=False, + message=f"No {parent_name} traces to check hierarchy", + ) + ) + return + + # Check if child spans exist within parent traces. + # Use the concrete child name for wildcard patterns. + concrete_child = child_name.replace("*", "server_info") + found_child = False + for trace_summary in traces: + trace_id = trace_summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + for span in spans: + op = span.get("name", "") + if concrete_child in op or ("*" not in child_name and op == child_name): + found_child = True + break + if found_child: + break + + report.add( + CheckResult( + name=f"span.hierarchy.{parent_name}->{child_name}", + category="span", + passed=found_child, + message=( + f"Found {child_name} as child of {parent_name}" + if found_child + else f"{child_name} not found in {parent_name} traces" + ), + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=f"span.hierarchy.{parent_name}->{child_name}", + category="span", + passed=False, + message=f"Hierarchy check failed: {exc}", + ) + ) + + +# --------------------------------------------------------------------------- +# Metric Validation (Prometheus API) +# --------------------------------------------------------------------------- + + +async def validate_metrics( + session: aiohttp.ClientSession, + prometheus_url: str, + report: ValidationReport, +) -> None: + """Validate that expected metrics appear in Prometheus with non-zero values. + + Args: + session: aiohttp client session. + prometheus_url: Base URL for Prometheus API (e.g., http://localhost:9090). + report: ValidationReport to accumulate results. + """ + logger.info("--- Metric Validation (Prometheus) ---") + + # Diagnostic: list all metric names in Prometheus. Helps debug name + # mismatches between expected_metrics.json and actual emissions. + try: + async with session.get( + f"{prometheus_url}/api/v1/label/__name__/values" + ) as resp: + label_data = await resp.json() + all_metrics = label_data.get("data", []) + # Log relevant metrics for debugging. + relevant = [ + m + for m in all_metrics + if m.startswith( + ( + "span_", + "rpc_method", + "cache_", + "txq_", + "object_count", + "load_factor", + "nodestore", + "ledgermaster", + "peer_finder", + "jobq_", + "total_bytes", + "total_messages", + "validation_agreement", + "validator_health", + "peer_quality", + "ledger_economy", + "state_tracking", + "storage_detail", + ) + ) + ] + logger.info( + "Prometheus metrics (relevant, %d of %d total): %s", + len(relevant), + len(all_metrics), + relevant, + ) + except Exception as exc: + logger.warning("Failed to fetch Prometheus metric names: %s", exc) + + with open(EXPECTED_METRICS_FILE) as f: + expected = json.load(f) + + # Check each metric category. + for category_key, category_data in expected.items(): + if category_key in ("description", "grafana_dashboards"): + continue + + metrics = category_data.get("metrics", []) + for metric_name in metrics: + await _check_prometheus_metric( + session, prometheus_url, metric_name, category_key, report + ) + + +async def _check_prometheus_metric( + session: aiohttp.ClientSession, + prometheus_url: str, + metric_name: str, + category: str, + report: ValidationReport, +) -> None: + """Query Prometheus for a specific metric and check it exists. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus base URL. + metric_name: Prometheus metric name. + category: Metric category for the report. + report: ValidationReport to accumulate results. + """ + try: + # Use the /api/v1/series endpoint instead of an instant query. + # Beast::insight StatsD gauges only mark dirty on value *changes*, + # so a gauge that stabilizes (e.g. peer count stays at 1) may go + # stale in Prometheus and disappear from instant queries. The + # series endpoint returns any metric that existed in the window, + # regardless of staleness. + # + # Poll rather than query once: late-populating gauges/counters may + # not have completed the export+scrape pipeline when this runs, so a + # single query races. Re-query until the metric appears or the poll + # window elapses; a metric that never appears still fails after the + # timeout. + params: dict[str, str] = {"match[]": metric_name} + series_count = 0 + deadline = time.monotonic() + METRIC_POLL_TIMEOUT_SEC + while True: + async with session.get( + f"{prometheus_url}/api/v1/series", params=params + ) as resp: + data = await resp.json() + series_count = len(data.get("data", [])) + if series_count > 0 or time.monotonic() >= deadline: + break + await asyncio.sleep(METRIC_POLL_INTERVAL_SEC) + report.add( + CheckResult( + name=f"metric.{category}.{metric_name}", + category="metric", + passed=series_count > 0, + message=( + f"{metric_name}: {series_count} series" + if series_count > 0 + else f"{metric_name}: 0 series (expected > 0)" + ), + details={"series_count": series_count}, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=f"metric.{category}.{metric_name}", + category="metric", + passed=False, + message=f"{metric_name}: query failed ({exc})", + ) + ) + + +# --------------------------------------------------------------------------- +# Log-Trace Correlation Validation (Loki API) +# --------------------------------------------------------------------------- + + +async def validate_log_trace_correlation( + session: aiohttp.ClientSession, + loki_url: str, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate that Loki logs contain trace_id/span_id for correlation. + + Checks: + 1. Logs with trace_id= field exist in Loki. + 2. A random trace_id from Tempo can be found in Loki logs. + + Args: + session: aiohttp client session. + loki_url: Base URL for Loki API (e.g., http://localhost:3100). + tempo_url: Base URL for Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- Log-Trace Correlation Validation (Loki) ---") + + # Check 1: Any logs with trace_id exist. + try: + params = { + "query": '{job="xrpld"} |= "trace_id="', + "limit": 5, + "direction": "backward", + } + async with session.get( + f"{loki_url}/loki/api/v1/query_range", params=params + ) as resp: + data = await resp.json() + streams = data.get("data", {}).get("result", []) + total_entries = sum(len(s.get("values", [])) for s in streams) + report.add( + CheckResult( + name="log.trace_id_present", + category="log", + passed=total_entries > 0, + message=( + f"Found {total_entries} log entries with trace_id" + if total_entries > 0 + else "No log entries with trace_id found" + ), + details={"log_count": total_entries}, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="log.trace_id_present", + category="log", + passed=False, + message=f"Loki query failed: {exc}", + ) + ) + + # Check 2: Cross-reference a trace_id from Tempo to Loki. + try: + # Get a recent trace from Tempo. + traces = await _tempo_search( + session, + tempo_url, + '{resource.service.name="xrpld"}', + limit=1, + ) + + if traces: + trace_id = traces[0].get("traceID", "") + if trace_id: + # Search Loki for this trace_id. + loki_params = { + "query": f'{{job="xrpld"}} |= "{trace_id}"', + "limit": 5, + "direction": "backward", + } + async with session.get( + f"{loki_url}/loki/api/v1/query_range", + params=loki_params, + ) as loki_resp: + loki_data = await loki_resp.json() + loki_streams = loki_data.get("data", {}).get("result", []) + loki_count = sum(len(s.get("values", [])) for s in loki_streams) + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=loki_count > 0, + message=( + f"trace_id {trace_id[:16]}... found in " + f"{loki_count} Loki entries" + if loki_count > 0 + else f"trace_id {trace_id[:16]}... not found " "in Loki" + ), + details={ + "trace_id": trace_id, + "loki_count": loki_count, + }, + ) + ) + else: + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=False, + message="No traces in Tempo to cross-reference", + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=False, + message=f"Cross-reference check failed: {exc}", + ) + ) + + +# --------------------------------------------------------------------------- +# Dashboard Validation (Grafana API) +# --------------------------------------------------------------------------- + + +async def validate_dashboards( + session: aiohttp.ClientSession, + grafana_url: str, + report: ValidationReport, +) -> None: + """Validate that all Grafana dashboards are accessible and return data. + + For each expected dashboard UID, queries the Grafana API to verify + the dashboard exists and is loadable. + + Args: + session: aiohttp client session. + grafana_url: Base URL for Grafana API (e.g., http://localhost:3000). + report: ValidationReport to accumulate results. + """ + logger.info("--- Dashboard Validation (Grafana) ---") + + with open(EXPECTED_METRICS_FILE) as f: + expected = json.load(f) + + dashboard_uids = expected.get("grafana_dashboards", {}).get("uids", []) + + for uid in dashboard_uids: + try: + async with session.get(f"{grafana_url}/api/dashboards/uid/{uid}") as resp: + if resp.status == 200: + data = await resp.json() + dashboard = data.get("dashboard", {}) + panel_count = len(dashboard.get("panels", [])) + report.add( + CheckResult( + name=f"dashboard.{uid}", + category="dashboard", + passed=True, + message=(f"{uid}: loaded ({panel_count} panels)"), + details={"panel_count": panel_count}, + ) + ) + else: + report.add( + CheckResult( + name=f"dashboard.{uid}", + category="dashboard", + passed=False, + message=f"{uid}: HTTP {resp.status}", + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=f"dashboard.{uid}", + category="dashboard", + passed=False, + message=f"{uid}: query failed ({exc})", + ) + ) + + +# --------------------------------------------------------------------------- +# Span duration validation +# --------------------------------------------------------------------------- + + +async def validate_span_durations( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate that span durations are within reasonable bounds. + + Checks that spans have duration > 0 and < 60s, flagging any anomalies. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- Span Duration Validation ---") + + try: + traces = await _tempo_search( + session, + tempo_url, + '{resource.service.name="xrpld"}', + limit=5, + ) + + if not traces: + report.add( + CheckResult( + name="span.duration_bounds", + category="span", + passed=False, + message="No traces available for duration check", + ) + ) + return + + total_spans = 0 + invalid_spans = 0 + max_duration_ns = 0 + + for trace_summary in traces: + trace_id = trace_summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + for span in spans: + start_ns = int(span.get("startTimeUnixNano", "0")) + end_ns = int(span.get("endTimeUnixNano", "0")) + duration_ns = end_ns - start_ns + total_spans += 1 + max_duration_ns = max(max_duration_ns, duration_ns) + # Invalid if negative or > 60 seconds. + if duration_ns < 0 or duration_ns > 60_000_000_000: + invalid_spans += 1 + + max_duration_ms = max_duration_ns / 1_000_000 + + report.add( + CheckResult( + name="span.duration_bounds", + category="span", + passed=invalid_spans == 0, + message=( + f"All {total_spans} spans have valid durations " + f"(max: {max_duration_ms:.1f}ms)" + if invalid_spans == 0 + else f"{invalid_spans}/{total_spans} spans have invalid " + "durations (<0 or >60s)" + ), + details={ + "total_spans": total_spans, + "invalid_spans": invalid_spans, + "max_duration_ms": round(max_duration_ms, 2), + }, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="span.duration_bounds", + category="span", + passed=False, + message=f"Duration check failed: {exc}", + ) + ) + + +# --------------------------------------------------------------------------- +# External Dashboard Parity Validation +# --------------------------------------------------------------------------- + +# Span attributes that external dashboards (validator-health, peer-quality, +# node-health) depend on. Each entry maps a span name to the +# attributes that must be present for external dashboard panels to render. +# Keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; +# dotted xrpl.* reserved for resource attributes). The amendment_blocked, +# server_state, and proposers_validated values that earlier external-dashboard +# work tracked are NOT span attributes — they exist only as MetricsRegistry +# metrics (validator_health{metric="amendment_blocked"}, +# state_tracking{metric="state_value"}, etc.), so they are validated by +# PARITY_VALUE_SANITY below rather than as span attributes here. +PARITY_SPAN_ATTRS: list[dict[str, str]] = [ + {"span": "tx.receive", "attr": "peer_version"}, + {"span": "consensus.validation.send", "attr": "ledger_hash"}, + {"span": "consensus.validation.send", "attr": "full_validation"}, + # peer.validation.receive shares the ledger_hash / full_validation keys with + # consensus.validation.send (same keys, told apart by span name). + {"span": "peer.validation.receive", "attr": "ledger_hash"}, + {"span": "peer.validation.receive", "attr": "full_validation"}, + {"span": "consensus.accept", "attr": "quorum"}, +] + +# Value sanity bounds for external-parity metrics. Each entry specifies a +# Prometheus query and the acceptable range [lo, hi] for the returned value. +PARITY_VALUE_SANITY: list[dict[str, Any]] = [ + { + "name": "validation_agreement_pct_1h", + "query": 'validation_agreement{metric="agreement_pct_1h"}', + "lo": 0, + "hi": 100, + }, + { + "name": "unl_expiry_days", + "query": 'validator_health{metric="unl_expiry_days"}', + "lo": 0, + "hi": None, + "exclusive_lo": True, + }, + { + "name": "peer_latency_p90_ms", + "query": 'peer_quality{metric="peer_latency_p90_ms"}', + "lo": 0, + "hi": None, + }, + { + "name": "state_value", + "query": 'state_tracking{metric="state_value"}', + "lo": 0, + "hi": 7, + }, +] + + +async def validate_parity_span_attrs( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate span attributes required by external dashboard panels. + + For each (span, attribute) pair in PARITY_SPAN_ATTRS, queries Tempo + for the span and checks that the attribute key exists on at least one + span in the returned traces. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- External Parity: Span Attribute Checks ---") + + for entry in PARITY_SPAN_ATTRS: + span_name = entry["span"] + attr_name = entry["attr"] + check_name = f"parity.span_attr.{span_name}.{attr_name}" + + try: + query = '{resource.service.name="xrpld" && name="' + span_name + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=5) + + if not traces: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=( + f"{span_name}: no traces found, " + f"cannot verify attr {attr_name}" + ), + ) + ) + continue + + # Fetch full trace and search spans for the attribute. + found = False + for trace_summary in traces: + trace_id = trace_summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + for span in spans: + if attr_name in _otlp_span_attr_keys(span): + found = True + break + if found: + break + + report.add( + CheckResult( + name=check_name, + category="parity", + passed=found, + message=( + f"{span_name}: attribute '{attr_name}' present" + if found + else f"{span_name}: attribute '{attr_name}' missing" + ), + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=f"{span_name}: attr check failed ({exc})", + ) + ) + + +async def validate_parity_value_sanity( + session: aiohttp.ClientSession, + prometheus_url: str, + report: ValidationReport, +) -> None: + """Validate that external-parity metric values fall within sane bounds. + + For each entry in PARITY_VALUE_SANITY, queries the current value from + Prometheus and checks it against the specified [lo, hi] range. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus API base URL. + report: ValidationReport to accumulate results. + """ + logger.info("--- External Parity: Value Sanity Checks ---") + + for entry in PARITY_VALUE_SANITY: + name = entry["name"] + query = entry["query"] + lo = entry["lo"] + hi = entry["hi"] + exclusive_lo = entry.get("exclusive_lo", False) + check_name = f"parity.value_sanity.{name}" + + try: + params = {"query": query} + async with session.get( + f"{prometheus_url}/api/v1/query", params=params + ) as resp: + data = await resp.json() + results = data.get("data", {}).get("result", []) + + if not results: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=f"{name}: no data returned from Prometheus", + ) + ) + continue + + # Use the first result's value. + value = float(results[0]["value"][1]) + + # Check bounds. + in_range = True + if exclusive_lo: + in_range = in_range and (value > lo) + else: + in_range = in_range and (value >= lo) + if hi is not None: + in_range = in_range and (value <= hi) + + # Build human-readable bound description. + lo_op = ">" if exclusive_lo else ">=" + bound_desc = f"{lo_op} {lo}" + if hi is not None: + bound_desc += f" and <= {hi}" + + report.add( + CheckResult( + name=check_name, + category="parity", + passed=in_range, + message=( + f"{name}: value {value} is within bounds ({bound_desc})" + if in_range + else f"{name}: value {value} out of bounds " + f"(expected {bound_desc})" + ), + details={"value": value, "lo": lo, "hi": hi}, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=f"{name}: sanity check failed ({exc})", + ) + ) + + +# --------------------------------------------------------------------------- +# Main validation orchestrator +# --------------------------------------------------------------------------- + + +async def run_validation( + tempo_url: str, + prometheus_url: str, + loki_url: str, + grafana_url: str, + skip_loki: bool = False, +) -> ValidationReport: + """Run all validation checks and return a report. + + Args: + tempo_url: Tempo API base URL. + prometheus_url: Prometheus API base URL. + loki_url: Loki API base URL. + grafana_url: Grafana API base URL. + skip_loki: If True, skip log-trace correlation checks. + + Returns: + ValidationReport with all check results. + """ + report = ValidationReport() + report.start_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + async with aiohttp.ClientSession() as session: + await validate_spans(session, tempo_url, report) + await validate_span_durations(session, tempo_url, report) + await validate_metrics(session, prometheus_url, report) + if not skip_loki: + await validate_log_trace_correlation(session, loki_url, tempo_url, report) + await validate_dashboards(session, grafana_url, report) + await validate_parity_span_attrs(session, tempo_url, report) + await validate_parity_value_sanity(session, prometheus_url, report) + + report.end_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Telemetry Validation Suite for xrpld", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run all validations with defaults: + python3 validate_telemetry.py + + # Write report to file: + python3 validate_telemetry.py --report /tmp/validation-report.json + + # Custom endpoints: + python3 validate_telemetry.py \\ + --tempo http://tempo:3200 --prometheus http://prom:9090 + + # Skip Loki checks (if log-trace correlation is not set up): + python3 validate_telemetry.py --skip-loki + """, + ) + parser.add_argument( + "--tempo", + type=str, + default=DEFAULT_TEMPO, + help=f"Tempo API URL (default: {DEFAULT_TEMPO})", + ) + parser.add_argument( + "--prometheus", + type=str, + default=DEFAULT_PROMETHEUS, + help=f"Prometheus API URL (default: {DEFAULT_PROMETHEUS})", + ) + parser.add_argument( + "--loki", + type=str, + default=DEFAULT_LOKI, + help=f"Loki API URL (default: {DEFAULT_LOKI})", + ) + parser.add_argument( + "--grafana", + type=str, + default=DEFAULT_GRAFANA, + help=f"Grafana API URL (default: {DEFAULT_GRAFANA})", + ) + parser.add_argument( + "--skip-loki", + action="store_true", + help="Skip log-trace correlation validation", + ) + parser.add_argument( + "--report", + type=str, + default=None, + help="Write JSON report to this file path", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the telemetry validation suite.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + report = asyncio.run( + run_validation( + tempo_url=args.tempo, + prometheus_url=args.prometheus, + loki_url=args.loki, + grafana_url=args.grafana, + skip_loki=args.skip_loki, + ) + ) + + # Print summary. + print("") + print("=" * 60) + print(" TELEMETRY VALIDATION REPORT") + print("=" * 60) + print(f" Total checks: {report.total_checks}") + print(f" Passed: {report.passed}") + print(f" Failed: {report.failed}") + print("=" * 60) + print("") + + # Print failures. + if report.failed > 0: + print("FAILED CHECKS:") + for check in report.checks: + if not check.passed: + print(f" [{check.category}] {check.name}: {check.message}") + print("") + + # Write report file. + report_dict = report.to_dict() + if args.report: + with open(args.report, "w") as f: + json.dump(report_dict, f, indent=2) + logger.info("Report written to %s", args.report) + else: + print(json.dumps(report_dict, indent=2)) + + # Exit with appropriate code for CI. + sys.exit(0 if report.all_passed else 1) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/workload-profiles.json b/docker/telemetry/workload/workload-profiles.json new file mode 100644 index 00000000000..84abcf3ed00 --- /dev/null +++ b/docker/telemetry/workload/workload-profiles.json @@ -0,0 +1,108 @@ +{ + "profiles": { + "full-validation": { + "description": "Full 18-dashboard coverage with burst/idle/plateau patterns", + "phases": [ + { + "name": "warmup", + "description": "Low load to populate baseline gauges and node health metrics", + "duration_sec": 30, + "rpc": { + "rate": 5, + "weights": { "server_info": 50, "fee": 30, "ledger": 20 } + }, + "tx": null + }, + { + "name": "steady-state", + "description": "Medium sustained load — plateau data for all dashboards", + "duration_sec": 60, + "rpc": { "rate": 30 }, + "tx": { "tps": 3 } + }, + { + "name": "rpc-burst", + "description": "Heavy RPC to saturate job queue and spike latency", + "duration_sec": 30, + "rpc": { "rate": 100 }, + "tx": null + }, + { + "name": "tx-flood", + "description": "High TX rate for fee escalation and TxQ pressure", + "duration_sec": 30, + "rpc": { "rate": 5, "weights": { "server_info": 50, "fee": 50 } }, + "tx": { + "tps": 20, + "weights": { "Payment": 70, "OfferCreate": 20, "TrustSet": 10 } + } + }, + { + "name": "txq-burst", + "description": "Single-type Payment burst at high TPS to force open-ledger fee escalation and TxQ queueing, exercising the txq.* spans (txq.enqueue / txq.accept / txq.accept_tx / txq.cleanup)", + "duration_sec": 30, + "rpc": { "rate": 5, "weights": { "fee": 100 } }, + "tx": { + "tps": 60, + "weights": { "Payment": 100 } + } + }, + { + "name": "mixed-peak", + "description": "Realistic peak load — consensus and ledger ops under stress", + "duration_sec": 60, + "rpc": { "rate": 50 }, + "tx": { "tps": 10 } + }, + { + "name": "cooldown", + "description": "Low load for recovery metrics and state transition data", + "duration_sec": 30, + "rpc": { "rate": 5, "weights": { "server_info": 80, "fee": 20 } }, + "tx": null + } + ], + "propagation_wait_sec": 60 + }, + "quick-smoke": { + "description": "Fast smoke test — minimal data for CI quick checks", + "phases": [ + { + "name": "smoke", + "description": "Single phase covering all generator types", + "duration_sec": 30, + "rpc": { "rate": 20 }, + "tx": { "tps": 3 } + } + ], + "propagation_wait_sec": 30 + }, + "stress": { + "description": "Heavy sustained load for performance benchmarking", + "phases": [ + { + "name": "ramp-up", + "description": "Gradually increasing load", + "duration_sec": 30, + "rpc": { "rate": 20 }, + "tx": { "tps": 5 } + }, + { + "name": "peak", + "description": "Maximum sustained load", + "duration_sec": 120, + "rpc": { "rate": 150 }, + "tx": { "tps": 25 } + }, + { + "name": "sustain", + "description": "Continued high load for stability check", + "duration_sec": 60, + "rpc": { "rate": 100 }, + "tx": { "tps": 15 } + } + ], + "propagation_wait_sec": 60 + } + } +} diff --git a/docker/telemetry/workload/workload_orchestrator.py b/docker/telemetry/workload/workload_orchestrator.py new file mode 100755 index 00000000000..d9697d49434 --- /dev/null +++ b/docker/telemetry/workload/workload_orchestrator.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +"""Workload Orchestrator for rippled telemetry validation. + +Reads a named profile from workload-profiles.json and executes sequential +load phases, each with configurable RPC and TX parameters. Produces a +combined report with per-phase results. + +Phases run sequentially. Within each phase, the RPC load generator and +transaction submitter run concurrently (if both are configured). + +Orchestration Flow:: + + workload-profiles.json + | + v + workload_orchestrator.py + | + +----+----+----+----+----+----+ + | Phase 1 | Phase 2 | ...... | Phase N | + +----+----+----+----+----+----+ + | | + +----+----+ +----+----+ + | rpc_load | | tx_sub | (concurrent within phase) + | _gen.py | | mitter | + +----+----+ +----+----+ + | | + v v + per-phase JSON reports + | + v + combined-report.json + +Usage: + python3 workload_orchestrator.py --profile full-validation + python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006 + python3 workload_orchestrator.py --profile stress --report /tmp/report.json + +Profiles are defined in workload-profiles.json in the same directory. +""" + +import argparse +import asyncio +import json +import logging +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger("workload_orchestrator") + +SCRIPT_DIR = Path(__file__).parent.resolve() +PROFILES_FILE = SCRIPT_DIR / "workload-profiles.json" + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class PhaseResult: + """Result of a single workload phase. + + Attributes: + name: Phase name from the profile. + duration_sec: Configured duration. + actual_sec: Actual elapsed time. + rpc_summary: JSON summary from rpc_load_generator, or None. + tx_summary: JSON summary from tx_submitter, or None. + errors: List of error messages from subprocess failures. + """ + + name: str + duration_sec: int + actual_sec: float = 0.0 + rpc_summary: dict[str, Any] | None = None + tx_summary: dict[str, Any] | None = None + errors: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Profile loading +# --------------------------------------------------------------------------- + + +def load_profile(profile_name: str) -> dict[str, Any]: + """Load a named profile from workload-profiles.json. + + Args: + profile_name: Key in the profiles dict (e.g., "full-validation"). + + Returns: + The profile dict with phases and propagation_wait_sec. + + Raises: + SystemExit: If the profile file or name is not found. + """ + if not PROFILES_FILE.exists(): + logger.error("Profiles file not found: %s", PROFILES_FILE) + sys.exit(2) + + with open(PROFILES_FILE) as f: + data = json.load(f) + + profiles = data.get("profiles", {}) + if profile_name not in profiles: + available = ", ".join(profiles.keys()) + logger.error("Profile '%s' not found. Available: %s", profile_name, available) + sys.exit(2) + + profile = profiles[profile_name] + + # Validate profile schema — fail fast on bad config. + phases = profile.get("phases", []) + if not isinstance(phases, list) or not phases: + logger.error("Profile '%s' has no valid phases", profile_name) + sys.exit(2) + for i, phase in enumerate(phases): + if not isinstance(phase.get("name"), str): + logger.error("Phase %d missing valid 'name'", i) + sys.exit(2) + if ( + not isinstance(phase.get("duration_sec"), (int, float)) + or phase["duration_sec"] <= 0 + ): + logger.error( + "Phase %d '%s' has invalid duration_sec", + i, + phase.get("name"), + ) + sys.exit(2) + + logger.info( + "Loaded profile '%s': %s (%d phases)", + profile_name, + profile.get("description", ""), + len(phases), + ) + return profile + + +# --------------------------------------------------------------------------- +# Subprocess execution +# --------------------------------------------------------------------------- + + +async def run_subprocess(cmd: list[str], label: str) -> tuple[int, str, str]: + """Run a subprocess and capture its stdout and stderr. + + Args: + cmd: Command and arguments. + label: Human-readable label for logging. + + Returns: + Tuple of (return_code, stdout_text, stderr_text). + """ + logger.debug("Starting %s: %s", label, " ".join(cmd)) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + logger.warning( + "%s exited with code %d: %s", + label, + proc.returncode, + stderr.decode().strip()[-500:], + ) + return proc.returncode, stdout.decode(), stderr.decode() + + +# --------------------------------------------------------------------------- +# Phase execution +# --------------------------------------------------------------------------- + + +def _collect_task_result( + label: str, + returncode: int, + stderr: str, + report_path: Path, + result: PhaseResult, +) -> None: + """Process the result of a completed subprocess task. + + Reads the JSON report file (if it exists) and records any errors. + + Args: + label: "rpc" or "tx". + returncode: Subprocess exit code. + stderr: Captured stderr text. + report_path: Path to the JSON report file. + result: PhaseResult to update. + """ + if report_path.exists(): + try: + with open(report_path) as f: + summary = json.load(f) + if label == "rpc": + result.rpc_summary = summary + elif label == "tx": + result.tx_summary = summary + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Failed to parse %s report %s: %s", label, report_path, exc) + result.errors.append(f"Failed to parse {label} report: {exc}") + + if returncode != 0: + snippet = stderr.strip()[-200:] if stderr else "" + result.errors.append( + f"{label.upper()} generator exited with {returncode}: {snippet}" + ) + + +def _build_rpc_cmd( + endpoints: list[str], rpc_cfg: dict[str, Any], duration: int, output: Path +) -> list[str]: + """Build the command list for the RPC load generator subprocess.""" + cmd = [ + sys.executable, + str(SCRIPT_DIR / "rpc_load_generator.py"), + "--endpoints", + *endpoints, + "--rate", + str(rpc_cfg.get("rate", 50)), + "--duration", + str(duration), + "--output", + str(output), + ] + weights = rpc_cfg.get("weights") + if weights: + cmd.extend(["--weights", json.dumps(weights)]) + return cmd + + +def _build_tx_cmd( + endpoint: str, tx_cfg: dict[str, Any], duration: int, output: Path +) -> list[str]: + """Build the command list for the TX submitter subprocess.""" + cmd = [ + sys.executable, + str(SCRIPT_DIR / "tx_submitter.py"), + "--endpoint", + endpoint, + "--tps", + str(tx_cfg.get("tps", 5)), + "--duration", + str(duration), + "--output", + str(output), + ] + weights = tx_cfg.get("weights") + if weights: + cmd.extend(["--weights", json.dumps(weights)]) + return cmd + + +async def run_phase( + phase: dict[str, Any], + endpoints: list[str], + report_dir: Path, + phase_idx: int, +) -> PhaseResult: + """Execute a single workload phase. + + Launches rpc_load_generator.py and/or tx_submitter.py as subprocesses + based on the phase configuration. Both run concurrently if configured. + + Args: + phase: Phase dict from the profile. + endpoints: List of WebSocket endpoint URLs. + report_dir: Directory for per-phase JSON reports. + phase_idx: Phase index (for file naming). + + Returns: + PhaseResult with subprocess outputs. + """ + name = phase["name"] + duration = phase["duration_sec"] + result = PhaseResult(name=name, duration_sec=duration) + prefix = f"phase{phase_idx + 1}-{name}" + + logger.info( + "=== Phase %d: %s (%ds) — %s ===", + phase_idx + 1, + name, + duration, + phase.get("description", ""), + ) + + tasks: list[tuple[str, Path, asyncio.Task]] = [] + t0 = time.monotonic() + + rpc_cfg = phase.get("rpc") + if rpc_cfg: + rpc_out = report_dir / f"{prefix}-rpc.json" + cmd = _build_rpc_cmd(endpoints, rpc_cfg, duration, rpc_out) + tasks.append( + ("rpc", rpc_out, asyncio.create_task(run_subprocess(cmd, f"RPC [{name}]"))) + ) + + tx_cfg = phase.get("tx") + if tx_cfg: + tx_out = report_dir / f"{prefix}-tx.json" + cmd = _build_tx_cmd(endpoints[0], tx_cfg, duration, tx_out) + tasks.append( + ("tx", tx_out, asyncio.create_task(run_subprocess(cmd, f"TX [{name}]"))) + ) + + if not tasks: + logger.warning( + "Phase %d: %s — no workload configured, skipping", phase_idx + 1, name + ) + return result + + for label, report_path, task in tasks: + returncode, _stdout, stderr = await task + _collect_task_result(label, returncode, stderr, report_path, result) + + result.actual_sec = time.monotonic() - t0 + logger.info( + "Phase %d complete: %.1fs actual, %d errors", + phase_idx + 1, + result.actual_sec, + len(result.errors), + ) + return result + + +# --------------------------------------------------------------------------- +# Profile execution +# --------------------------------------------------------------------------- + + +async def run_profile( + profile: dict[str, Any], + endpoints: list[str], + report_dir: Path, +) -> dict[str, Any]: + """Execute all phases in a profile sequentially. + + Args: + profile: Profile dict with phases and propagation_wait_sec. + endpoints: WebSocket endpoints for the rippled cluster. + report_dir: Directory for phase reports. + + Returns: + Combined report dict with per-phase results and totals. + """ + phases = profile.get("phases", []) + propagation_wait = profile.get("propagation_wait_sec", 60) + results: list[PhaseResult] = [] + + total_start = time.monotonic() + + for idx, phase in enumerate(phases): + result = await run_phase(phase, endpoints, report_dir, idx) + results.append(result) + + # Wait for telemetry data to propagate through the collector pipeline. + logger.info("Waiting %ds for telemetry data to propagate...", propagation_wait) + await asyncio.sleep(propagation_wait) + + total_elapsed = time.monotonic() - total_start + + # Build combined report from all phase results. + total_rpc_sent = 0 + total_rpc_errors = 0 + total_tx_submitted = 0 + total_tx_errors = 0 + phase_reports = [] + + for r in results: + pr: dict[str, Any] = { + "name": r.name, + "duration_sec": r.duration_sec, + "actual_sec": round(r.actual_sec, 1), + "errors": r.errors, + } + if r.rpc_summary: + pr["rpc"] = r.rpc_summary + total_rpc_sent += r.rpc_summary.get("total_sent", 0) + total_rpc_errors += r.rpc_summary.get("total_errors", 0) + if r.tx_summary: + pr["tx"] = r.tx_summary + total_tx_submitted += r.tx_summary.get("total_submitted", 0) + total_tx_errors += r.tx_summary.get("total_errors", 0) + phase_reports.append(pr) + + report = { + "profile": profile.get("description", ""), + "total_elapsed_sec": round(total_elapsed, 1), + "phases": phase_reports, + "totals": { + "rpc_sent": total_rpc_sent, + "rpc_errors": total_rpc_errors, + "tx_submitted": total_tx_submitted, + "tx_errors": total_tx_errors, + }, + } + + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Workload Orchestrator for rippled telemetry validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Profiles: + full-validation Full 18-dashboard coverage (~5 min load + 1 min propagation) + quick-smoke Fast CI smoke test (~30s load + 30s propagation) + stress Heavy sustained load for benchmarking (~3.5 min + 1 min) + +Examples: + python3 workload_orchestrator.py --profile full-validation + python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006 + python3 workload_orchestrator.py --profile stress --report /tmp/report.json + """, + ) + parser.add_argument( + "--profile", + type=str, + required=True, + help="Named profile from workload-profiles.json", + ) + parser.add_argument( + "--endpoints", + nargs="+", + default=["ws://localhost:6006"], + help="WebSocket endpoints (default: ws://localhost:6006)", + ) + parser.add_argument( + "--report", + type=str, + default=None, + help="Write combined JSON report to this file", + ) + parser.add_argument( + "--report-dir", + type=str, + default="/tmp/xrpld-validation/reports", + help="Directory for per-phase reports", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the workload orchestrator.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + profile = load_profile(args.profile) + report_dir = Path(args.report_dir) + report_dir.mkdir(parents=True, exist_ok=True) + + report = asyncio.run(run_profile(profile, args.endpoints, report_dir)) + + print(json.dumps(report, indent=2)) + + if args.report: + with open(args.report, "w") as f: + json.dump(report, f, indent=2) + logger.info("Combined report written to %s", args.report) + + # Exit with error if either generator had high error rates. + totals = report["totals"] + rpc_err_rate = ( + totals["rpc_errors"] / totals["rpc_sent"] * 100 if totals["rpc_sent"] > 0 else 0 + ) + tx_err_rate = ( + totals["tx_errors"] / totals["tx_submitted"] * 100 + if totals["tx_submitted"] > 0 + else 0 + ) + # TX threshold is higher because short-lived CI test environments lack + # pre-funded accounts, causing expected failures for complex transactions + # (AMMCreate, EscrowFinish, etc.) that require specific ledger state. + if rpc_err_rate > 50 or tx_err_rate > 95: + logger.error( + "High error rates: RPC=%.1f%%, TX=%.1f%%", rpc_err_rate, tx_err_rate + ) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/xrpld-validator.cfg.template b/docker/telemetry/workload/xrpld-validator.cfg.template new file mode 100644 index 00000000000..725faf550cf --- /dev/null +++ b/docker/telemetry/workload/xrpld-validator.cfg.template @@ -0,0 +1,96 @@ +# xrpld validator node configuration template for workload harness. +# +# Placeholders (replaced by docker-compose entrypoint): +# {{NODE_INDEX}} — Node number (1-based) +# {{RPC_PORT}} — HTTP RPC port +# {{WS_PORT}} — WebSocket port +# {{PEER_PORT}} — Peer protocol port +# {{DATA_DIR}} — Node data directory +# {{VALIDATION_SEED}} — Validator seed from key generation +# {{VALIDATORS_FILE}} — Path to shared validators.txt +# {{IPS_FIXED}} — Peer addresses (one per line) +# {{OTEL_ENDPOINT}} — OTel Collector OTLP/HTTP traces endpoint +# {{OTEL_METRICS_ENDPOINT}} — OTel Collector OTLP/HTTP metrics endpoint +# {{LOG_LEVEL}} — Log level (debug, info, warning, error) + +[server] +port_rpc +port_ws +port_peer + +[port_rpc] +port = {{RPC_PORT}} +ip = 0.0.0.0 +admin = 0.0.0.0 +protocol = http + +[port_ws] +port = {{WS_PORT}} +ip = 0.0.0.0 +admin = 0.0.0.0 +protocol = ws + +[port_peer] +port = {{PEER_PORT}} +ip = 0.0.0.0 +protocol = peer + +[node_db] +type=NuDB +path={{DATA_DIR}}/nudb +online_delete=256 + +[database_path] +{{DATA_DIR}}/db + +[debug_logfile] +{{DATA_DIR}}/debug.log + +[validation_seed] +{{VALIDATION_SEED}} + +[validators_file] +{{VALIDATORS_FILE}} + +[ips_fixed] +{{IPS_FIXED}} + +[peer_private] +1 + +# --- OpenTelemetry tracing (all categories enabled) --- +[telemetry] +enabled=1 +service_instance_id=validator-{{NODE_INDEX}} +endpoint={{OTEL_ENDPOINT}} +exporter=otlp_http +batch_size=512 +batch_delay_ms=2000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 + +# --- Native OTel metrics (beast::insight over OTLP/HTTP) --- +# The collector has no StatsD receiver (metrics pipeline is [otlp, spanmetrics]), +# so beast::insight exports natively over OTLP. prefix=xrpld matches the OTel +# resource service name and the metric names the dashboards query. +[insight] +server=otel +endpoint={{OTEL_METRICS_ENDPOINT}} +prefix=xrpld + +[rpc_startup] +{ "command": "log_level", "severity": "{{LOG_LEVEL}}" } + +[ssl_verify] +0 + +# --- Network tuning for local cluster --- +[network_id] +0 + +[sntp_servers] +time.google.com diff --git a/docker/telemetry/xrpld-telemetry-mainnet.cfg b/docker/telemetry/xrpld-telemetry-mainnet.cfg new file mode 100644 index 00000000000..afca57e4fa0 --- /dev/null +++ b/docker/telemetry/xrpld-telemetry-mainnet.cfg @@ -0,0 +1,141 @@ +# xrpld configuration for Mainnet with full OpenTelemetry tracing. +# +# Connects to XRP Ledger Mainnet as an observer-only tracking node +# (no validator credentials) and exercises ALL instrumented workflows: +# RPC, transactions, consensus, peer overlay, ledger ops, and +# pathfinding. +# +# Usage: +# 1. Start the observability stack: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# 2. Run xrpld: +# ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg +# 3. Wait for sync (server_state=full), then exercise workflows: +# curl -s http://localhost:5005 -d '{"method":"server_info"}' +# 4. View traces in Grafana Explore -> Tempo: http://localhost:3000 + +# --- Server ports ----------------------------------------------------------- + +[server] +port_rpc_admin_local +port_ws_admin_local +port_ws_public +port_peer + +[port_rpc_admin_local] +port = 5005 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_ws_admin_local] +port = 6006 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = ws + +[port_ws_public] +port = 6005 +ip = 0.0.0.0 +protocol = ws + +[port_peer] +port = 51235 +ip = 0.0.0.0 +protocol = peer + +# --- Network ---------------------------------------------------------------- + +[network_id] +main + +# [ips] omitted on purpose. xrpld uses its built-in mainnet hub list +# (r.ripple.com, sahyadri.isrdc.in, hubs.xrpkuwait.com, +# hub.xrpl-commons.org) for peer discovery. + +[validators_file] +validators-mainnet.txt + +[peer_private] +0 + +[peers_max] +21 + +# --- Pathfinding (exercises ripple_path_find / path_find workflows) --------- + +[path_search] +7 + +[path_search_fast] +2 + +[path_search_max] +10 + +# --- Signing (allows sign/sign_for RPC for test tx submission) -------------- + +[signing_support] +true + +# --- Database --------------------------------------------------------------- + +[node_db] +type=NuDB +path=docker/telemetry/data/nudb +online_delete=2000 +advisory_delete=0 + +[database_path] +docker/telemetry/data + +[ledger_history] +1000 + +# --- Logging ---------------------------------------------------------------- + +# Path is resolved relative to this config file's directory (docker/telemetry), +# so this writes to docker/telemetry/data/logs/mainnet/debug.log — the same +# dir the compose stack bind-mounts into the collector as /var/log/xrpld. +[debug_logfile] +data/logs/mainnet/debug.log + +[rpc_startup] +{ "command": "log_level", "severity": "warning" } + +# --- SSL -------------------------------------------------------------------- + +[ssl_verify] +0 + +# --- Insight (native OTel metrics via beast::insight) ----------------------- + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +# Sets the OTel service.instance.id resource attribute, which Prometheus +# exposes as the `service_instance_id` label. Dashboards filter on it via the +# $node template variable, so without this every insight-backed panel is +# empty. Matches [telemetry] service_instance_id for a single node identity. +service_instance_id=xrpld-mainnet + +# --- OpenTelemetry tracing -------------------------------------------------- + +[telemetry] +enabled=1 +service_instance_id=xrpld-mainnet +endpoint=http://localhost:4318/v1/traces +metrics_endpoint=http://localhost:4318/v1/metrics +exporter=otlp_http +# Mainnet has high span throughput across peer/ledger/consensus. Head +# sampling is fixed at 1.0 (sample everything) and not configurable; +# reduce Tempo/collector load with collector-side tail sampling. +batch_size=512 +batch_delay_ms=5000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 diff --git a/docker/telemetry/xrpld-telemetry.cfg b/docker/telemetry/xrpld-telemetry.cfg new file mode 100644 index 00000000000..bd7454e5983 --- /dev/null +++ b/docker/telemetry/xrpld-telemetry.cfg @@ -0,0 +1,136 @@ +# xrpld configuration for Devnet with full OpenTelemetry tracing. +# +# Connects to the XRP Ledger Devnet and exercises ALL instrumented +# workflows: RPC, transactions, consensus, peer overlay, ledger ops, +# and pathfinding. +# +# Usage: +# 1. Start the observability stack: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# 2. Run xrpld: +# ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg +# 3. Wait for sync (server_state=full), then exercise workflows: +# curl -s http://localhost:5005 -d '{"method":"server_info"}' +# 4. View traces in Grafana Explore -> Tempo: http://localhost:3000 + +# --- Server ports ----------------------------------------------------------- + +[server] +port_rpc_admin_local +port_ws_admin_local +port_ws_public +port_peer + +[port_rpc_admin_local] +port = 5005 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_ws_admin_local] +port = 6006 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = ws + +[port_ws_public] +port = 6005 +ip = 0.0.0.0 +protocol = ws + +[port_peer] +port = 51235 +ip = 0.0.0.0 +protocol = peer + +# --- Network ---------------------------------------------------------------- + +[network_id] +devnet + +[ips] +s.devnet.rippletest.net 51235 + +[validators_file] +validators-devnet.txt + +[peer_private] +0 + +[peers_max] +21 + +# --- Pathfinding (exercises ripple_path_find / path_find workflows) --------- + +[path_search] +7 + +[path_search_fast] +2 + +[path_search_max] +10 + +# --- Signing (allows sign/sign_for RPC for test tx submission) -------------- + +[signing_support] +true + +# --- Database --------------------------------------------------------------- + +[node_db] +type=NuDB +path=docker/telemetry/data/nudb +online_delete=2000 +advisory_delete=0 + +[database_path] +docker/telemetry/data + +[ledger_history] +1000 + +# --- Logging ---------------------------------------------------------------- + +# Path is resolved relative to this config file's directory (docker/telemetry), +# so this writes to docker/telemetry/data/logs/devnet/debug.log — the same +# dir the compose stack bind-mounts into the collector as /var/log/xrpld. +[debug_logfile] +data/logs/devnet/debug.log + +[rpc_startup] +{ "command": "log_level", "severity": "debug" } + +# --- SSL -------------------------------------------------------------------- + +[ssl_verify] +0 + +# --- Insight (native OTel metrics via beast::insight) ----------------------- + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +# Sets the OTel service.instance.id resource attribute, which Prometheus +# exposes as the `service_instance_id` label. Dashboards filter on it via the +# $node template variable, so without this every insight-backed panel is +# empty. Matches [telemetry] service_instance_id for a single node identity. +service_instance_id=xrpld-devnet + +# --- OpenTelemetry tracing -------------------------------------------------- + +[telemetry] +enabled=1 +service_instance_id=xrpld-devnet +endpoint=http://localhost:4318/v1/traces +metrics_endpoint=http://localhost:4318/v1/metrics +exporter=otlp_http +batch_size=512 +batch_delay_ms=5000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 diff --git a/docs/build/telemetry.md b/docs/build/telemetry.md new file mode 100644 index 00000000000..f4681b3309d --- /dev/null +++ b/docs/build/telemetry.md @@ -0,0 +1,161 @@ +# OpenTelemetry Tracing for xrpld + +This document explains how to build xrpld with OpenTelemetry distributed tracing support, configure the runtime telemetry options, and set up the observability backend to view traces. + +- [OpenTelemetry Tracing for xrpld](#opentelemetry-tracing-for-xrpld) + - [Overview](#overview) + - [Building with Telemetry](#building-with-telemetry) + - [Summary](#summary) + - [Build steps](#build-steps) + - [Install dependencies](#install-dependencies) + - [Call CMake](#call-cmake) + - [Build](#build) + - [Building without telemetry](#building-without-telemetry) + - [Troubleshooting](#troubleshooting) + - [Conan lockfile error](#conan-lockfile-error) + - [CMake target not found](#cmake-target-not-found) + - [Conditional compilation](#conditional-compilation) + - [Span lifetime and cross-thread handling](#span-lifetime-and-cross-thread-handling) + +## Overview + +xrpld supports optional [OpenTelemetry](https://opentelemetry.io/) distributed tracing. +When enabled, it instruments RPC requests with trace spans that are exported via +OTLP/HTTP to an OpenTelemetry Collector, which forwards them to a tracing backend +such as Grafana Tempo. + +Telemetry is **off by default** at both compile time and runtime: + +- **Compile time**: The Conan option `telemetry` and CMake option `telemetry` must be set to `True`/`ON`. + When disabled, all `SpanGuard` calls compile to inline no-ops (defined in `SpanGuard.h`) + with zero overhead — no OTel SDK dependency required. +- **Runtime**: The `[telemetry]` config section must set `enabled=1`. + When disabled at runtime, a no-op implementation is used. + +## Building with Telemetry + +### Summary + +Follow the same instructions as mentioned in [BUILD.md](../../BUILD.md) but with the following changes: + +1. Pass `-o telemetry=True` to `conan install` to pull the `opentelemetry-cpp` dependency. +2. CMake will automatically pick up `telemetry=ON` from the Conan-generated toolchain. +3. Build as usual. + +--- + +### Build steps + +```bash +cd /path/to/xrpld +rm -rf .build +mkdir .build +cd .build +``` + +#### Install dependencies + +The `telemetry` option adds `opentelemetry-cpp/1.26.0` as a dependency. +If the Conan lockfile does not yet include this package, bypass it with `--lockfile=""`. + +```bash +conan install .. \ + --output-folder . \ + --build missing \ + --settings build_type=Debug \ + -o telemetry=True \ + -o tests=True \ + -o xrpld=True \ + --lockfile="" +``` + +> **Note**: The first build with telemetry may take longer as `opentelemetry-cpp` +> and its transitive dependencies are compiled from source. + +#### Call CMake + +The Conan-generated toolchain file sets `telemetry=ON` automatically. +No additional CMake flags are needed beyond the standard ones. + +```bash +cmake .. -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -Dtests=ON -Dxrpld=ON +``` + +You should see in the CMake output: + +``` +-- OpenTelemetry tracing enabled +``` + +#### Build + +```bash +cmake --build . --parallel $(nproc) +``` + +## Building without telemetry + +Omit the `-o telemetry=True` option (or pass `-o telemetry=False`). +The `opentelemetry-cpp` dependency will not be downloaded, +the `XRPL_ENABLE_TELEMETRY` preprocessor define will not be set, +and all tracing macros will compile to no-ops. +The resulting binary is identical to one built before telemetry support was added. + +## Troubleshooting + +### Conan lockfile error + +If you see `ERROR: Requirement 'opentelemetry-cpp/1.26.0' not in lockfile 'requires'`, +the lockfile was generated without the telemetry dependency. +Pass `--lockfile=""` to bypass the lockfile, or regenerate it with telemetry enabled. + +### CMake target not found + +If CMake reports that `opentelemetry-cpp` targets are not found, +ensure you ran `conan install` with `-o telemetry=True` and that the +Conan-generated toolchain file is being used. +The Conan package provides a single umbrella target +`opentelemetry-cpp::opentelemetry-cpp` (not individual component targets). + +## Conditional compilation + +All OpenTelemetry SDK types are hidden behind the pimpl idiom in `SpanGuard.cpp`. +When `XRPL_ENABLE_TELEMETRY` is not defined, `SpanGuard.h` provides an all-inline +no-op stub class with zero overhead and zero OTel dependencies. +At runtime, if `enabled=0` is set in config (or the section is omitted), a +`NullTelemetry` implementation is used that returns no-op spans. +This two-layer approach ensures zero overhead when telemetry is not wanted. + +## Span lifetime and cross-thread handling + +`SpanGuard` is an RAII wrapper: it activates a span on the current thread when +constructed and ends it when destroyed. A normal (scoped) guard binds an OTel +`Scope` to the constructing thread's thread-local context stack. Because of +this, a scoped guard must be used and destroyed on the thread that created it. + +Two entry points cover the cases where the default scoped behaviour is wrong: + +- **`SpanGuard::rootSpan(cat, prefix, name)`** starts a span as a fresh trace + root, ignoring whatever span is already active on the current thread. Use it + at an inbound entry point that runs on a shared worker thread (for example a + peer message received while unrelated work is still on the stack), so those + unrelated spans do not become parents of the new trace. + +- **`std::move(guard).detached()`** returns a new guard that holds the same + span with **no** thread-local `Scope`. Call it on the origin thread before + handing the guard to a job that runs on another thread. The returned guard + can be moved freely across threads and ended anywhere; only its final + destruction ends the span. + +### Why are unrelated spans in my trace? + +If a trace shows hundreds of unrelated spans nested under one operation, a +scoped guard was likely moved to another thread (for example stored in a job +and destroyed on a worker thread). Destroying a scoped guard off its origin +thread pops the wrong context stack, leaving the origin thread's active span +in place so later spans inherit it. Fix it by calling `.detached()` on the +origin thread before the hand-off, or by starting the operation with +`rootSpan()` so it never inherits an ambient parent. diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md new file mode 100644 index 00000000000..615e8f148fd --- /dev/null +++ b/docs/telemetry-runbook.md @@ -0,0 +1,1405 @@ +# xrpld Telemetry Operator Runbook + +## Table of Contents + +- [Overview](#overview) +- [Quick Start](#quick-start) +- [Configuration Reference](#configuration-reference) +- [Exporting to Grafana Cloud](#exporting-to-grafana-cloud) +- [Span Reference](#span-reference) + - [RPC Spans](#rpc-spans) + - [Transaction Spans](#transaction-spans) + - [Transaction Queue Spans](#transaction-queue-spans) + - [PathFinding Spans](#pathfinding-spans) + - [Consensus Spans](#consensus-spans) + - [Ledger Spans](#ledger-spans) + - [Peer Spans](#peer-spans) +- [Insights and Sample Queries](#insights-and-sample-queries) + - [Transaction Workflow Analysis](#transaction-workflow-analysis) + - [DEX (OfferCreate / OfferCancel)](#dex-offercreate--offercancel) + - [Apply Pipeline by Stage](#apply-pipeline-by-stage) + - [Transaction Queue Health](#transaction-queue-health) + - [RPC Debugging](#rpc-debugging) + - [PathFinding Performance](#pathfinding-performance) + - [Consensus Health](#consensus-health) + - [Cross-Subsystem Correlation](#cross-subsystem-correlation) +- [Cross-Node Trace Propagation](#cross-node-trace-propagation) +- [Prometheus Metrics (Spanmetrics)](#prometheus-metrics-spanmetrics) +- [System Metrics (OTel native -- beast::insight)](#system-metrics-otel-native----beastinsight) +- [Grafana Dashboards](#grafana-dashboards) +- [Alerting](#alerting) +- [Log-Trace Correlation](#log-trace-correlation) +- [Troubleshooting](#troubleshooting) +- [Performance Tuning](#performance-tuning) +- [Disabling Telemetry](#disabling-telemetry) +- [Validating Telemetry Stack](#validating-telemetry-stack) +- [Performance Benchmarking](#performance-benchmarking) + +## Overview + +xrpld supports OpenTelemetry distributed tracing to provide visibility into RPC requests, transaction processing, and consensus rounds. + +This runbook covers operating a running node and querying its traces. For +building xrpld with telemetry support and the internal architecture, see +[build/telemetry.md](build/telemetry.md). + +## Quick Start + +### 1. Start the observability stack + +```bash +docker compose -f docker/telemetry/docker-compose.yml up -d +``` + +This starts: + +- **OTel Collector** on ports 4317 (gRPC) and 4318 (HTTP), and 13133 (health) +- **Tempo** on http://localhost:3200 (trace backend) +- **Prometheus** on http://localhost:9090 +- **Loki** on http://localhost:3100 (log aggregation) +- **Grafana** on http://localhost:3000 (Tempo pre-configured as datasource) + +### 2. Enable telemetry in xrpld + +Add to your `xrpld.cfg`: + +```ini +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +``` + +### 3. Build with telemetry support + +```bash +conan install . --build=missing -o telemetry=True +cmake --preset default -Dtelemetry=ON +cmake --build --preset default +``` + +### 4. Run against a live network + +Two ready-made configs connect a tracking node (no validator credentials) to a +public network with all tracing and native metrics enabled: + +| Config | Network | +| ---------------------------------------------- | ------- | +| `docker/telemetry/xrpld-telemetry.cfg` | Devnet | +| `docker/telemetry/xrpld-telemetry-mainnet.cfg` | Mainnet | + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry-mainnet.cfg +``` + +Both set `[insight] server=otel` (native metrics → collector → Prometheus, which +drives the dashboards) and `service_instance_id`, exposed by Prometheus as the +`service_instance_id` label that the `$node` dashboard variable filters on. The +mainnet config logs to `/var/log/xrpld/mainnet/debug.log` — the path +the collector's filelog receiver tails for log-trace correlation. + +Metrics begin flowing as soon as the node connects to peers (`server_state` +≥ `connected`); full ledger and consensus panels populate after sync +(`server_state` = `full`). Check progress with: + +```bash +curl -s http://localhost:5005 -d '{"method":"server_info"}' | + jq '.result.info | {server_state, peers, complete_ledgers}' +``` + +> Mainnet sync is bandwidth- and disk-heavy. For a quick check use the devnet +> config or the standalone test in `docker/telemetry/TESTING.md`, which +> generates spans without waiting for a live sync. + +## Configuration Reference + +| Option | Default | Description | +| -------------------------- | --------------------------------- | --------------------------------------------------------- | +| `enabled` | `0` | Master switch for telemetry | +| `endpoint` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint | +| `service_name` | `xrpld` | OpenTelemetry service name resource attribute | +| `service_instance_id` | node public key | OpenTelemetry service instance ID resource attribute | +| `trace_rpc` | `1` | Enable RPC request tracing | +| `trace_transactions` | `1` | Enable transaction tracing | +| `trace_consensus` | `1` | Enable consensus tracing | +| `trace_peer` | `1` | Enable peer message tracing (high volume) | +| `trace_ledger` | `1` | Enable ledger tracing | +| `consensus_trace_strategy` | `deterministic` | Consensus trace ID strategy (`deterministic` or `random`) | +| `batch_size` | `512` | Max spans per batch export | +| `batch_delay_ms` | `5000` | Delay between batch exports | +| `max_queue_size` | `2048` | Max spans queued before dropping | +| `use_tls` | `0` | Use TLS for exporter connection | +| `tls_ca_cert` | (empty) | Path to CA certificate bundle | +| `tls_client_cert` | (empty) | Client cert (PEM) for mutual TLS; empty = one-way TLS | +| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert` | + +## Exporting to Grafana Cloud + +The collector can ship traces, metrics, and logs to a hosted **Grafana +Cloud** stack instead of (or alongside) the local Tempo/Prometheus/Loki +backends. This is a runtime choice — no xrpld rebuild and no change to the +base stack. xrpld still exports to the local collector exactly as before; +the collector adds one OTLP/HTTP exporter that forwards all three signals to +the Grafana Cloud OTLP gateway, which fans them out to hosted Tempo, Mimir, +and Loki. + +### Credentials + +Find these under **Grafana Cloud → Connections → OpenTelemetry (OTLP)**: + +| Value | Used as | Notes | +| ----------------------------- | ----------------- | -------------------------------------------------- | +| `GRAFANA_CLOUD_OTLP_ENDPOINT` | exporter endpoint | Full gateway URL incl. `/otlp` path | +| `GRAFANA_CLOUD_INSTANCE_ID` | Basic-auth user | Numeric stack/instance id | +| `GRAFANA_CLOUD_API_TOKEN` | Basic-auth pass | Access-policy token with `*:write` for all signals | + +### Enable + +1. Copy the template and fill in the three values: + + ```bash + cp docker/telemetry/.env.grafanacloud.example docker/telemetry/.env.grafanacloud + # edit .env.grafanacloud — this file is gitignored, never commit tokens + ``` + +2. Bring the stack up with the base file **and** the Grafana Cloud override: + + ```bash + docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml up -d + ``` + +To return to local-only export, bring the stack up with just the base +`docker-compose.yml`. + +### Files + +| File | Role | +| ----------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `otel-collector-config.grafanacloud.yaml` | Collector config: local backends **plus** a Grafana Cloud OTLP exporter on all three pipelines | +| `docker-compose.grafanacloud.yaml` | Override that mounts that config and injects the credentials | +| `.env.grafanacloud.example` | Credential template (copy to `.env.grafanacloud`) | + +### Local + cloud vs cloud-only + +The prepared config **dual-exports**: data goes to both the local stack and +Grafana Cloud, so the on-box backends remain a fallback. For cloud-only, +remove the local exporters (`debug`, `otlp/tempo`, `prometheus`, +`otlphttp/loki`) from the respective pipelines in +`otel-collector-config.grafanacloud.yaml`, leaving only +`otlphttp/grafanacloud`. + +> **Note**: shipping logs to Grafana Cloud requires keeping xrpld file +> logging on (at least `warning` level) so the collector's filelog receiver +> has a `debug.log` to tail. Traces and metrics are unaffected by log level. + +### Importing dashboards to Grafana Cloud + +Shipping data (above) is independent of installing the dashboards. The local +stack auto-provisions dashboards from a mounted folder +(`grafana/provisioning/dashboards/dashboards.yaml`, `type: file`); Grafana +Cloud cannot read your filesystem, so its dashboards must be imported over the +HTTP API or the UI. + +The dashboard JSON in `docker/telemetry/grafana/dashboards/` references its +backends through datasource **template variables** (`${DS_PROMETHEUS}`, +`${DS_TEMPO}`) rather than fixed UIDs. On import, Grafana binds each variable +to a datasource of the matching type — auto-selecting it when only one exists +(the usual case: one Mimir, one Tempo). This is what makes the same files work +unchanged on both the local stack and Cloud. + +> Dashboards are parameterized by `grafana/parameterize-datasources.py`. If you +> add a dashboard exported with hardcoded UIDs, re-run that script (idempotent) +> before committing so it stays portable. + +To import: + +1. In Grafana Cloud, go to **Dashboards → New → Import**. +2. Upload a file from `docker/telemetry/grafana/dashboards/` (or paste its + JSON), then click **Load**. +3. At the datasource prompt, confirm the auto-selected **Prometheus/Mimir** + datasource — and **Tempo** for dashboards that query traces — then + **Import**. +4. Repeat per dashboard. Only `consensus-health` uses Tempo; the rest need + only the Prometheus/Mimir datasource. + +## Span Reference + +All spans instrumented in xrpld, grouped by subsystem: + +### RPC Spans + +| Span Name | Source File | Attributes | Description | +| -------------------- | ----------------- | ----------------------------------------------------------- | ----------------------------------------------------- | +| `rpc.http_request` | ServerHandler.cpp | `request_payload_size` | Top-level HTTP RPC request | +| `rpc.ws_upgrade` | ServerHandler.cpp | — | WebSocket upgrade handshake | +| `rpc.ws_message` | ServerHandler.cpp | `command` | WebSocket RPC message | +| `rpc.process` | ServerHandler.cpp | `is_batch`, `batch_size` | RPC processing (child of rpc.http_request/ws_message) | +| `rpc.command.` | RPCHandler.cpp | `command`, `version`, `rpc_role`, `rpc_status`, `load_type` | Per-command span (e.g., `rpc.command.server_info`) | + +### Transaction Spans + +| Span Name | Source File | Attributes | Description | +| --------------- | --------------- | --------------------------------------------------------------------------------- | ------------------------------------- | +| `tx.process` | NetworkOPs.cpp | `tx_hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied` | Transaction submission and processing | +| `tx.receive` | PeerImp.cpp | `peer_id`, `tx_hash`, `tx_type`, `peer_version`, `suppressed`, `tx_status` | Transaction received from peer relay | +| `tx.apply` | BuildLedger.cpp | `ledger_seq`, `tx_count`, `tx_failed` | Transaction set applied per ledger | +| `tx.preflight` | applySteps.cpp | `stage`, `tx_type`, `ter_result` | Stateless checks stage | +| `tx.preclaim` | applySteps.cpp | `stage`, `tx_type`, `ter_result` | Ledger-aware checks stage | +| `tx.transactor` | Transactor.cpp | `stage`, `tx_type`, `ter_result`, `applied` | Apply stage (transactor runs) | + +The three apply-pipeline spans (`tx.preflight`, `tx.preclaim`, `tx.transactor`) +share a deterministic `trace_id` from `txID[0:16]`, so they group under one +trace per transaction. The `stage` attribute (`preflight` / `preclaim` / +`apply`) drives the collector spanmetrics `stage` dimension, giving per-stage +RED metrics on the _Transaction Overview_ dashboard. + +### Transaction Queue Spans + +| Span Name | Source File | Attributes | Description | +| ------------------ | ----------- | -------------------------------------------------------- | -------------------------------------------------- | +| `txq.enqueue` | TxQ.cpp | `tx_hash`, `tx_type` | Transaction enqueue decision (child of tx.process) | +| `txq.apply_direct` | TxQ.cpp | -- | Direct apply attempt (bypassing queue) | +| `txq.batch_clear` | TxQ.cpp | -- | Batch clear of queued transactions for an account | +| `txq.accept` | TxQ.cpp | `queue_size`, `ledger_changed` | Ledger-close accept loop over queued transactions | +| `txq.accept_tx` | TxQ.cpp | `tx_hash`, `retries_remaining`, `ter_code`, `txq_status` | Per-transaction apply during accept | +| `txq.cleanup` | TxQ.cpp | `ledger_seq` | Post-close cleanup of expired queue entries | + +### PathFinding Spans + +| Span Name | Source File | Attributes | Description | +| --------------------- | --------------------------------- | -------------------------------------------------- | ------------------------------------------------------- | +| `pathfind.request` | PathFind.cpp / RipplePathFind.cpp | `pathfind_source_account`, `pathfind_dest_account` | Path-find RPC entry (accounts hashed; set when present) | +| `pathfind.compute` | PathRequest.cpp | `pathfind_fast`, `pathfind_dest_currency` | Path computation for one request (`doUpdate`) | +| `pathfind.discover` | PathRequest.cpp | `pathfind_search_level`, `pathfind_num_paths` | Graph exploration (one per RPC call in `findPaths`) | +| `pathfind.update_all` | PathRequestManager.cpp | `pathfind_ledger_index`, `pathfind_num_requests` | Async recomputation of active requests on ledger close | + +### Consensus Spans + +| Span Name | Source File | Attributes | Description | +| ------------------------------ | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | RCLConsensus.cpp | `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy`, `consensus_round_id` | Root span for a consensus round (deterministic or random trace ID) | +| `consensus.phase.open` | Consensus.h | -- | Open phase duration (child of round) | +| `consensus.proposal.send` | RCLConsensus.cpp | `consensus_round`, `is_bow_out` | Consensus proposal broadcast | +| `consensus.ledger_close` | RCLConsensus.cpp | `ledger_seq`, `consensus_mode` | Ledger close event | +| `consensus.establish` | Consensus.h | `converge_percent`, `establish_count`, `proposers` | Establish phase duration (child of round) | +| `consensus.update_positions` | Consensus.h | `converge_percent`, `proposers`, `disputes_count` | Position update and dispute resolution (see Events below) | +| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, `consensus_stalled`, `establish_count`, `consensus_result` | Consensus threshold check | +| `consensus.accept` | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | Ledger accepted by consensus | +| `consensus.accept.apply` | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below) | +| `consensus.validation.send` | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time` | Validation sent after accept (follows-from link) | +| `consensus.mode_change` | RCLConsensus.cpp | `mode_old`, `mode_new` | Consensus mode transition | +| `consensus.proposal.receive` | PeerImp.cpp | `proposal_trusted`, `consensus_round` | Proposal received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | +| `consensus.validation.receive` | PeerImp.cpp | `validation_trusted`, `ledger_seq` | Validation received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | + +#### Consensus Span Events + +| Parent Span | Event Name | Event Attributes | Description | +| ---------------------------- | ----------------- | ----------------------------------------------------------- | ------------------------------------------------------- | +| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied | +| `consensus.accept.apply` | `tx.included` | `tx_id` | Emitted per transaction included in the accepted ledger | + +#### Close Time Queries (Tempo TraceQL) + +Span attributes are filtered with `span.` inside `{}`. Combine conditions with `&&`. + +``` +# Find rounds where validators disagreed on close time +{name="consensus.accept.apply" && span.close_time_correct = false} + +# Find consensus failures (moved_on) +{name="consensus.accept.apply" && span.consensus_state = "moved_on"} + +# Find slow ledger applications (>5s) +{name="consensus.accept.apply" && duration > 5000ms} + +# Find specific ledger's consensus details +{name="consensus.accept.apply" && span.ledger_seq = 92345678} + +# Find all spans in a consensus round (deterministic trace strategy) +{name="consensus.round" && span.consensus_round_id = ""} + +# Find dispute resolutions +{name="consensus.update_positions"} >> {event:name="dispute.resolve"} +``` + +### Ledger Spans + +| Span Name | Source File | Attributes | Description | +| ----------------- | -------------------- | ------------------------------------- | ----------------------------- | +| `ledger.build` | BuildLedger.cpp:31 | `ledger_seq`, `tx_count`, `tx_failed` | Ledger build during consensus | +| `ledger.validate` | LedgerMaster.cpp:915 | `ledger_seq`, `validations` | Ledger promoted to validated | +| `ledger.store` | LedgerMaster.cpp:409 | `ledger_seq` | Ledger stored in history | + +### Peer Spans + +| Span Name | Source File | Attributes | Description | +| ------------------------- | ---------------- | ------------------------------- | ----------------------------- | +| `peer.proposal.receive` | PeerImp.cpp:1667 | `peer_id`, `proposal_trusted` | Proposal received from peer | +| `peer.validation.receive` | PeerImp.cpp:2264 | `peer_id`, `validation_trusted` | Validation received from peer | + +Both peer receive spans are `kConsumer` inbound entry points started as fresh +trace roots. They never inherit an ambient span left active on the peer thread, +so they do not nest under an unrelated transaction's trace. The distributed +child span that links back to the sending node is the separate +`consensus.*.receive` / `tx.receive` span (see Cross-Node Trace Propagation). + +--- + +## Insights and Sample Queries + +This section shows what questions you can answer using the span attributes, with example Tempo TraceQL queries. + +**TraceQL syntax note:** span attributes must be referenced with the `span.` prefix inside `{}`. +Conditions are combined with `&&`. The `|` pipeline operator is not supported on this Tempo version. + +``` +# General pattern +{name="" && span. = && span. != } + +# Duration filter (no prefix needed) +{name="" && duration > 500ms} + +# Regex match +{name="" && span. =~ ".*"} + +# Multiple span names +{name = "" || name = ""} + +# Name regex +{name =~ ".*" && span. = } + +# Structural: find parent spans that have a matching child/event +{name=""} >> {event:name=""} +``` + +### Transaction Workflow Analysis + +``` +# Find all AMM transactions (AMMDeposit, AMMWithdraw, AMMVote) +{name="tx.process" && span.tx_type =~ "AMM.*"} + +# Find a specific AMM operation +{name="tx.process" && span.tx_type = "AMMDeposit"} +{name="tx.process" && span.tx_type = "AMMWithdraw"} +{name="tx.process" && span.tx_type = "AMMVote"} + +# Find Payment transactions that failed +{name="tx.process" && span.tx_type = "Payment" && span.ter_result != "tesSUCCESS"} + +# Find Payment failures due to path issues +{name="tx.process" && span.tx_type = "Payment" && span.ter_result =~ "tecPATH.*"} + +# Compare latency of different transaction types +{name="tx.process" && span.tx_type = "OfferCreate"} +{name="tx.process" && span.tx_type = "Payment"} + +# Find high-fee transactions (fee > 1 XRP = 1000000 drops) +{name="tx.process" && span.fee > 1000000} + +# Find transactions that were not applied +{name="tx.process" && span.applied = false} + +# Find NFTokenMint across tx and txq spans +{name =~ "tx.*|txq.*" && span.tx_type = "NFTokenMint"} + +# Find all NFT-related activity +{name =~ "tx.*|txq.*" && span.tx_type =~ "NFToken.*"} + +# Find TrustSet transactions (IOU trust lines) +{name="tx.process" && span.tx_type = "TrustSet"} + +# Find oracle price updates +{name="tx.process" && span.tx_type = "OracleSet"} +``` + +### DEX (OfferCreate / OfferCancel) + +``` +# All DEX offer creates +{name="tx.process" && span.tx_type = "OfferCreate"} + +# Offers killed (ImmediateOrCancel/FillOrKill with no fill) +{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecKILLED"} + +# Offers that failed due to insufficient funds +{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecUNFUNDED_OFFER"} + +# Offers failed due to insufficient reserve to place the offer +{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecINSUF_RESERVE_OFFER"} + +# Offer cancellations +{name="tx.process" && span.tx_type = "OfferCancel"} + +# OfferCreate transactions received from peers (cross-node relay) +{name="tx.receive" && span.tx_type = "OfferCreate"} +``` + +### Apply Pipeline by Stage + +``` +# All three stages of one transaction (preflight -> preclaim -> apply) +{name=~"tx.preflight|tx.preclaim|tx.transactor"} + +# Transactions that failed at the preclaim stage +{name="tx.preclaim"} | ter_result != "tesSUCCESS" + +# Transactions that hard-failed preflight (never reached preclaim/apply) +{name="tx.preflight"} | ter_result != "tesSUCCESS" +``` + +PromQL on the span-derived metrics (dashboard: _Transaction Overview_): + +``` +# Per-stage throughput — the funnel preflight >= preclaim >= apply +sum by (stage) (rate(span_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.transactor"}[5m])) + +# Per-stage p95 latency +histogram_quantile(0.95, sum by (le, stage) (rate(span_duration_milliseconds_bucket{span_name=~"tx.preflight|tx.preclaim|tx.transactor"}[5m]))) + +# Per-stage failure rate (ter_result != tesSUCCESS; a failing ter completes the +# span normally, so filter on the attribute, not status_code which only flags exceptions) +sum by (stage) (rate(span_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.transactor", ter_result!~"tesSUCCESS|"}[5m])) +``` + +> **Alerting**: a rising `tx.preflight` / `tx.preclaim` failure rate points to +> malformed or stale-sequence submissions (often spam or a misbehaving client); +> a rising `tx.transactor` failure rate points to apply-time problems. Alert per +> stage rather than on a single aggregate so the failing stage is obvious. + +> **Sampling caveat**: these stage metrics are span-derived and inherit the +> **tracer head-sampling** ratio (`sampling_ratio`). At `sampling_ratio < 1.0` +> they undercount proportionally — treat them as relative trends, not absolute +> transaction counts. Native StatsD metrics are unsampled. + +### Transaction Queue Health + +``` +# Find transactions rejected from the queue +{name="txq.accept_tx" && span.txq_status = "failed"} + +# Find transactions being retried +{name="txq.accept_tx" && span.txq_status = "retried"} + +# Find transactions that exhausted retries +{name="txq.accept_tx" && span.txq_status = "retried" && span.retries_remaining = 0} + +# Which transaction types get queued most often? +{name="txq.enqueue" && span.tx_type = "Payment"} +{name="txq.enqueue" && span.tx_type = "OfferCreate"} +{name="txq.enqueue" && span.tx_type =~ "NFToken.*"} + +# Find ledger closes that applied queued transactions +{name="txq.accept" && span.ledger_changed = true} +``` + +### RPC Debugging + +``` +# Find batch RPC requests +{name="rpc.process" && span.is_batch = true} + +# Find large RPC payloads (>100KB) +{name="rpc.http_request" && span.request_payload_size > 100000} + +# Find resource-heavy RPC commands (by load_type) +{name =~ "rpc.command.*" && span.load_type = "exceptioned RPC"} + +# Find a specific WebSocket command +{name="rpc.ws_message" && span.command = "subscribe"} + +# Find server_info calls +{name="rpc.command.server_info"} + +# Find slow pathfinding with many source assets +{name="pathfind.discover" && span.pathfind_num_source_assets > 10} +``` + +### PathFinding Performance + +``` +# Find pathfinding for specific currencies +{name="pathfind.compute" && span.pathfind_dest_currency = "USD"} + +# Find expensive pathfinding (many source assets to explore) +{name="pathfind.discover" && span.pathfind_num_source_assets > 20} + +# Find slow pathfinding requests +{name="pathfind.compute" && duration > 1000ms} +``` + +### Consensus Health + +``` +# Find rounds where consensus timed out (expired) +{name="consensus.accept" && span.consensus_state = "expired"} + +# Find rounds where we moved on without full agreement +{name="consensus.accept" && span.consensus_state = "moved_on"} + +# Find rounds with many disputes +{name="consensus.accept" && span.disputes_count > 5} + +# Find slow consensus rounds (>5s) +{name="consensus.accept" && span.round_time_ms > 5000} + +# Find bow-out proposals (node resigned from round) +{name="consensus.proposal.send" && span.is_bow_out = true} + +# Correlate validation with its ledger +{name="consensus.validation.send" && span.ledger_hash = ""} + +# Find rounds where validators disagreed on close time +{name="consensus.accept.apply" && span.close_time_correct = false} + +# Find both validation send and receive (compare sender vs receiver latency) +{name = "consensus.validation.send" || name = "consensus.validation.receive"} +``` + +### Cross-Subsystem Correlation + +``` +# Follow a transaction from receive through queue to ledger +{name =~ "tx.*|txq.*" && span.tx_type = "Payment" && duration > 500ms} + +# Find all NFT-related activity across tx and txq spans +{name =~ "tx.*|txq.*" && span.tx_type =~ "NFToken.*"} + +# Find all AMM activity across tx and txq spans +{name =~ "tx.*|txq.*" && span.tx_type =~ "AMM.*"} + +# Find cross-node transaction receives (no errors) +{name="tx.receive" && status != error} +``` + +### Where to Look (Quick Reference) + +| Question | Span | Key Attributes | +| ----------------------------------- | --------------------------- | ---------------------------------------- | +| "Which tx type is slowest?" | `tx.process` | `span.tx_type` + duration | +| "Why was my tx rejected?" | `tx.process` | `span.ter_result`, `span.applied` | +| "What AMM operations happened?" | `tx.process` | `span.tx_type =~ "AMM.*"` | +| "What DEX offers failed?" | `tx.process` | `span.tx_type`, `span.ter_result` | +| "What NFT activity occurred?" | `tx.process`, `txq.enqueue` | `span.tx_type =~ "NFToken.*"` | +| "Is the TxQ backing up?" | `txq.accept` | `span.queue_size`, `span.ledger_changed` | +| "Why was my tx dropped from queue?" | `txq.accept_tx` | `span.txq_status`, `span.ter_code` | +| "Are batch requests a problem?" | `rpc.process` | `span.is_batch`, `span.batch_size` | +| "Which RPC is expensive?" | `rpc.command.*` | `span.load_type`, duration | +| "Did consensus reach threshold?" | `consensus.check` | `span.consensus_result` | +| "Was consensus outcome normal?" | `consensus.accept` | `span.consensus_state` | +| "Did a validator bow out?" | `consensus.proposal.send` | `span.is_bow_out` | +| "Which ledger was validated?" | `consensus.validation.send` | `span.ledger_hash` | +| "Did close time agreement fail?" | `consensus.accept.apply` | `span.close_time_correct` | + +--- + +## Cross-Node Trace Propagation + +xrpld propagates trace context across nodes via protobuf `TraceContext` fields +embedded in peer-to-peer messages. When Node A sends a transaction, proposal, +or validation, it injects its active span's trace/span IDs into the protobuf +message. Node B extracts that context on receipt and creates a child span, +linking the two nodes into a single distributed trace. + +### How It Works + +``` +Node A (sender) Node B (receiver) ++-----------------------------+ +-------------------------------+ +| tx.process / consensus.* | | PeerImp::onMessage() | +| | | | | | +| v | | v | +| SpanGuard::getTraceBytes() | | extract TraceContext from | +| | | | protobuf message | +| v | send | | | +| injectSpanContext() --------|--------->| v | +| sets TraceContext fields | proto | txReceiveSpan() | +| (trace_id, span_id, flags) | msg | proposalReceiveSpan() | ++-----------------------------+ | validationReceiveSpan() | + | | | + | v | + | child span with parent link | + +-------------------------------+ +``` + +### Send-Side Injection + +| Message Type | Injection Point | Mechanism | +| ------------- | -------------------------- | ------------------------------------------ | +| TMTransaction | `NetworkOPs::apply()` | Injects `tx.process` span into relay msg | +| TMProposeSet | `RCLConsensus::propose()` | Injects active context into proposal msg | +| TMValidation | `RCLConsensus::validate()` | Injects active context into validation msg | + +### Receive-Side Extraction + +| Message Type | Extraction Point | Helper Function | +| ------------- | ----------------------------------- | -------------------------------------------------- | +| TMTransaction | `PeerImp::onMessage(TMTransaction)` | `TxTracing::txReceiveSpan()` | +| TMProposeSet | `PeerImp::onMessage(TMProposeSet)` | `ConsensusReceiveTracing::proposalReceiveSpan()` | +| TMValidation | `PeerImp::onMessage(TMValidation)` | `ConsensusReceiveTracing::validationReceiveSpan()` | + +### Key Files + +| File | Role | +| ------------------------------------------------- | ----------------------------------------------- | +| `src/xrpld/telemetry/PropagationHelpers.h` | `injectSpanContext()` — SpanGuard to protobuf | +| `include/xrpl/telemetry/TraceContextPropagator.h` | OTel context <-> protobuf conversion primitives | +| `src/xrpld/telemetry/ConsensusReceiveTracing.h` | Proposal/validation receive span factories | +| `src/xrpld/telemetry/TxTracing.h` | Transaction receive span factory | + +### Backwards Compatibility + +Older peers that do not populate `TraceContext` fields in their messages will +simply produce empty trace bytes on the receive side. The extraction helpers +detect this and create standalone (root) spans instead of child spans. No +errors are logged and no data is lost — the receive span is still created with +all its normal attributes, it just lacks a cross-node parent link. + +### Example Tempo Queries + +``` +# Find cross-node transaction traces (tx.receive spans with no errors) +{name="tx.receive" && status != error} + +# Find proposals received with cross-node parent context +{} >> {name="consensus.proposal.receive"} + +# Trace a transaction across the network by its hash +{name =~ "tx.*" && span.tx_hash = ""} + +# Find all spans in a cross-node consensus trace +{resource.service.name="xrpld" && span.consensus_round_id = ""} + +# Compare latency between sender and receiver for validations +{name = "consensus.validation.send" || name = "consensus.validation.receive"} +``` + +## Prometheus Metrics (Spanmetrics) + +The OTel Collector's spanmetrics connector automatically derives RED (Rate, Errors, Duration) metrics from every span. No custom metrics code is needed in xrpld. + +### Generated Metric Names + +| Prometheus Metric | Type | Description | +| ----------------------------------- | --------- | ---------------------------- | +| `span_calls_total` | Counter | Total span invocations | +| `span_duration_milliseconds_bucket` | Histogram | Latency distribution buckets | +| `span_duration_milliseconds_count` | Histogram | Latency observation count | +| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | + +### Metric Labels + +Every metric carries these standard labels: + +| Label | Source | Example | +| -------------- | ------------------ | ---------------------------------------- | +| `span_name` | Span name | `rpc.command.server_info` | +| `status_code` | Span status | `STATUS_CODE_UNSET`, `STATUS_CODE_ERROR` | +| `service_name` | Resource attribute | `xrpld` | +| `span_kind` | Span kind | `SPAN_KIND_INTERNAL` | + +Additionally, span attributes configured as dimensions in the collector +become metric labels. The span attribute keys are already underscore form +(the naming convention forbids dots), so the label name matches the attribute +name verbatim. Prometheus' dots → underscores sanitization only fires for +dotted attribute names (e.g. resource attributes like `service.name`), which +does not apply to these dimensions. + +| Span Attribute | Metric Label | Applies To | +| -------------------- | -------------------- | ------------------------------- | +| `command` | `command` | `rpc.command.*` spans | +| `rpc_status` | `rpc_status` | `rpc.command.*` spans | +| `consensus_mode` | `consensus_mode` | `consensus.ledger_close` spans | +| `local` | `local` | `tx.process` spans | +| `proposal_trusted` | `proposal_trusted` | `peer.proposal.receive` spans | +| `validation_trusted` | `validation_trusted` | `peer.validation.receive` spans | + +### Histogram Buckets + +Configured in `otel-collector-config.yaml`: + +``` +1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s +``` + +## System Metrics (OTel native -- beast::insight) + +xrpld has a built-in metrics framework (`beast::insight`) that exports metrics natively via OTLP to the OTel Collector. These complement the span-derived RED metrics by providing system-level gauges, counters, and timers that don't map to individual trace spans. + +### Configuration + +Add to `xrpld.cfg`: + +```ini +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +``` + +The `OTelCollector` implementation exports metrics via OTLP/HTTP to the same OTel Collector that receives traces. No separate StatsD receiver is needed. + +> **Fallback**: Set `server=statsd` and `address=127.0.0.1:8125` to use the legacy StatsD UDP path. This requires re-enabling the `statsd` receiver in `otel-collector-config.yaml` and uncommenting port 8125 in `docker-compose.yml`. + +### Metric Reference + +#### Gauges + +| Prometheus Metric | Source | Description | +| ------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h:373 | Age of validated ledger (seconds) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h:374 | Age of published ledger (seconds) | +| `state_accounting_{mode}_duration` | NetworkOPs.cpp:774 | Time in each operating mode (Disconnected/Connected/Syncing/Tracking/Full) | +| `state_accounting_{mode}_transitions` | NetworkOPs.cpp:780 | Transition count per mode | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp:214 | Active inbound peer connections | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp:215 | Active outbound peer connections | +| `overlay_peer_disconnects` | OverlayImpl.h:557 | Peer disconnect count | +| `jobq_job_count` | JobQueue.cpp:26 | Current job queue depth (emitted as `jobq_job_count`: the JobQueue collector is wrapped in `group("jobq")`, so the registered `job_count` gauge gains the `jobq_` prefix) | +| `{category}_bytes_in/out` | OverlayImpl.h:535 | Overlay traffic bytes per category (57 categories) | +| `{category}_messages_in/out` | OverlayImpl.h:535 | Overlay traffic messages per category | + +#### OTel MetricsRegistry Gauges + +These gauges are exported via the OTel Metrics SDK `PeriodicMetricReader` (10s interval), NOT through beast::insight. + +| Prometheus Metric | Source | Description | +| --------------------------------------------------- | ------------------- | -------------------------------------------- | +| `server_info{metric="server_state"}` | MetricsRegistry.cpp | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | MetricsRegistry.cpp | Seconds since server start | +| `server_info{metric="peers"}` | MetricsRegistry.cpp | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | MetricsRegistry.cpp | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | MetricsRegistry.cpp | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | MetricsRegistry.cpp | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | MetricsRegistry.cpp | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | MetricsRegistry.cpp | Last close convergence time (ms) | +| `build_info{version=""}` | MetricsRegistry.cpp | Info-style metric (always 1) | +| `complete_ledgers{bound="start\|end",index=""}` | MetricsRegistry.cpp | Complete ledger range start/end pairs | +| `db_metrics{metric="db_kb_total"}` | MetricsRegistry.cpp | Total database size (KB) | +| `db_metrics{metric="db_kb_ledger"}` | MetricsRegistry.cpp | Ledger database size (KB) | +| `db_metrics{metric="db_kb_transaction"}` | MetricsRegistry.cpp | Transaction database size (KB) | +| `db_metrics{metric="historical_perminute"}` | MetricsRegistry.cpp | Historical ledger fetches per minute | +| `cache_metrics{metric="AL_size"}` | MetricsRegistry.cpp | AcceptedLedger cache size | +| `nodestore_state{metric="node_reads_duration_us"}` | MetricsRegistry.cpp | Cumulative read time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | MetricsRegistry.cpp | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | MetricsRegistry.cpp | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | MetricsRegistry.cpp | Total read threads configured | + +#### Counters + +| Prometheus Metric | Source | Description | +| ------------------------- | --------------------- | ------------------------------ | +| `rpc_requests` | ServerHandler.cpp:108 | Total RPC request count | +| `ledger_fetches` | InboundLedgers.cpp:44 | Ledger fetch request count | +| `ledger_history_mismatch` | LedgerHistory.cpp:16 | Ledger hash mismatch count | +| `warn` | Logic.h:33 | Resource manager warning count | +| `drop` | Logic.h:34 | Resource manager drop count | + +#### Histograms + +| Prometheus Metric | Source | Description | +| ----------------- | --------------------- | ------------------------------ | +| `rpc_time` | ServerHandler.cpp:110 | RPC response time (ms) | +| `rpc_size` | ServerHandler.cpp:109 | RPC response size (bytes) | +| `ios_latency` | Application.cpp:438 | I/O service loop latency (ms) | +| `pathfind_fast` | PathRequests.h:23 | Fast pathfinding duration (ms) | +| `pathfind_full` | PathRequests.h:24 | Full pathfinding duration (ms) | + +## Deployment Tiers + +Multiple xrpld instances can send telemetry to per-tier collectors that all +forward to one Grafana stack. Four resource attributes segregate the data so +one dashboard set serves every deployment: + +| Dimension | Attribute | Set by | Example values | +| ----------- | ------------------------ | ---------- | ------------------------------------------------ | +| Node | `service.instance.id` | xrpld cfg | `alice-laptop`, `ci-runner-7` | +| Service | `service.name` | xrpld cfg | `xrpld`, `xrpld-validator` | +| Network | `xrpl.network.type` | xrpld node | `mainnet`, `testnet`, `devnet`, `perf` | +| Environment | `deployment.environment` | collector | `local`, `test`, `ci`, `prod` | +| Work Item | `xrpl.work.item` | perf-iac | `RIPD-7455` (empty outside perf runs) | +| Branch | `xrpl.branch` | perf-iac | `baseline::`, `test::` | +| Node Role | `xrpl.node.role` | perf-iac | `validator`, `peer` | + +Dashboards expose these as the template variables `$node`, `$service_name`, +`$xrpl_network_type`, `$deployment_environment`, `$xrpl_work_item`, +`$xrpl_branch`, and `$xrpl_node_role` (each variable name matches its +Prometheus label). Select them top-down — work item → branch → node role → +node for a perf comparison run, or environment → network → service → node for +general use. Selecting **All** matches every value, including series lacking +the label, so mixed old/new data never disappears. + +The last three (`$xrpl_work_item`, `$xrpl_branch`, `$xrpl_node_role`) are +populated only during perf-iac comparison runs, which stamp them as resource +attributes from their own alloy pipeline. Outside those runs the labels are +absent; leaving the filters on **All** keeps every dashboard rendering +normally. + +### Who owns which attribute + +- **Node and service** come from xrpld config (`service_instance_id`, + `service_name`). Unique per process. +- **Network** is a property of the chain the node joined; the node derives it + from `[network_id]` and stamps `xrpl.network.type` on all three signals. +- **Environment** is a property of where the collector runs; each collector + serves one environment and stamps it. + +### The upsert vs insert rule + +The collector's `resource/tier` processor uses two actions on purpose: + +- `deployment.environment` → **`upsert`** (overwrite). The collector _is_ the + environment, so it is authoritative. +- `xrpl.network.type` → **`insert`** (fill only if absent). The node knows + its real network, so the collector must not overwrite it — `insert` only + supplies a value when the source did not (e.g. an older xrpld build). This + is what lets a local node connected to mainnet report `network=mainnet`, + not the collector's default. + +### Configuring a collector for a tier + +Each tier runs its own collector. Set the two values in the `resource/tier` +processor of the collector config (`otel-collector-config.yaml` for local +backends, `otel-collector-config.grafanacloud.yaml` for Grafana Cloud): + +```yaml +processors: + resource/tier: + attributes: + - key: deployment.environment + value: # local | test | ci | prod + action: upsert + - key: xrpl.network.type + value: # mainnet | testnet | devnet (fallback only) + action: insert +``` + +Suggested per-tier values: + +| Collector | `deployment.environment` | `xrpl.network.type` (fallback) | +| ------------------- | ------------------------ | ------------------------------ | +| Developer laptop | `local` | `devnet` | +| Test machines | `test` | `testnet` | +| CI runs | `ci` | `testnet` | +| Production observer | `prod` | `mainnet` | + +The `xrpl.network.type` value is only a fallback: when the node stamps its +own network (all current builds do), the node's value wins. Set it to the +network the collector most commonly serves. + +### How the tier labels reach metrics + +Resource attributes do not become Prometheus labels automatically. Two +collector settings make it work, both already enabled: + +- `prometheus.resource_to_telemetry_conversion: enabled: true` promotes + resource attributes to metric labels on the local scrape surface. +- `spanmetrics.resource_metrics_key_attributes` lists the tier attributes so + span-derived series stay grouped per node and tier. + +Traces and logs carry resource attributes natively; Grafana Cloud ingests all +three signals' attributes over OTLP directly. + +## Grafana Dashboards + +Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: + +### RPC Performance (`rpc-performance`) + +| Panel | Type | PromQL | Labels Used | +| --------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| RPC Request Rate by Command | timeseries | `sum by (command) (rate(span_calls_total{span_name=~"rpc.command.*"}[5m]))` | `command` | +| RPC Latency p95 by Command | timeseries | `histogram_quantile(0.95, sum by (le, command) (rate(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])))` | `command` | +| RPC Error Rate | bargauge | Error spans / total spans × 100, grouped by `command` | `command`, `status_code` | +| RPC Latency Heatmap | heatmap | `sum(increase(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])) by (le)` | `le` (bucket boundaries) | +| Overall RPC Throughput | timeseries | `rpc.request` + `rpc.process` rate | — | +| RPC Success vs Error | timeseries | by `status_code` (UNSET vs ERROR) | `status_code` | +| Top Commands by Volume | bargauge | `topk(10, ...)` by `command` | `command` | +| WebSocket Message Rate | stat | `rpc.ws_message` rate | — | + +### Transaction Overview (`transaction-overview`) + +| Panel | Type | PromQL | Labels Used | +| ---------------------------------- | -------------- | -------------------------------------------------------------------------------------------- | ----------------------------------- | +| Transaction Processing Rate | timeseries | `rate(span_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | +| Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | +| Transaction Path Distribution | piechart | `sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m]))` | `local` | +| Transaction Receive vs Suppressed | timeseries | `rate(span_calls_total{span_name="tx.receive"}[5m])` | — | +| TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | +| TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | +| Peer TX Receive Rate | timeseries | `tx.receive` rate | — | +| TX Apply Failed Rate | stat | `rate(span_calls_total{span_name="tx.transactor",stage="apply",ter_result!~"tesSUCCESS\|"})` | `stage`, `ter_result` | +| TxQ Accept: Applied Ratio per Node | state-timeline | applied / (applied+failed) of `span_calls_total{span_name="txq.accept_tx"}` per node | `txq_status`, `service_instance_id` | + +### Consensus Health (`consensus-health`) + +| Panel | Type | PromQL | Labels Used | +| ----------------------------- | ---------- | --------------------------------------------------------------------------- | ---------------- | +| Consensus Round Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept"})` | — | +| Consensus Proposals Sent Rate | timeseries | `rate(span_calls_total{span_name="consensus.proposal.send"}[5m])` | — | +| Ledger Close Duration | timeseries | `histogram_quantile(0.95, ... {span_name="consensus.ledger_close"})` | — | +| Validation Send Rate | stat | `rate(span_calls_total{span_name="consensus.validation.send"}[5m])` | — | +| Ledger Apply Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept.apply"})` | — | +| Close Time Agreement | timeseries | `rate(span_calls_total{span_name="consensus.accept.apply"}[5m])` | — | +| Consensus Mode Over Time | timeseries | `consensus.ledger_close` by `consensus_mode` | `consensus_mode` | +| Accept vs Close Rate | timeseries | `consensus.accept` vs `consensus.ledger_close` rate | — | +| Validation vs Close Rate | timeseries | `consensus.validation.send` vs `consensus.ledger_close` | — | +| Accept Duration Heatmap | heatmap | `consensus.accept` histogram buckets | `le` | + +### Ledger Operations (`ledger-operations`) + +| Panel | Type | PromQL | Labels Used | +| ----------------------- | ---------- | ---------------------------------------------- | ----------- | +| Ledger Build Rate | stat | `ledger.build` call rate | — | +| Ledger Build Duration | timeseries | p95/p50 of `ledger.build` | — | +| Ledger Validation Rate | stat | `ledger.validate` call rate | — | +| Build Duration Heatmap | heatmap | `ledger.build` histogram buckets | `le` | +| TX Apply Duration | timeseries | p95/p50 of `tx.apply` | — | +| TX Apply Rate | timeseries | `tx.apply` call rate | — | +| Ledger Store Rate | stat | `ledger.store` call rate | — | +| Build vs Close Duration | timeseries | p95 `ledger.build` vs `consensus.ledger_close` | — | + +### Peer Network (`peer-network`) + +Requires `trace_peer=1` in the `[telemetry]` config section. + +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | ------------------------------ | -------------------- | +| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | +| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | +| Proposals Trusted vs Untrusted | piechart | by `proposal_trusted` | `proposal_trusted` | +| Validations Trusted vs Untrusted | piechart | by `validation_trusted` | `validation_trusted` | + +### Node Health -- System Metrics (`node-health`) + +| Panel | Type | PromQL | Labels Used | +| -------------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `jobq_job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Server State | stat | `server_info{metric="server_state"}` | `metric` | +| Uptime | stat | `server_info{metric="uptime"}` | `metric` | +| Peer Count | stat | `server_info{metric="peers"}` | `metric` | +| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | +| Build Version | stat | `build_info` | `version` | +| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | +| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | +| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | + +### Network Traffic -- System Metrics (`network-traffic`) + +| Panel | Type | PromQL | Labels Used | +| ------------------------------------ | ---------- | -------------------------------------------------------------------------------------------------------------------------- | ----------- | +| Active Peers | timeseries | `peer_finder_active_*_peers` | — | +| Peer Disconnects | timeseries | `increase(overlay_peer_disconnects[$__rate_interval])` | — | +| Total Network Bytes | timeseries | `rate(total_bytes_in/out[$__rate_interval])` | — | +| Total Network Messages | timeseries | `rate(total_messages_in/out[$__rate_interval])` | — | +| Transaction Traffic | timeseries | `rate(transactions_messages_in/out[$__rate_interval])` | — | +| Proposal Traffic | timeseries | `rate(proposals_messages_in/out[$__rate_interval])` | — | +| Validation Traffic | timeseries | `rate(validations_messages_in/out[$__rate_interval])` | — | +| Traffic by Category | bargauge | `topk(10, label_replace(sum by (service_instance_id)(rate([$__rate_interval])),"__name__","","","") or …)` | — | +| Duplicate Traffic (Wasted Bandwidth) | timeseries | `rate(*_duplicate_bytes_in/out[$__rate_interval])` | — | +| All Traffic Categories (Detail) | timeseries | `topk(15, label_replace(sum by (service_instance_id)(rate([$__rate_interval])),"__name__","","","") or …)` | — | + +> **Why the per-category panels enumerate each metric.** A bare +> `rate({__name__=~".*_bytes_in"}[…])` fails on Mimir/Cloud with _"vector +> cannot contain metrics with the same labelset"_: `rate()` drops the +> `__name__` label, so the many matched counters collapse to identical +> labelsets. Wrapping in `sum by (__name__, …)` does **not** help (the inner +> vector is rejected before the outer `sum`). The working form enumerates each +> `*_bytes_in` metric and re-attaches its name with `label_replace(..., +"__name__", "", "", "")`, so the existing `{{__name__}}` legend and +> the per-series display-name overrides keep working. + +### RPC & Pathfinding -- System Metrics (`rpc-pathfinding`) + +| Panel | Type | PromQL | Labels Used | +| ------------------------- | ---------- | ------------------------------------------------ | ----------- | +| RPC Request Rate | stat | `rate(rpc_requests[5m])` | — | +| RPC Response Time | timeseries | `histogram_quantile(0.95, rpc_time_bucket)` | — | +| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_bucket)` | — | +| RPC Response Time Heatmap | heatmap | `rpc_time_bucket` | — | +| Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | — | +| Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | — | +| Resource Warnings Rate | stat | `rate(warn[5m])` | — | +| Resource Drops Rate | stat | `rate(drop[5m])` | — | + +### Span → Metric → Dashboard Summary + +| Span Name | Prometheus Metric Filter | Grafana Dashboard | +| ------------------------------ | -------------------------------------------- | --------------------------------------------- | +| `rpc.http_request` | `{span_name="rpc.http_request"}` | RPC Performance (Overall Throughput) | +| `rpc.ws_upgrade` | `{span_name="rpc.ws_upgrade"}` | -- (available but not paneled) | +| `rpc.ws_message` | `{span_name="rpc.ws_message"}` | RPC Performance (WebSocket Rate) | +| `rpc.process` | `{span_name="rpc.process"}` | RPC Performance (Overall Throughput) | +| `rpc.command.*` | `{span_name=~"rpc.command.*"}` | RPC Performance (Rate, Latency, Error, Top) | +| `tx.process` | `{span_name="tx.process"}` | Transaction Overview (Rate, Latency, Heatmap) | +| `tx.receive` | `{span_name="tx.receive"}` | Transaction Overview (Rate, Receive) | +| `tx.apply` | `{span_name="tx.apply"}` | Transaction Overview + Ledger Ops (Apply) | +| `txq.enqueue` | `{span_name="txq.enqueue"}` | -- (available but not paneled) | +| `txq.apply_direct` | `{span_name="txq.apply_direct"}` | -- (available but not paneled) | +| `txq.batch_clear` | `{span_name="txq.batch_clear"}` | -- (available but not paneled) | +| `txq.accept` | `{span_name="txq.accept"}` | -- (available but not paneled) | +| `txq.accept_tx` | `{span_name="txq.accept_tx"}` | -- (available but not paneled) | +| `txq.cleanup` | `{span_name="txq.cleanup"}` | -- (available but not paneled) | +| `consensus.round` | `{span_name="consensus.round"}` | -- (available but not paneled) | +| `consensus.phase.open` | `{span_name="consensus.phase.open"}` | -- (available but not paneled) | +| `consensus.establish` | `{span_name="consensus.establish"}` | -- (available but not paneled) | +| `consensus.update_positions` | `{span_name="consensus.update_positions"}` | -- (available but not paneled) | +| `consensus.check` | `{span_name="consensus.check"}` | -- (available but not paneled) | +| `consensus.accept` | `{span_name="consensus.accept"}` | Consensus Health (Duration, Rate, Heatmap) | +| `consensus.proposal.send` | `{span_name="consensus.proposal.send"}` | Consensus Health (Proposals Rate) | +| `consensus.ledger_close` | `{span_name="consensus.ledger_close"}` | Consensus Health (Close, Mode) | +| `consensus.validation.send` | `{span_name="consensus.validation.send"}` | Consensus Health (Validation Rate) | +| `consensus.accept.apply` | `{span_name="consensus.accept.apply"}` | Consensus Health (Apply Duration, Close Time) | +| `consensus.mode_change` | `{span_name="consensus.mode_change"}` | -- (available but not paneled) | +| `consensus.proposal.receive` | `{span_name="consensus.proposal.receive"}` | -- (available but not paneled) | +| `consensus.validation.receive` | `{span_name="consensus.validation.receive"}` | -- (available but not paneled) | +| `ledger.build` | `{span_name="ledger.build"}` | Ledger Ops (Build Rate, Duration, Heatmap) | +| `ledger.validate` | `{span_name="ledger.validate"}` | Ledger Ops (Validation Rate) | +| `ledger.store` | `{span_name="ledger.store"}` | Ledger Ops (Store Rate) | +| `peer.proposal.receive` | `{span_name="peer.proposal.receive"}` | Peer Network (Rate, Trusted/Untrusted) | +| `peer.validation.receive` | `{span_name="peer.validation.receive"}` | Peer Network (Rate, Trusted/Untrusted) | + +## Alerting + +xrpld provisions six Grafana alert rules on the health-critical metrics, so a +stock stack alerts out of the box with no UI setup. Rules are provisioned from +`docker/telemetry/grafana/provisioning/alerting/` and load automatically when +the Grafana container starts. They appear under **Alerting → Alert rules**, +folder **xrpld**. + +### Alert catalogue + +All rules evaluate every minute against the Prometheus datasource, over a +5-minute window, and group by `service_instance_id` so each node alerts on its +own. Alerts fire only after the condition holds for the `for` dwell time. + +| Alert | Severity | Fires when | For | +| ----------------------- | -------- | ----------------------------------------- | --- | +| `LedgerHistoryMismatch` | critical | `rate(ledger_history_mismatch_total)` > 0 | 5m | +| `LedgerCloseStalled` | critical | `rate(ledgers_closed_total)` ≈ 0 | 3m | +| `ValidationsMissed` | warning | `rate(validation_missed_total)` > 0 | 5m | +| `ValidationsNotChecked` | warning | `rate(validations_checked_total)` ≈ 0 | 5m | +| `JobQueueTxOverflow` | warning | `rate(jq_trans_overflow_total)` > 0 | 5m | +| `JobQueueLatencyHigh` | warning | p99 `job_queued_us` > 1s | 5m | + +#### Consensus / ledger health + +**LedgerHistoryMismatch** — The node closed a ledger whose history diverges +from the validated network chain. Likely causes: corrupted local state, a bug, +or a node that fell out of sync and rebuilt incorrectly. Investigate the node's +ledger acquisition logs; a healthy node never mismatches. + +**LedgerCloseStalled** — No ledgers closed for 3 minutes. A healthy node closes +one every ~3-5s. Likely causes: lost peer connectivity, consensus stall, or the +process is hung. This rule also fires on _NoData_ — if the series disappears the +node is likely down. Check peer count and process health first. + +#### Validator health + +**ValidationsMissed** — This validator's validations are not agreeing with the +validated ledger. Sustained misses risk removal from UNLs. Check clock sync, +peer connectivity, and whether the node is keeping up with ledger close. + +**ValidationsNotChecked** — The node has stopped checking incoming validations +from peers. Likely causes: overlay/peer disconnection or a stalled validation +pipeline. Fires on NoData as well. + +#### Job queue / resource health + +**JobQueueTxOverflow** — The transaction job queue is full and transactions are +being dropped. The node is shedding load it cannot process. Check CPU, the +`JobQueueLatencyHigh` alert, and offered load. + +**JobQueueLatencyHigh** — p99 queue wait exceeds 1 second, i.e. jobs back up +before running. The node is saturated. Correlate with CPU and the Job Queue +dashboard. + +### Tuning thresholds + +Thresholds live in +`docker/telemetry/grafana/provisioning/alerting/rules.yaml` as the `params` +array of each rule's `C` (threshold) node. Common tunables: + +- **`JobQueueLatencyHigh`** — `params: [1000000]` is 1 000 000 µs (1s). Lower + it for latency-sensitive deployments. +- **`LedgerCloseStalled` / `ValidationsNotChecked`** — use `lt` with a tiny + epsilon (`0.001`) rather than `0`, so floating-point rate noise near zero + does not suppress the alert. + +Edit the file and restart the Grafana container to reload: + +```bash +docker compose -f docker/telemetry/docker-compose.yml restart grafana +``` + +### Sending alerts somewhere real + +Two contact points are provisioned in +`docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml`: + +| Contact point | Receivers | Gets | +| ---------------- | ------------- | ------------------------ | +| `xrpld-default` | Slack | warning-severity alerts | +| `xrpld-critical` | Slack + email | critical-severity alerts | + +The severity split lives in +`docker/telemetry/grafana/provisioning/alerting/policies.yaml`: the root route +sends everything to `xrpld-default`, and a child route matching +`severity = critical` overrides to `xrpld-critical`. So a critical alert goes +to Slack **and** email; a warning goes to Slack only. Both group by +`alertname` + `service_instance_id`; critical alerts re-page hourly vs the 4h default. + +#### Configure delivery (no secrets in git) + +The Slack webhook and email address are **not** hard-coded — the YAML +references `${SLACK_WEBHOOK_URL}` and `${ALERT_EMAIL_TO}`, which Grafana +expands from the environment at startup. Supply them through a gitignored +env file: + +```bash +cp docker/telemetry/.env.alerting.example docker/telemetry/.env.alerting +# edit .env.alerting — this file is gitignored, never commit the webhook/address +docker compose -f docker/telemetry/docker-compose.yml up -d grafana +``` + +- **Slack** — set `SLACK_WEBHOOK_URL` to an incoming-webhook URL. Drives both + tiers. +- **Email** — set `ALERT_EMAIL_TO` (comma-separated) **and** point the + `GF_SMTP_*` vars at a real relay with `GF_SMTP_ENABLED=true`. Grafana can + only send mail once SMTP is configured. + +Any variable left blank disables that path; the stack still runs. To add a +third destination (PagerDuty, Opsgenie, a custom webhook), add a receiver to +the relevant contact point. + +### Verifying alert provisioning loaded + +After the stack is up: + +```bash +# All six rules present? +curl -s http://localhost:3000/api/v1/provisioning/alert-rules | jq '.[].title' + +# Contact points present? +curl -s http://localhost:3000/api/v1/provisioning/contact-points | jq '.[].name' +``` + +Grafana logs a provisioning error and skips the file if the YAML is malformed: + +```bash +docker compose -f docker/telemetry/docker-compose.yml logs grafana | grep -i alerting +``` + +## Log-Trace Correlation + +When xrpld is built with `telemetry=ON`, log lines emitted within an active OpenTelemetry span automatically include `trace_id` and `span_id` fields: + +``` +2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +``` + +This enables bidirectional navigation between logs and traces in Grafana: + +- **Tempo -> Loki**: Click "Logs for this trace" on any trace in Grafana Tempo to see all log lines from that trace. +- **Loki -> Tempo**: Click the `TraceID` derived field link on any log line containing `trace_id=` to jump to the full trace in Tempo. + +### Log Ingestion Pipeline + +Log files are ingested by the OTel Collector's `filelog` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. + +The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. + +### LogQL Query Examples + +The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrpld"`). + +```logql +# Find all logs for a specific trace +{service_name="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" + +# Error logs with trace context (log lines with ERR severity that have a trace_id) +{service_name="xrpld"} |= "ERR" |= "trace_id=" + +# All logs from a specific partition that were emitted during a span +{service_name="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P[a-f0-9]+)` | trace_id != "" + +# Logs from a specific subsystem during a span (e.g. LedgerConsensus) +{service_name="xrpld"} |= "LedgerConsensus" |= "trace_id=" + +# Logs from the last hour containing trace context +{service_name="xrpld"} |= "trace_id=" | regexp `(?P\S+):(?P\S+)\s+trace_id=(?P[a-f0-9]+)` + +# Count of traced vs untraced log lines +count_over_time({service_name="xrpld"} |= "trace_id=" [5m]) +``` + +### Verifying Log Correlation + +1. Start the observability stack and xrpld with telemetry enabled. +2. Send an RPC request: `curl http://localhost:5005 -d '{"method":"server_info"}'` +3. Check the debug.log for `trace_id=` entries: `grep trace_id= /path/to/debug.log` +4. Open Grafana at http://localhost:3000 -> Explore -> Loki and search for `{service_name="xrpld"} |= "trace_id="`. +5. Click the TraceID link to navigate to the corresponding trace in Tempo. + +## Troubleshooting + +### No traces appearing in Tempo + +1. Check xrpld logs for `Telemetry starting` message +2. Verify `enabled=1` in the `[telemetry]` config section +3. Test collector connectivity: `curl -v http://localhost:4318/v1/traces` +4. Check collector logs: `docker compose -f docker/telemetry/docker-compose.yml logs otel-collector` +5. Verify Tempo is receiving data: open Grafana → Explore → select Tempo datasource → search by `service.name = xrpld` +6. Check Tempo logs: `docker compose -f docker/telemetry/docker-compose.yml logs tempo` + +### No system metrics in Prometheus + +1. Check xrpld logs for `OTelCollector starting` message +2. Verify `server=otel` in the `[insight]` config section +3. Verify the endpoint in `[insight]` points to the OTLP/HTTP port (default: `http://localhost:4318/v1/metrics`) +4. Check that the `otlp` receiver is in the metrics pipeline receivers in `otel-collector-config.yaml` +5. Query Prometheus directly: `curl 'http://localhost:9090/api/v1/query?query=jobq_job_count'` + +### Server info gauge shows server_state=0 + +This is normal during startup. The server starts in DISCONNECTED mode (0) and +progresses through CONNECTED (1), SYNCING (2), TRACKING (3), to FULL (4). +Wait for the node to sync with the network. + +### Database metrics showing zero + +The `getKBUsed*()` methods require SQLite databases to exist. If running with +`--standalone` or before the first ledger is stored, these will be zero. + +### High memory usage + +- Reduce trace volume with collector-side tail sampling (xrpld head sampling is + fixed at 1.0 and is not configurable) +- Reduce `max_queue_size` and `batch_size` +- Disable high-volume trace categories: `trace_peer=0` + +### Collector connection failures + +- Verify endpoint URL matches collector address +- Check firewall rules for ports 4317/4318 +- If using TLS, verify certificate path with `tls_ca_cert` + +### No trace_id in log output + +- Verify xrpld was built with `telemetry=ON` (the `XRPL_ENABLE_TELEMETRY` preprocessor flag) +- Verify `enabled=1` in the `[telemetry]` config section +- Log lines only contain `trace_id`/`span_id` when emitted inside an active span — background logs outside of RPC/consensus/transaction processing will not have trace context +- Check that the specific trace category is enabled (e.g., `trace_rpc=1`) + +### No logs in Loki + +- Verify the log file mount in docker-compose.yml points to the correct xrpld log directory (default source `docker/telemetry/data/logs`, or the `XRPLD_LOG_DIR` override) and that xrpld actually writes `debug.log` there +- Check OTel Collector logs for filelog receiver errors: `docker compose logs otel-collector` +- Verify Loki is running: `curl http://localhost:3100/ready` +- Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root + +## Performance Tuning + +| Scenario | Recommendation | +| ------------------------ | --------------------------------------------------------- | +| Production mainnet | `trace_peer=0`; reduce volume via collector tail sampling | +| Testnet/devnet | Full tracing (head sampling fixed at 1.0) | +| Debugging specific issue | Full tracing (head sampling fixed at 1.0) | +| High-throughput node | Increase `batch_size=1024`, `max_queue_size=4096` | + +## Disabling Telemetry + +Set `enabled=0` in config (runtime disable) or build without the flag: + +```bash +cmake --preset default -Dtelemetry=OFF +``` + +When telemetry is compiled out, all trace macros expand to no-ops with zero overhead. + +## Validating Telemetry Stack + +After deploying telemetry, use the Phase 10 workload tools to validate the full stack end-to-end. + +### Quick Validation + +```bash +# Run the full validation suite (starts cluster, generates load, validates): +docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld + +# Check the report: +cat /tmp/xrpld-validation/reports/validation-report.json | jq '.summary' +``` + +### What Gets Validated + +| Category | Checks | Description | +| ---------- | -------------- | ------------------------------------------------------- | +| Spans | 16+ span types | All span names appear in Tempo with required attributes | +| Metrics | 30+ metrics | SpanMetrics, StatsD gauges/counters, Phase 9 metrics | +| Logs | 2 checks | trace_id/span_id present in Loki, cross-reference works | +| Dashboards | 10 dashboards | All Grafana dashboards load without errors | + +### Running Individual Tools + +```bash +# RPC load only: +python3 docker/telemetry/workload/rpc_load_generator.py \ + --endpoints ws://localhost:6006 --rate 50 --duration 120 + +# Transaction mix only: +python3 docker/telemetry/workload/tx_submitter.py \ + --endpoint ws://localhost:6006 --tps 5 --duration 120 + +# Validation only (assumes load already ran): +python3 docker/telemetry/workload/validate_telemetry.py \ + --report /tmp/report.json +``` + +### Interpreting Failures + +- **Span failures**: Check that the relevant trace category is enabled in `[telemetry]` config (e.g., `trace_rpc=1`). +- **Metric failures**: Verify the OTel Collector is running and Prometheus is scraping port 8889. Check `docker compose logs otel-collector`. +- **Dashboard failures**: Ensure Grafana provisioning is mounted correctly. Check `docker compose logs grafana`. + +## Performance Benchmarking + +Measure the overhead of the telemetry stack against a baseline: + +```bash +docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300 +``` + +### Benchmark Thresholds + +| Metric | Target | Description | +| ----------------- | ------ | -------------------------------------- | +| CPU overhead | < 3% | Average CPU increase across nodes | +| Memory overhead | < 5MB | Peak RSS increase per node | +| RPC p99 latency | < 2ms | Additional p99 latency for server_info | +| Throughput impact | < 5% | Reduction in ledger close rate | +| Consensus impact | < 1% | Increase in consensus round time | + +### Tuning for Production + +If benchmarks exceed thresholds: + +1. **Reduce sampling**: `sampling_ratio=0.01` (1% of traces) +2. **Disable peer tracing**: `trace_peer=0` (highest volume category) +3. **Increase batch delay**: `batch_delay_ms=10000` (less frequent exports) +4. **Reduce queue size**: `max_queue_size=1024` (back-pressure earlier) + +See `docker/telemetry/workload/README.md` for full documentation. diff --git a/include/xrpl/beast/insight/OTelCollector.h b/include/xrpl/beast/insight/OTelCollector.h new file mode 100644 index 00000000000..e043531f997 --- /dev/null +++ b/include/xrpl/beast/insight/OTelCollector.h @@ -0,0 +1,135 @@ +#pragma once + +/** + * @file OTelCollector.h + * @brief OpenTelemetry-based implementation of the beast::insight::Collector + * interface for native OTLP metric export. + * + * When XRPL_ENABLE_TELEMETRY is defined, OTelCollector maps each + * beast::insight instrument type (Counter, Gauge, Event, Meter, Hook) to + * the corresponding OpenTelemetry Metrics SDK instrument and exports + * them via OTLP/HTTP to an OpenTelemetry Collector. + * + * When XRPL_ENABLE_TELEMETRY is NOT defined, OTelCollector::New() returns + * a NullCollector so the binary compiles without OTel dependencies. + * + * Dependency diagram: + * + * +-----------------+ +-------------------+ + * | Collector (ABC) |<----| OTelCollector | + * +-----------------+ | (public header) | + * ^ +-------------------+ + * | | + * +-----------------+ +-------------------+ + * | NullCollector | | OTelCollectorImp | + * | (fallback when | | (impl in .cpp, | + * | no telemetry) | | uses OTel SDK) | + * +-----------------+ +-------------------+ + * | + * +-------------------+ + * | OTel Metrics SDK | + * | MeterProvider | + * | OTLP HTTP Metric | + * | Exporter | + * +-------------------+ + */ + +#include +#include + +#include +#include + +namespace beast::insight { + +/** + * @brief A Collector that exports metrics via OpenTelemetry OTLP/HTTP. + * + * Replaces StatsD-based metric collection with native OTel Metrics SDK + * instruments. Each beast::insight instrument maps to an OTel equivalent: + * + * - Counter -> OTel Counter + * - Gauge -> OTel ObservableGauge (async callback) + * - Event -> OTel Histogram (duration in milliseconds) + * - Meter -> OTel Counter (monotonic, unsigned) + * - Hook -> Called by PeriodicMetricReader at collection time + * + * Example — primary use (create the collector and record a metric): + * @code + * auto collector = beast::insight::OTelCollector::New( + * "http://localhost:4318/v1/metrics", // OTLP/HTTP endpoint + * "xrpld", // metric name prefix + * "node-1", // service.instance.id + * "xrpld", // service.name + * "mainnet", // xrpl.network.type + * journal); + * + * auto counter = collector->makeCounter("ledgers", "closed"); + * ++counter; // exported on the next PeriodicMetricReader tick + * @endcode + * + * Example — edge case (telemetry disabled at compile time): New() + * returns a NullCollector, so callers need no #ifdef guard. The same + * instruments compile and run, but recording is a no-op. + * @code + * auto collector = beast::insight::OTelCollector::New( + * endpoint, prefix, instanceId, serviceName, networkType, journal); + * auto gauge = collector->makeGauge("peers", "count"); + * gauge = 42; // silently discarded when XRPL_ENABLE_TELEMETRY is off + * @endcode + * + * @note Thread safety: instrument recording (Counter::Add, + * Histogram::Record, and atomic gauge writes) is thread-safe and + * may be called concurrently. Instrument *creation* (make_counter, + * make_gauge, etc.) is serialized by an internal mutex. Hook and + * observable-gauge callbacks run on the SDK's collection thread, so + * any state they read must itself be thread-safe. + * @note Limitations: metrics export over OTLP/HTTP only (no gRPC); the + * PeriodicMetricReader interval is fixed at 1s; and gauge values + * are stored as int64_t, so fractional gauges are truncated. + * + * @see StatsDCollector for the StatsD-based alternative. + * @see NullCollector for the no-op fallback. + */ +class OTelCollector : public Collector +{ +public: + explicit OTelCollector() = default; + + /** + * @brief Factory method to create an OTelCollector instance. + * + * When XRPL_ENABLE_TELEMETRY is defined, creates a real OTel-backed + * collector that exports metrics via OTLP/HTTP. When telemetry is + * disabled at compile time, returns a NullCollector. + * + * @param endpoint OTLP/HTTP metrics endpoint URL + * (e.g. "http://localhost:4318/v1/metrics"). + * @param prefix Prefix prepended to all metric names + * (e.g. "xrpld"). + * @param instanceId Unique identifier for this node instance, + * emitted as the `service.instance.id` OTel + * resource attribute. Defaults to empty string + * (attribute omitted when empty). + * @param serviceName Value for the `service.name` OTel resource + * attribute. When empty, defaults to "xrpld". + * Matches the trace exporter's service.name so + * metrics and traces share one service identity. + * @param networkType Value for the `xrpl.network.type` OTel resource + * attribute (e.g. "mainnet"). When empty, the + * attribute is omitted. Supplied by the node so + * metrics carry the same network label as traces. + * @param journal Journal for logging. + * @return Shared pointer to the created Collector. + */ + static std::shared_ptr + // NOLINTNEXTLINE(readability-identifier-naming) + New(std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal); +}; + +} // namespace beast::insight diff --git a/include/xrpl/core/ServiceRegistry.h b/include/xrpl/core/ServiceRegistry.h index 592964134b0..3dca3c46358 100644 --- a/include/xrpl/core/ServiceRegistry.h +++ b/include/xrpl/core/ServiceRegistry.h @@ -24,6 +24,10 @@ class Manager; namespace perf { class PerfLog; } // namespace perf +namespace telemetry { +class Telemetry; +class MetricsRegistry; +} // namespace telemetry // This is temporary until we migrate all code to use ServiceRegistry. class Application; @@ -224,6 +228,16 @@ class ServiceRegistry virtual perf::PerfLog& getPerfLog() = 0; + virtual telemetry::Telemetry& + getTelemetry() = 0; + + /** + * Return the MetricsRegistry, or nullptr if telemetry is disabled. + * Used by PerfLog and other hot paths to record OTel metrics. + */ + virtual telemetry::MetricsRegistry* + getMetricsRegistry() = 0; + // Configuration and state [[nodiscard]] virtual bool isStopping() const = 0; diff --git a/include/xrpl/proto/xrpl.proto b/include/xrpl/proto/xrpl.proto index d49920201ed..c2b28c87dda 100644 --- a/include/xrpl/proto/xrpl.proto +++ b/include/xrpl/proto/xrpl.proto @@ -85,6 +85,30 @@ message TMPublicKey { // If you want to send an amount that is greater than any single address of yours // you must first combine coins from one address to another. +// Trace context for OpenTelemetry distributed tracing across nodes. +// Uses W3C Trace Context format internally. +// +// Field numbering note: this message is embedded as field 1001 on +// TMTransaction, TMProposeSet, and TMValidation. Field numbers >= 1000 +// are reserved for optional, observability-only additions that must not +// collide with protocol-semantic fields (which historically use 1-99). +// Older peers that do not understand field 1001 will simply ignore it +// per protobuf wire-format rules, preserving backwards compatibility. +// +// trace_state is reserved for future use (secure tracing pipeline, +// OpenTelemetryPlan/secure-OTel.md). It is currently neither populated +// on inject nor read on extract; consumers must not rely on it. +message TraceContext { + optional bytes trace_id = 1; // 16-byte trace identifier + optional bytes span_id = 2; // 8-byte parent span identifier + optional uint32 trace_flags = 3; // bit 0 = sampled + // TODO: trace_state is reserved for W3C tracestate vendor-specific + // key-value pairs but is not yet read or written by + // TraceContextPropagator. Wire it when cross-vendor trace + // propagation is needed. + optional string trace_state = 4; // RESERVED — see TraceContext header note +} + enum TransactionStatus { tsNEW = 1; // origin node did/could not validate tsCURRENT = 2; // scheduled to go in this ledger @@ -101,6 +125,9 @@ message TMTransaction { required TransactionStatus status = 2; optional uint64 receiveTimestamp = 3; optional bool deferred = 4; // not applied to open ledger + + // Optional trace context for OpenTelemetry distributed tracing + optional TraceContext trace_context = 1001; } message TMTransactions { @@ -149,6 +176,9 @@ message TMProposeSet { // Number of hops traveled optional uint32 hops = 12 [deprecated = true]; + + // Optional trace context for OpenTelemetry distributed tracing + optional TraceContext trace_context = 1001; } enum TxSetStatus { @@ -194,6 +224,9 @@ message TMValidation { // Number of hops traveled optional uint32 hops = 3 [deprecated = true]; + + // Optional trace context for OpenTelemetry distributed tracing + optional TraceContext trace_context = 1001; } // An array of Endpoint messages diff --git a/include/xrpl/server/NetworkOPs.h b/include/xrpl/server/NetworkOPs.h index d3d3bb9f70f..4839b5f1f31 100644 --- a/include/xrpl/server/NetworkOPs.h +++ b/include/xrpl/server/NetworkOPs.h @@ -103,6 +103,14 @@ class NetworkOPs : public InfoSub::Source [[nodiscard]] virtual OperatingMode getOperatingMode() const = 0; + /** + * Time spent in the current operating mode so far, in microseconds. + * Same value as `server_state_duration_us` in server_info; exposed as a + * lightweight accessor so metrics can read it without building the full + * server_info JSON on every collection tick. + */ + [[nodiscard]] virtual std::chrono::microseconds + getServerStateDurationUs() const = 0; [[nodiscard]] virtual std::string strOperatingMode(OperatingMode const mode, bool const admin = false) const = 0; [[nodiscard]] virtual std::string diff --git a/include/xrpl/telemetry/DiscardFlag.h b/include/xrpl/telemetry/DiscardFlag.h new file mode 100644 index 00000000000..1bd1f668bb8 --- /dev/null +++ b/include/xrpl/telemetry/DiscardFlag.h @@ -0,0 +1,95 @@ +#pragma once + +/** + * Thread-local discard signaling between SpanGuard and the span processor. + * + * SpanGuard::discard() wants to drop a span without sending it to the + * exporter. The OTel SDK calls SpanProcessor::OnEnd() synchronously on the + * same thread that calls Span::End(), so a thread-local flag set just before + * End() and read inside OnEnd() lets FilteringSpanProcessor drop the span + * before it enters the batch export queue. + * + * This side-channel avoids inspecting the Recordable's internals (which vary + * by exporter type — SpanData vs OtlpRecordable). + * + * The flag is a *private* thread-local member of DiscardScope, mutated only by + * its constructor and destructor. This gives real access control rather than a + * naming convention: no code that includes this header can flip the flag + * directly — it can only enter a DiscardScope (which sets and clears the flag + * over its own lifetime) and observe the state via DiscardScope::isActive(). + * Binding set/clear to a scope also means the flag cannot leak onto the next + * span even if End() were to throw. + * + * Kept in a separate header to avoid transitive include bloat: SpanGuard.h + * only needs this signaling, not the full Telemetry.h with BasicConfig/Journal. + * + * Usage: + * @code + * // In SpanGuard::discard(): + * { + * DiscardScope discardScope; // flag set for this scope only + * span->End(); // OnEnd() runs synchronously, sees flag + * } // flag cleared here, unconditionally + * + * // In FilteringSpanProcessor::OnEnd(): + * if (DiscardScope::isActive()) + * return; // drop the span + * @endcode + * + * @note Thread safety: the flag is thread-local, so each thread observes only + * its own discard signal — no synchronization is required. + * + * @see SpanGuard::discard(), FilteringSpanProcessor (Telemetry.cpp) + */ + +namespace xrpl::telemetry { + +/** + * RAII guard that marks the current thread's span for discard. + * + * Sets the thread-local discard flag on construction and clears it on + * destruction, so a span ended within the guard's scope is dropped by + * FilteringSpanProcessor::OnEnd() while the flag stays confined to that scope. + * The flag is private and mutated only here, so no other code can set it. + * Non-copyable and non-movable — its sole purpose is the scoped flag lifetime. + */ +class DiscardScope +{ +public: + DiscardScope() noexcept + { + discarding = true; + } + + ~DiscardScope() + { + discarding = false; + } + + DiscardScope(DiscardScope const&) = delete; + DiscardScope& + operator=(DiscardScope const&) = delete; + DiscardScope(DiscardScope&&) = delete; + DiscardScope& + operator=(DiscardScope&&) = delete; + + /** + * @return true if the current thread is inside a DiscardScope, i.e. the + * span ending now should be dropped rather than exported. Read by + * FilteringSpanProcessor::OnEnd(). + */ + [[nodiscard]] static bool + isActive() noexcept + { + return discarding; + } + +private: + /** + * Thread-local discard flag. Private, so only this class's ctor/dtor can + * mutate it; observers use isActive(). One instance per thread. + */ + inline static thread_local bool discarding = false; +}; + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/Redaction.h b/include/xrpl/telemetry/Redaction.h new file mode 100644 index 00000000000..3b1e8f67ef9 --- /dev/null +++ b/include/xrpl/telemetry/Redaction.h @@ -0,0 +1,66 @@ +#pragma once + +/** + * Account-address redaction for telemetry span attributes. + * + * Path-finding RPC handlers would otherwise emit the caller's raw + * account addresses as span attributes. To keep plaintext addresses out + * of the telemetry backend, they are hashed at the point of emission. + * This header exposes a single pure helper that turns an address into a + * short, stable, obfuscated token. + * + * Data flow: + * + * handler -> redactAccount(addr) -> span attribute -> OTLP export + * + * The returned token is the first 16 hex characters (lowercase) of the + * SHA-512Half digest of the address. It is deterministic (same address + * always maps to the same token) so operators can still correlate spans + * for a given account across nodes and restarts. + * + * The hash is unsalted, so it is obfuscation, not a secrecy guarantee: + * XRP account addresses are a public, enumerable set, so a determined + * observer with the telemetry stream could rebuild the address->token + * mapping. The goal here is to keep plaintext addresses out of traces + * and dashboards, not to defend against a precomputation attack. A salt + * is intentionally omitted because it would break cross-node/restart + * correlation, which is the reason for hashing rather than dropping. + * + * A second, independent hashing layer runs in the OpenTelemetry + * Collector (an `attributes/hash` processor) as defense-in-depth for + * any node that emits a raw value. + * + * @note This function is pure and reentrant: it holds no global state, + * performs no I/O, and is safe to call concurrently from any thread. + * + * Usage example: + * @code + * #include + * using namespace xrpl::telemetry; + * + * span.setAttribute( + * pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + * @endcode + * + * Edge case (empty input yields empty output): + * @code + * assert(redactAccount("") == ""); + * @endcode + */ + +#include +#include + +namespace xrpl::telemetry { + +/** + * Hash an account address into a short, stable, obfuscated token. + * + * @param addr The account address to redact (e.g. an r-address). + * @return The first 16 lowercase hex characters of sha512Half(addr), + * or an empty string when @p addr is empty. + */ +[[nodiscard]] std::string +redactAccount(std::string_view addr); + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/SpanGuard.h b/include/xrpl/telemetry/SpanGuard.h new file mode 100644 index 00000000000..78031187e0e --- /dev/null +++ b/include/xrpl/telemetry/SpanGuard.h @@ -0,0 +1,696 @@ +#pragma once + +/** + * RAII guard for OpenTelemetry trace spans. + * + * Wraps an OTel Span and Scope behind the pimpl idiom so that no + * opentelemetry headers are exposed in this public header. When + * XRPL_ENABLE_TELEMETRY is not defined, SpanGuard is an empty class + * with all-inline no-op methods — zero overhead, zero dependencies. + * + * Dependency diagram: + * + * +--------------------------------------------+ + * | SpanGuard | + * +--------------------------------------------+ + * | - impl_ : unique_ptr (pimpl) | + * +--------------------------------------------+ + * | + span(cat, prefix, name) [static] | + * | + rootSpan(cat, prefix, name) [static] | + * | + childSpan(name) : SpanGuard | + * | + linkedSpan(name) : SpanGuard | + * | + detached() : SpanGuard | + * | + captureContext() : SpanContext | + * | + setAttribute(key, value) | + * | + setOk() / setError(desc) | + * | + addEvent(name) | + * | + recordException(e) | + * | + discard() | + * | + operator bool() | + * +--------------------------------------------+ + * | hides (pimpl) + * +-------+-------------+ + * | | + * +--------+ +---------------------------+ + * | Span | | optional | + * | (OTel) | | (OTel, non-movable) | + * | | | present : scoped guard | + * | | | nullopt : detached guard | + * +--------+ +---------------------------+ + * + * Static factory methods access the global Telemetry instance + * internally (via Telemetry::getInstance()), check whether tracing + * is enabled for the requested subsystem, and return either an + * active guard or a null (no-op) guard. Callers never need a + * Telemetry reference. + * + * Usage examples: + * + * Span names and attribute keys come from per-module `*SpanNames.h` + * headers (e.g. RpcSpanNames.h, TxSpanNames.h) as typed compile-time + * constants — never raw string literals — so the naming spec is + * enforced at the call site and dashboards stay in sync. + * + * 1. Basic RPC tracing (factory method with category): + * @code + * #include + * using namespace xrpl::telemetry; + * + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * span.setAttribute(rpc_span::attr::command, commandName); + * span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success); + * // span ended automatically on scope exit + * @endcode + * + * 2. Error recording: + * @code + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * try { + * doWork(); + * span.setOk(); + * } catch (std::exception const& e) { + * span.recordException(e); + * } + * @endcode + * + * 3. Cross-thread context propagation: + * @code + * #include + * using namespace xrpl::telemetry; + * + * // Thread A: create span and capture context + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::process); + * auto ctx = span.captureContext(); + * + * // Thread B: create child with captured context + * auto child = SpanGuard::childSpan(rpc_span::op::process, ctx); + * @endcode + * + * 4. Conditional check (rarely needed — methods are no-ops on null): + * @code + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::httpRequest); + * if (span) { + * // expensive attribute computation only when active + * span.setAttribute(rpc_span::attr::requestPayloadSize, computeSize()); + * } + * @endcode + * + * 5. Tail-based filtering via discard(): + * @code + * auto span = SpanGuard::span( + * TraceCategory::Transactions, tx_span::prefix::tx, tx_span::op::process); + * auto result = preflight(tx); + * if (result != tesSUCCESS) { + * span.discard(); // drop span, never exported + * return result; + * } + * @endcode + * + * 6. Fresh trace root at an inbound entry point (primary rootSpan use): + * @code + * #include + * using namespace xrpl::telemetry; + * + * // A peer message handled on a shared worker thread that may + * // already have unrelated spans active — start a clean root so + * // those do not become parents of this trace. Names come from a + * // *SpanNames.h header, never raw literals. + * auto span = SpanGuard::rootSpan( + * TraceCategory::Peer, seg::peer, peer_span::op::validationReceive); + * span.setAttribute(peer_span::attr::ledgerHash, hashStr); + * @endcode + * + * 7. Hand a span to a job on another thread (edge case, detached): + * @code + * #include + * using namespace xrpl::telemetry; + * + * // Build the guard on THIS thread, then strip its thread-local + * // Scope so it can be safely moved into a job and ended there. + * auto span = SpanGuard::span( + * TraceCategory::Ledger, seg::ledger, ledger_span::op::build); + * jobQueue.addJob( + * [g = std::move(span).detached()]() mutable { + * doWork(); + * // g's span ends when the job's lambda is destroyed, + * // on the worker thread — no origin-stack corruption. + * }); + * @endcode + * + * @note Thread safety: A SCOPED guard (from span(), rootSpan(), + * childSpan(), linkedSpan()) must only be used on the thread where it + * was constructed — its internal Scope binds to that thread's + * thread-local context stack, and destroying it elsewhere would pop + * the wrong stack. A guard returned by detached() holds no Scope, so + * it may be moved to and destroyed on another thread; detached() + * itself must be called on the origin (constructing) thread. Use + * captureContext() to propagate the trace context to other threads. + * + * @note Move semantics: Move construction transfers ownership of + * the pimpl pointer — no double-Scope issues. Move assignment is + * deleted to prevent re-scoping mid-flight. + * + * @note Known limitations: + * - Attributes cannot be removed per the OTel spec; use + * setAttribute with an empty value as a convention. + * - SpanGuard::span() (raw Span access) is intentionally not + * exposed — all interaction goes through the public methods. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace protocol { +class TraceContext; +} // namespace protocol + +namespace xrpl::telemetry { + +/** + * Trace subsystem categories for conditional span creation. + * + * Each value maps to a runtime config flag (e.g. `trace_rpc=1`). + * Used by SpanGuard::span(TraceCategory, prefix, name) to decide + * whether to create a real span or return a null guard. + */ +enum class TraceCategory { Rpc, Transactions, Consensus, Peer, Ledger }; + +/** + * Raw trace context bytes for cross-node propagation. + * + * Holds the binary trace_id, span_id, and trace_flags extracted from + * an active span. Used by protocol-layer code to inject trace context + * into outgoing protobuf messages without depending on OTel types. + * + * @see SpanGuard::getTraceBytes(), TraceContextPropagator.h + */ +struct TraceBytes +{ + std::array traceId{}; ///< 16-byte W3C trace identifier. + std::array spanId{}; ///< 8-byte span id of current span. + std::uint8_t traceFlags{0}; ///< W3C trace flags (bit 0 = sampled). + bool valid{false}; ///< True if this struct holds data from an active span. +}; + +/** + * Key-value pair for span event attributes. + * Used by addEvent(name, attrs) to attach structured metadata to events. + */ +using EventAttribute = std::pair; + +/** + * Opaque wrapper for an OTel context snapshot. + * + * Used to propagate trace context across threads. Created by + * SpanGuard::captureContext(), consumed by SpanGuard::childSpan() + * or SpanGuard::linkedSpan() with an explicit parent/link context. + */ +class SpanContext +{ + friend class SpanGuard; + +#ifdef XRPL_ENABLE_TELEMETRY + struct Impl; + std::shared_ptr impl_; + explicit SpanContext(std::shared_ptr impl); +#endif + +public: + SpanContext() = default; + + /** + * @return true if this context holds a valid trace context. + */ +#ifdef XRPL_ENABLE_TELEMETRY + [[nodiscard]] bool + isValid() const; +#else + // NOLINTBEGIN(readability-convert-member-functions-to-static) + [[nodiscard]] bool + isValid() const + { + return false; + } + // NOLINTEND(readability-convert-member-functions-to-static) +#endif +}; + +// --------------------------------------------------------------------------- +// Real implementation (pimpl, compiled in SpanGuard.cpp) +// --------------------------------------------------------------------------- +#ifdef XRPL_ENABLE_TELEMETRY + +/** + * RAII wrapper that activates a span on construction and ends it on + * destruction. All OTel types are hidden behind the Impl pointer. + * Non-copyable, move-constructible. + */ +class SpanGuard +{ + struct Impl; + std::unique_ptr impl_; + + explicit SpanGuard(std::unique_ptr impl); + +public: + /** + * Construct a null (no-op) guard. All methods are safe to call. + */ + SpanGuard(); + ~SpanGuard(); + + SpanGuard(SpanGuard&& other) noexcept; + SpanGuard& + operator=(SpanGuard&&) = delete; + SpanGuard(SpanGuard const&) = delete; + SpanGuard& + operator=(SpanGuard const&) = delete; + + // --- Static factory methods ---------------------------------------- + + /** + * Create a span guarded by a TraceCategory flag. + * The span name is built as "prefix.name". Returns a null guard + * if the category is disabled in config. + * @param cat Trace subsystem category. + * @param prefix Span name prefix (e.g. "rpc.command"). + * @param name Span name suffix (e.g. "submit"). + */ + [[nodiscard]] static SpanGuard + span(TraceCategory cat, std::string_view prefix, std::string_view name); + + /** + * Create a span that always starts a fresh trace root. + * + * Like span(), but ignores this thread's active span so the new + * span never inherits an ambient parent. Use at an inbound entry + * point (e.g. a peer message received on a shared worker thread) + * so unrelated work already on the stack does not pollute the + * trace. The span name is built as "prefix.name". Returns a null + * guard if the category is disabled in config. + * + * @param cat Trace subsystem category. + * @param prefix Span name prefix (e.g. "peer"). + * @param name Span name suffix (e.g. "validation.receive"). + * @return An active root-span guard, or a null guard if disabled. + * @note Must be called on the thread that will own the span, like + * span(); the returned guard is scoped to that thread. + */ + [[nodiscard]] static SpanGuard + rootSpan(TraceCategory cat, std::string_view prefix, std::string_view name); + + // --- Child / linked span creation ---------------------------------- + + /** + * Create a child span parented to this guard's active context. + * @param name Span name for the child. + * @return A new guard, or null if this guard is inactive. + */ + [[nodiscard]] SpanGuard + childSpan(std::string_view name) const; + + /** + * Create a child span parented to an explicit captured context. + * @param name Span name for the child. + * @param parentCtx Context captured via captureContext(). + * @return A new guard, or null if parentCtx is invalid. + */ + [[nodiscard]] static SpanGuard + childSpan(std::string_view name, SpanContext const& parentCtx); + + /** + * Create a span linked (follows-from) to this guard's span. + * The new span is NOT a child — it starts a new sub-tree but + * carries a causal link to this span. + * @param name Span name for the linked span. + * @return A new guard, or null if this guard is inactive. + */ + [[nodiscard]] SpanGuard + linkedSpan(std::string_view name) const; + + /** + * Create a span linked to an explicit captured context. + * @param name Span name for the linked span. + * @param linkCtx Context to link from. + * @return A new guard, or null if linkCtx is invalid. + */ + [[nodiscard]] static SpanGuard + linkedSpan(std::string_view name, SpanContext const& linkCtx); + + /** + * Detach this guard's span from the current thread's context stack. + * + * A scoped guard holds an OTel Scope bound to the constructing + * thread's context stack. Moving such a guard to another thread + * (e.g. into a job queue) and destroying it there would pop the + * wrong stack, leaving the origin thread's stack corrupted so + * later spans inherit a stale parent. detached() pops the Scope + * now, on the origin thread, and returns a new guard that holds + * the same span with no thread-local binding. + * + * Consumes this guard (rvalue-qualified): after the call this + * guard is null and the returned guard owns the span. + * + * @return A scope-less guard safe to move to and destroy on + * another thread, or a null guard if this guard was null. + * @note Must be called on the origin (constructing) thread. The + * returned guard may be freely moved across threads; only + * its final destruction ends the span. + */ + [[nodiscard]] SpanGuard + detached() &&; + + // --- Hash-derived span (category-gated) ----------------------------- + + /** + * Create a span whose trace_id is derived from arbitrary hash data. + * trace_id = hashData[0:16], span_id = random. Gated by the given + * TraceCategory. All nodes using the same hash independently produce + * spans under the same trace_id, enabling cross-node correlation + * without context propagation. + * @param cat Trace subsystem category. + * @param name Full span name (e.g. "tx.receive"). + * @param hashData Pointer to at least 16 bytes of hash data. + * @param hashSize Size of the hash buffer (must be >= 16). + * @param followsFrom Optional captured context to attach as a + * follows-from link. Use to stitch sequential + * top-level spans (e.g. consecutive consensus + * rounds). Ignored if nullptr or invalid. + */ + static SpanGuard + hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + SpanContext const* followsFrom = nullptr); + + /** + * Create a hash-derived span with a remote parent. + * trace_id = hashData[0:16], parent span_id from protobuf context + * propagation. Produces a child span of the sender's span while + * sharing the deterministic trace_id. + * @param cat Trace subsystem category. + * @param name Full span name. + * @param hashData Pointer to at least 16 bytes of hash data. + * @param hashSize Size of the hash buffer (must be >= 16). + * @param parentSpanId Pointer to 8 bytes of parent span ID. + * @param parentSpanSize Size of parent span ID buffer (must be 8). + * @param traceFlags Trace flags from remote context. + */ + static SpanGuard + hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + std::uint8_t const* const parentSpanId, + std::size_t const parentSpanSize, + std::uint8_t const traceFlags); + + // --- Context capture ----------------------------------------------- + + /** + * Snapshot the current thread's OTel context for cross-thread use. + * @return An opaque SpanContext, or an invalid one if null guard. + */ + [[nodiscard]] SpanContext + captureContext() const; + + /** + * Extract raw trace context bytes from this span for propagation. + * + * Unlike captureContext() which captures the thread-local runtime + * context, this method reads the span's own SpanContext directly. + * Safe to call from any thread that holds a reference to this guard. + * + * @return A TraceBytes struct with valid=true if the span is active + * and has a valid context, or valid=false otherwise. + */ + [[nodiscard]] TraceBytes + getTraceBytes() const; + + /** + * Inject the calling thread's currently-active OTel context into a + * protobuf TraceContext message for cross-node propagation. + * + * Encapsulates `RuntimeContext::GetCurrent()` + `injectToProtobuf` + * so callers in app-layer code (e.g. RCLConsensus broadcasting + * TMProposeSet / TMValidation) don't depend on any OTel headers. + * No-op if telemetry is disabled or no span is active. + * + * @param proto The protobuf TraceContext to populate. + */ + static void + injectCurrentContextToProtobuf(protocol::TraceContext& proto); + + // --- Attribute setters (explicit overloads, no OTel types) --------- + + /** + * Set a string attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, std::string_view value); + + /** + * Set a string attribute (C-string overload). No-op on a null guard. + */ + void + setAttribute(std::string_view key, char const* value); + + /** + * Set an integer attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, std::int64_t value); + + /** + * Set a floating-point attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, double value); + + /** + * Set a boolean attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, bool value); + + // --- Status / events ----------------------------------------------- + + /** + * Mark the span status as OK. No-op on a null guard. + */ + void + setOk(); + + /** + * Mark the span status as error. No-op on a null guard. + * @param description Optional human-readable error description. + */ + void + setError(std::string_view description = ""); + + /** + * Add a named event to the span's timeline. No-op on a null guard. + * @param name Event name. + */ + void + addEvent(std::string_view name); + + /** + * Add a named event with key-value attributes to the span's timeline. + * No-op on a null guard. + * @param name Event name. + * @param attrs Attribute pairs (all string_view for simplicity). + */ + void + addEvent(std::string_view name, std::initializer_list attrs); + + /** + * Record an exception as a span event following OTel semantic + * conventions, and mark the span status as error. + * No-op on a null guard. + * @param e The exception to record. + */ + void + recordException(std::exception const& e); + + /** + * Mark this span for discard and end it immediately. + * The FilteringSpanProcessor drops the span before it enters the + * batch export queue. After discard(), the guard is inert. + */ + void + discard(); + + /** + * @return true if this guard holds an active span. + */ + explicit + operator bool() const; +}; + +// --------------------------------------------------------------------------- +// No-op stub (all inline, zero overhead, no OTel dependency) +// --------------------------------------------------------------------------- +#else // XRPL_ENABLE_TELEMETRY not defined + +class SpanGuard +{ +public: + SpanGuard() = default; + ~SpanGuard() = default; + SpanGuard(SpanGuard&&) noexcept = default; + SpanGuard& + operator=(SpanGuard&&) = delete; + SpanGuard(SpanGuard const&) = delete; + SpanGuard& + operator=(SpanGuard const&) = delete; + + [[nodiscard]] static SpanGuard + span(TraceCategory, std::string_view, std::string_view) + { + return {}; + } + + [[nodiscard]] static SpanGuard + rootSpan(TraceCategory, std::string_view, std::string_view) + { + return {}; + } + + // NOLINTBEGIN(readability-convert-member-functions-to-static) + [[nodiscard]] SpanGuard + childSpan(std::string_view) const + { + return {}; + } + [[nodiscard]] static SpanGuard + childSpan(std::string_view, SpanContext const&) + { + return {}; + } + [[nodiscard]] SpanGuard + linkedSpan(std::string_view) const + { + return {}; + } + [[nodiscard]] static SpanGuard + linkedSpan(std::string_view, SpanContext const&) + { + return {}; + } + + [[nodiscard]] SpanGuard + detached() && + { + return {}; + } + + [[nodiscard]] static SpanGuard + hashSpan( + TraceCategory, + std::string_view, + std::uint8_t const*, + std::size_t, + SpanContext const* = nullptr) + { + return {}; + } + [[nodiscard]] static SpanGuard + hashSpan( + TraceCategory, + std::string_view, + std::uint8_t const*, + std::size_t, + std::uint8_t const*, + std::size_t, + std::uint8_t) + { + return {}; + } + + [[nodiscard]] SpanContext + captureContext() const + { + return {}; + } + [[nodiscard]] TraceBytes + getTraceBytes() const + { + return {}; + } + + static void + injectCurrentContextToProtobuf(protocol::TraceContext&) + { + } + // NOLINTEND(readability-convert-member-functions-to-static) + + void + setAttribute(std::string_view, std::string_view) + { + } + void + setAttribute(std::string_view, char const*) + { + } + void + setAttribute(std::string_view, std::int64_t) + { + } + void + setAttribute(std::string_view, double) + { + } + void + setAttribute(std::string_view, bool) + { + } + + void + setOk() + { + } + void + setError(std::string_view = "") + { + } + void + addEvent(std::string_view) + { + } + void + addEvent(std::string_view, std::initializer_list) + { + } + void + recordException(std::exception const&) + { + } + void + discard() + { + } + + explicit + operator bool() const + { + return false; + } +}; + +#endif // XRPL_ENABLE_TELEMETRY + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/SpanNames.h b/include/xrpl/telemetry/SpanNames.h new file mode 100644 index 00000000000..9bc70a99b17 --- /dev/null +++ b/include/xrpl/telemetry/SpanNames.h @@ -0,0 +1,158 @@ +#pragma once + +/** + * Compile-time string concatenation utility and shared telemetry constants. + * + * Provides StaticStr — a compile-time string buffer that implicitly + * converts to std::string_view — and join() for dot-separated concatenation. + * Module-specific span names (e.g. RPC, consensus) live in their respective + * modules and build upon these shared primitives. + * + * @note These constants are NOT guarded by XRPL_ENABLE_TELEMETRY because + * call sites reference them even when SpanGuard methods are no-ops + * (the no-op stubs still accept string_view parameters). The compiler + * elides all inline constexpr values whose only uses are in dead code. + * + * @note Json::StaticString (jss.h) is a pointer wrapper without + * concatenation support. boost::static_string is not constexpr. + * StaticStr exists specifically for compile-time dot-join composition. + * + * Naming conventions (see spec 2026-05-13-span-attr-naming-design): + * - Per-span attribute keys: bare field name (span name carries the domain). + * - Collision qualifier: _ when bare name collides across + * domains or with OTel reserved `status` (e.g. rpc_status, grpc_status). + * - Shared cross-span attributes: _ (underscore) form + * (e.g. tx_hash, peer_id, ledger_seq, consensus_round). + * - Resource attribute keys: xrpl.. (dotted) form is + * RESERVED for process-identity attributes set once at startup on the + * OTel resource (e.g. xrpl.network.id, xrpl.network.type). Do not use + * this form for span attributes — it parses awkwardly in TraceQL and + * blurs the resource/span scope distinction. + * - Span prefixes: [.]. + */ + +#include +#include + +namespace xrpl::telemetry { + +// ===== Compile-time string utility ========================================= + +/** + * Fixed-size character buffer for compile-time string operations. + * Implicitly converts to std::string_view at zero cost. + */ +template +struct StaticStr +{ + char data[N + 1]{}; + static constexpr std::size_t size = N; + + constexpr StaticStr() = default; + + constexpr explicit StaticStr(char const (&str)[N + 1]) + { + for (std::size_t i = 0; i <= N; ++i) + data[i] = str[i]; + } + + constexpr + operator std::string_view() const noexcept + { + return {data, N}; + } +}; + +/** + * Deduction guide: StaticStr from string literal. + */ +template +StaticStr(char const (&)[N]) -> StaticStr; + +/** + * Create a StaticStr from a string literal. + */ +template +constexpr auto +makeStr(char const (&str)[N]) +{ + return StaticStr(str); +} + +/** + * Concatenate two StaticStr values with a dot separator. + */ +template +constexpr auto +join(StaticStr const& lhs, StaticStr const& rhs) +{ + constexpr std::size_t len = A + 1 + B; // lhs + '.' + rhs + StaticStr result; + std::size_t pos = 0; + for (std::size_t i = 0; i < A; ++i) + result.data[pos++] = lhs.data[i]; + result.data[pos++] = '.'; + for (std::size_t i = 0; i < B; ++i) + result.data[pos++] = rhs.data[i]; + result.data[pos] = '\0'; + return result; +} + +// ===== Shared root segments ================================================ + +namespace seg { +inline constexpr auto xrpl = makeStr("xrpl"); +inline constexpr auto rpc = makeStr("rpc"); +inline constexpr auto tx = makeStr("tx"); +inline constexpr auto consensus = makeStr("consensus"); +inline constexpr auto peer = makeStr("peer"); +inline constexpr auto ledger = makeStr("ledger"); +inline constexpr auto network = makeStr("network"); +inline constexpr auto link = makeStr("link"); +} // namespace seg + +// ===== Shared attribute keys (used across modules) ========================= + +namespace attr { +inline constexpr auto networkId = join(join(seg::xrpl, seg::network), makeStr("id")); +inline constexpr auto networkType = join(join(seg::xrpl, seg::network), makeStr("type")); + +/** + * Canonical shared attrs (rule 5 — _ underscore form). + * + * Per the naming convention header note: shared cross-span attribute + * keys use the underscore form, reserving the dotted xrpl.. + * form for resource attributes set on the OTel resource at startup. + * Defined once here, aliased by domain-specific headers. These are + * literal underscore-joined names, not dot-joined via `join()`, since + * `join()` always inserts `.` between its arguments. + */ +inline constexpr auto txHash = makeStr("tx_hash"); +inline constexpr auto peerId = makeStr("peer_id"); +inline constexpr auto ledgerSeq = makeStr("ledger_seq"); + +/** + * Shared close-time attrs — bare names, reused by consensus and ledger. + */ +inline constexpr auto closeTime = makeStr("close_time"); +inline constexpr auto closeTimeCorrect = makeStr("close_time_correct"); +inline constexpr auto closeResolutionMs = makeStr("close_resolution_ms"); +/** + * Shared validation attrs — reused by the consensus and peer validation + * spans. Same concept, same key on every span; the span name tells them + * apart, so neither is emitter-prefixed. `ledgerHash` is a ledger-object + * property (bare, like ledgerSeq); `fullValidation` is the is-full-validation + * flag. Never the dotted xrpl. form (reserved for resource attrs). + */ +inline constexpr auto ledgerHash = makeStr("ledger_hash"); +inline constexpr auto fullValidation = makeStr("full_validation"); +} // namespace attr + +// ===== Shared attribute values ============================================= + +namespace attr_val { +inline constexpr auto success = makeStr("success"); +inline constexpr auto error = makeStr("error"); +} // namespace attr_val + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/Telemetry.h b/include/xrpl/telemetry/Telemetry.h new file mode 100644 index 00000000000..51f5ab61848 --- /dev/null +++ b/include/xrpl/telemetry/Telemetry.h @@ -0,0 +1,477 @@ +#pragma once + +/** + * Abstract interface for OpenTelemetry distributed tracing. + * + * Provides the Telemetry base class that all components use to create trace + * spans. Three concrete implementations exist, selected at construction time + * by makeTelemetry(): + * + * - TelemetryImpl (Telemetry.cpp): real OTel SDK integration, compiled + * only when XRPL_ENABLE_TELEMETRY is defined and enabled at runtime. + * - NullTelemetry (NullTelemetry.cpp): no-op stub used when telemetry is + * disabled at compile time or runtime. + * - NullTelemetryOtel (Telemetry.cpp): no-op stub that still depends on + * the OTel API (used during transition or for testing). + * + * Inheritance / dependency diagram: + * + * +--------------------+ + * | Telemetry | (abstract, this file) + * | <> | + * +---------+----------+ + * | + * +---------+-----------+-------------------+ + * | | | + * +---+------------+ +-----+---------+ +------+----------+ + * | TelemetryImpl | | NullTelemetry | | NullTelemetryOtel| + * | (Telemetry.cpp)| |(NullTelemetry | | (Telemetry.cpp) | + * | OTel SDK | | .cpp) | | noop w/ OTel API | + * +----------------+ +---------------+ +------------------+ + * + * The Setup struct holds all configuration parsed from the [telemetry] + * section of xrpld.cfg. See TelemetryConfig.cpp for the parser and + * cfg/xrpld-example.cfg for the available options. + * + * OTel SDK headers are conditionally included behind XRPL_ENABLE_TELEMETRY + * so that builds without telemetry have zero dependency on opentelemetry-cpp. + * + * Usage examples: + * + * 1. Root span at a subsystem entry point (typical usage): + * @code + * #include + * using namespace xrpl::telemetry; + * + * // In an RPC handler dispatch: + * auto guard = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * guard.setAttribute(rpc_span::attr::command, commandName); + * // ... process request + * // guard destructor automatically ends the span on scope exit + * @endcode + * + * 2. Child span for a sub-operation (scoped child): + * @code + * auto parent = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::process); + * { + * auto child = parent.childSpan(rpc_span::op::process); + * child.setAttribute(rpc_span::attr::version, apiVersion); + * // child ends here + * } + * @endcode + * + * 3. Unrelated span (cross-scope, same thread): + * @code + * // gRPC and RPC handlers can be active simultaneously + * auto grpcSpan = SpanGuard::span( + * TraceCategory::Rpc, grpc_span::prefix::grpc, grpc_span::attr::method); + * auto rpcSpan = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * // both spans end on scope exit + * @endcode + * + * 4. Cross-thread context propagation: + * @code + * // Thread A: capture the active context while span is in scope + * auto ctx = parentGuard.captureContext(); + * + * // Thread B: create child span with explicit parent + * auto child = SpanGuard::childSpan(rpc_span::op::process, ctx); + * @endcode + * + * @note Thread safety: The Telemetry interface is safe for concurrent reads + * (isEnabled, shouldTrace*, getTracer, startSpan) after start() completes. + * setServiceInstanceId() must be called before start() and is not thread-safe. + * The OTel SDK's TracerProvider and Tracer are internally thread-safe. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#include +#endif + +namespace xrpl::telemetry { + +#ifdef XRPL_ENABLE_TELEMETRY +/** + * OTel instrumentation scope (tracer) name. Identifies this library as the + * source of spans; distinct from the `service.name` resource attribute + * (Setup::serviceName), which is config-overridable. + */ +inline constexpr std::string_view kTracerName{"xrpld"}; + +/** + * OTel instrumentation scope (meter) name. Identifies this library as the + * source of metrics; symmetric with kTracerName for the tracing side. + */ +inline constexpr std::string_view kMeterName{"xrpld"}; + +/** + * OTel instrumentation scope version reported for the meter. + */ +inline constexpr std::string_view kMeterVersion{"1.0.0"}; +#endif + +class Telemetry +{ + /** + * Global singleton pointer, set by start()/stop() in the active + * implementation. Allows SpanGuard factory methods to access the + * Telemetry instance without callers passing it explicitly. + * + * Atomic with acquire/release ordering: start()/stop() store on + * the initialization thread, factory methods load on worker threads. + * @see setInstance(), getInstance() + */ + inline static std::atomic instance{nullptr}; + +public: + /** + * Get the global Telemetry instance. + * @return Pointer to the active instance, or nullptr if not started. + */ + static Telemetry* + getInstance() + { + return instance.load(std::memory_order_acquire); + } + + /** + * Set the global Telemetry instance. + * Called by start()/stop() in concrete implementations. + * Tests can call this with a mock to override the global instance. + * @param t Pointer to the Telemetry instance, or nullptr to clear. + */ + static void + setInstance(Telemetry* t) + { + instance.store(t, std::memory_order_release); + } + + /** + * Configuration parsed from the [telemetry] section of xrpld.cfg. + * + * All fields have sensible defaults so the section can be minimal + * or omitted entirely. See TelemetryConfig.cpp for the parser. + */ + struct Setup + { + /** + * Master switch: true to enable tracing at runtime. + */ + bool enabled = false; + + /** + * OTel resource attribute `service.name`. + */ + std::string serviceName = "xrpld"; + + /** + * OTel resource attribute `service.version` (set from BuildInfo). + */ + std::string serviceVersion; + + /** + * OTel resource attribute `service.instance.id` (defaults to node + * public key). + */ + std::string serviceInstanceId; + + /** + * OTLP/HTTP endpoint URL where spans are sent. + */ + std::string exporterEndpoint = "http://localhost:4318/v1/traces"; + + /** + * Whether to use TLS for the exporter connection. + */ + bool useTls = false; + + /** + * Path to a CA certificate bundle for TLS verification. + */ + std::string tlsCertPath; + + /** + * Path to this node's client certificate (PEM), presented to the + * collector for mutual TLS. Empty disables client-side auth, in + * which case only server (one-way) TLS is used. + */ + std::string tlsClientCertPath; + + /** + * Path to the private key (PEM) for tlsClientCertPath. Required + * whenever tlsClientCertPath is set. + */ + std::string tlsClientKeyPath; + + /** + * Head-based sampling ratio. Intentionally fixed at 1.0 (sample + * everything) and NOT read from config. A per-node ratio would let + * nodes make divergent keep/drop decisions for the same distributed + * trace, producing broken/partial traces. The ratio sampler is wrapped + * in a ParentBasedSampler (see Telemetry.cpp) so spans inheriting a + * remote parent honor the upstream sampled flag. Volume reduction is + * delegated to the collector's tail sampling; for node-local post-hoc + * dropping see SpanGuard::discard(). + */ + static constexpr double samplingRatio = 1.0; + + /** + * Maximum number of spans per batch export. + */ + std::uint32_t batchSize = 512; + + /** + * Delay between batch exports. + */ + std::chrono::milliseconds batchDelay = std::chrono::milliseconds{5000}; + + /** + * Maximum number of spans queued before dropping. + */ + std::uint32_t maxQueueSize = 2048; + + /** + * Network identifier, added as an OTel resource attribute. + */ + std::uint32_t networkId = 0; + + /** + * Network type label (e.g. "mainnet", "testnet", "devnet"). + */ + std::string networkType = "mainnet"; + + /** + * Enable tracing for transaction processing. + */ + bool traceTransactions = true; + + /** + * Enable tracing for consensus rounds. + */ + bool traceConsensus = true; + + /** + * Enable tracing for RPC request handling. + */ + bool traceRpc = true; + + /** + * Enable tracing for peer-to-peer messages (enabled by default; + * high volume). + */ + bool tracePeer = true; + + /** + * Enable tracing for ledger close/accept. + */ + bool traceLedger = true; + + /** + * Strategy for cross-node consensus trace correlation. + * "deterministic" — derive trace_id from ledger hash so all + * validators in the same round share the same trace_id. + * "attribute" — random trace_id, correlate via ledger_id attribute. + */ + std::string consensusTraceStrategy = "deterministic"; + }; + + virtual ~Telemetry() = default; + + /** + * Update the service instance ID (OTel resource attribute + * `service.instance.id`). + * + * Must be called before start(). The node public key is not available + * when Telemetry is constructed (during the ApplicationImp member + * initializer list), so this setter allows Application::setup() to + * inject the identity once nodeIdentity_ is known. + * + * @param id The node's base58-encoded public key or custom identifier. + */ + virtual void + setServiceInstanceId(std::string const& id) + { + // Default no-op for NullTelemetry implementations. + (void)id; + } + + /** + * Initialize the tracing pipeline (exporter, processor, provider). + * Call after construction. + */ + virtual void + start() = 0; + + /** + * Flush pending spans and shut down the tracing pipeline. + * Call before destruction. + */ + virtual void + stop() = 0; + + /** + * @return true if this instance is actively exporting spans. + */ + [[nodiscard]] virtual bool + isEnabled() const = 0; + + /** + * @return true if transaction processing should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceTransactions() const = 0; + + /** + * @return true if consensus rounds should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceConsensus() const = 0; + + /** + * @return true if RPC request handling should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceRpc() const = 0; + + /** + * @return true if peer-to-peer messages should be traced. + */ + [[nodiscard]] virtual bool + shouldTracePeer() const = 0; + + /** + * @return true if ledger close/accept should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceLedger() const = 0; + + /** + * @return The configured consensus trace correlation strategy. + */ + [[nodiscard]] virtual std::string const& + getConsensusTraceStrategy() const = 0; + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Get or create a named tracer instance. + * + * @param name Tracer name used to identify the instrumentation library. + * @return A shared pointer to the Tracer. + */ + virtual opentelemetry::nostd::shared_ptr + getTracer(std::string_view name = kTracerName) = 0; + + /** + * Get or create a named meter instance. + * + * Returns the raw OTel Meter, giving developers direct access to the + * full metrics API. From the returned Meter any instrument type can be + * created: Counter, UpDownCounter, Gauge, and Histogram (synchronous), + * plus their observable/async variants (ObservableCounter, + * ObservableUpDownCounter, ObservableGauge). + * + * @param name Meter name used to identify the instrumentation scope. + * @return A shared pointer to the Meter. + */ + virtual opentelemetry::nostd::shared_ptr + getMeter(std::string_view name = kMeterName) = 0; + + /** + * Start a new span on the current thread's context. + * + * The span becomes a child of the current active span (if any) via + * OpenTelemetry's context propagation. + * + * @param name Span name (typically "rpc.command."). + * @param kind The span kind (defaults to kInternal). Possible values: + * - kInternal: default, in-process operation + * - kServer: incoming synchronous request (e.g. RPC) + * - kClient: outgoing synchronous request + * - kProducer: async message send (e.g. peer broadcast) + * - kConsumer: async message receive + * @return A shared pointer to the new Span. + */ + virtual opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::trace::SpanKind kind = opentelemetry::trace::SpanKind::kInternal) = 0; + + /** + * Start a new span with an explicit parent context. + * + * Use this overload when the parent span is not on the current + * thread's context stack (e.g. cross-thread trace propagation). + * + * @param name Span name. + * @param parentContext The parent span's context. + * @param kind The span kind (defaults to kInternal). + * @return A shared pointer to the new Span. + */ + virtual opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::context::Context const& parentContext, + opentelemetry::trace::SpanKind kind = opentelemetry::trace::SpanKind::kInternal) = 0; +#endif +}; + +/** + * Create a Telemetry instance. + * + * Returns a TelemetryImpl when setup.enabled is true, or a + * NullTelemetry no-op stub otherwise. + * + * @param setup Configuration from the [telemetry] config section. + * @param journal Journal for log output during initialization. + */ +std::unique_ptr +makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal); + +/** + * Parse the [telemetry] config section into a Setup struct. + * + * @param section The [telemetry] config section. + * @param nodePublicKey Node public key, used as default instance ID. + * @param version Build version string. + * @param networkId Network identifier from [network_id] config + * (0 = mainnet, 1 = testnet, 2 = devnet). + * @return A populated Setup struct with defaults for missing values. + */ +Telemetry::Setup +makeTelemetrySetup( + Section const& section, + std::string const& nodePublicKey, + std::string const& version, + std::uint32_t networkId); + +/** + * Derive a human-readable network type label from the numeric network ID. + * + * Shared by the trace and metric export paths so both stamp the same + * `xrpl.network.type` resource attribute value. + * + * @param networkId The network identifier from [network_id] config. + * @return "mainnet" (0), "testnet" (1), "devnet" (2), or "unknown". + */ +std::string +networkTypeFromId(std::uint32_t networkId); + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/TraceContextPropagator.h b/include/xrpl/telemetry/TraceContextPropagator.h new file mode 100644 index 00000000000..e8d851e53e2 --- /dev/null +++ b/include/xrpl/telemetry/TraceContextPropagator.h @@ -0,0 +1,115 @@ +#pragma once + +/** + * Utilities for trace context propagation across nodes. + * + * Provides serialization/deserialization of OTel trace context to/from + * Protocol Buffer TraceContext messages (P2P cross-node propagation). + * Wired into the P2P message flow via PropagationHelpers.h for + * TMTransaction, TMProposeSet, and TMValidation messages. + * + * Only compiled when XRPL_ENABLE_TELEMETRY is defined. + * + * @see PropagationHelpers.h (high-level inject helpers), + * TxTracing.h (transaction receive-side extraction), + * ConsensusReceiveTracing.h (proposal/validation receive-side). + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +/** + * Extract OTel context from a protobuf TraceContext message. + * + * @param proto The protobuf TraceContext received from a peer. + * @return An OTel Context with the extracted parent span, or an empty + * context if the protobuf fields are missing or invalid. + */ +inline opentelemetry::context::Context +extractFromProtobuf(protocol::TraceContext const& proto) +{ + namespace trace = opentelemetry::trace; + + // Reject malformed or all-zero ids from the peer before trusting + // them as a parent. See TraceContextValidation.h. + if (!isValidTraceContext(proto)) + { + return opentelemetry::context::Context{}; + } + + auto const* rawTraceId = reinterpret_cast(proto.trace_id().data()); + auto const* rawSpanId = reinterpret_cast(proto.span_id().data()); + trace::TraceId const traceId( + opentelemetry::nostd::span(rawTraceId, 16)); + trace::SpanId const spanId(opentelemetry::nostd::span(rawSpanId, 8)); + trace::TraceFlags const flags( + proto.has_trace_flags() ? static_cast(proto.trace_flags()) + : static_cast(0)); + + trace::SpanContext const spanCtx(traceId, spanId, flags, /* remote = */ true); + + return opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr(new trace::DefaultSpan(spanCtx))); +} + +/** + * Inject the current span's trace context into a protobuf TraceContext. + * + * @param ctx The OTel context containing the span to propagate. + * @param proto The protobuf TraceContext to populate. + */ +inline void +injectToProtobuf(opentelemetry::context::Context const& ctx, protocol::TraceContext& proto) +{ + namespace trace = opentelemetry::trace; + + auto const span = trace::GetSpan(ctx); + if (!span) + return; + + auto const& spanCtx = span->GetContext(); + if (!spanCtx.IsValid()) + return; + + // Serialize trace_id (16 bytes) + auto const& traceId = spanCtx.trace_id(); + proto.set_trace_id(traceId.Id().data(), trace::TraceId::kSize); + + // Serialize span_id (8 bytes) + auto const& spanId = spanCtx.span_id(); + proto.set_span_id(spanId.Id().data(), trace::SpanId::kSize); + + // Serialize flags + proto.set_trace_flags(spanCtx.trace_flags().flags()); + + // TODO(observability/secure-OTel): the protobuf TraceContext message + // also carries `trace_state` (field 4), which is currently neither + // populated here nor read by extractFromProtobuf above. The field is + // reserved for the secure tracing pipeline outlined in + // OpenTelemetryPlan/secure-OTel.md, where an authenticated token in + // tracestate will let receivers reject spoofed/poisoned trace context. + // Wire trace_state through inject/extract once the consumer lands. +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/include/xrpl/telemetry/TraceContextValidation.h b/include/xrpl/telemetry/TraceContextValidation.h new file mode 100644 index 00000000000..e69b2d17ca1 --- /dev/null +++ b/include/xrpl/telemetry/TraceContextValidation.h @@ -0,0 +1,96 @@ +#pragma once + +/** + * Validation predicates for peer-supplied trace context. + * + * A protobuf TraceContext arrives inside untrusted peer messages + * (TMTransaction, TMProposeSet, TMValidation). Before a receiving node + * builds a span from it, the context must be checked: a peer can send a + * malformed or all-zero trace_id/span_id, either by accident or to + * pollute traces. This header is the single place those checks live, so + * every receive site agrees on what "valid" means. + * + * Validity follows the OpenTelemetry spec: a trace_id is 16 bytes, a + * span_id is 8 bytes, and an all-zero id is invalid and must not be + * used as a parent. + * + * Dependency diagram: + * + * protocol::TraceContext (untrusted, from peer) + * | + * v + * isValidTraceContext() / isValidSpanId() / isValidTraceId() + * | + * +--- valid ----> build child span from peer context + * | + * +--- invalid --> ignore peer context, start fresh span + * + * This header depends only on the generated protobuf type. It pulls in + * no OpenTelemetry headers, so receive sites that use SpanGuard keep + * SpanGuard's encapsulation of OTel types. + * + * @note Thread-safe: all functions are pure and operate only on their + * arguments. No shared state. + * + * Usage: + * @code + * // Full parent context (both ids come from the peer): + * if (isValidTraceContext(msg.trace_context())) + * buildChildSpan(...); + * + * // Span-only (trace_id is derived locally, not from the peer): + * if (tc.has_span_id() && isValidSpanId(tc.span_id())) + * buildChildSpan(...); + * @endcode + */ + +#include + +#include +#include + +namespace xrpl::telemetry { + +/** + * True if the bytes are a valid OTel trace_id: 16 bytes, not all-zero. + * + * @param traceId The raw trace_id bytes from a protobuf TraceContext. + * @return true if usable as a trace identifier, false otherwise. + */ +inline bool +isValidTraceId(std::string const& traceId) +{ + return traceId.size() == 16 && std::ranges::any_of(traceId, [](char c) { return c != 0; }); +} + +/** + * True if the bytes are a valid OTel span_id: 8 bytes, not all-zero. + * + * @param spanId The raw span_id bytes from a protobuf TraceContext. + * @return true if usable as a span identifier, false otherwise. + */ +inline bool +isValidSpanId(std::string const& spanId) +{ + return spanId.size() == 8 && std::ranges::any_of(spanId, [](char c) { return c != 0; }); +} + +/** + * True if the context carries a usable parent: a valid trace_id and a + * valid span_id together. + * + * Use this where both ids are taken from the peer (consensus receive, + * generic extraction). The transaction path derives its trace_id + * locally from the txID, so it checks isValidSpanId() alone instead. + * + * @param tc The protobuf TraceContext received from a peer. + * @return true if both ids are present and valid, false otherwise. + */ +inline bool +isValidTraceContext(protocol::TraceContext const& tc) +{ + return tc.has_trace_id() && isValidTraceId(tc.trace_id()) && tc.has_span_id() && + isValidSpanId(tc.span_id()); +} + +} // namespace xrpl::telemetry diff --git a/include/xrpl/tx/detail/TxApplySpanNames.h b/include/xrpl/tx/detail/TxApplySpanNames.h new file mode 100644 index 00000000000..1228bd57ef4 --- /dev/null +++ b/include/xrpl/tx/detail/TxApplySpanNames.h @@ -0,0 +1,141 @@ +#pragma once + +/** + * Compile-time span name constants for the transaction apply pipeline. + * + * Defines the span names and attribute keys used by the three apply-pipeline + * stages — preflight, preclaim, and transactor (apply) — that run inside the + * library (`src/libxrpl/tx/`). Built on the StaticStr/join() primitives from + * . + * + * Why a separate header from TxSpanNames.h: + * TxSpanNames.h lives under src/xrpld/ (daemon) and serves the overlay/app + * lifecycle spans (tx.receive, tx.process). Library code (applySteps.cpp, + * Transactor.cpp) must not depend on daemon headers, so the apply-pipeline + * constants live here instead. The attribute strings ("tx_type", + * "ter_result", "applied") intentionally match TxSpanNames.h so the collector + * spanmetrics connector aggregates both sets under the same dimensions. + * + * Span hierarchy (deterministic trace_id derived from txID[0:16]): + * + * The three stages run sequentially and often on different threads, so they + * do not auto-parent. Each uses a hash-derived trace_id keyed on the same + * transaction id, placing all three under one trace without context + * propagation. A transaction that hard-fails preflight or preclaim never + * reaches the transactor span — the stage attribute identifies where it + * stopped. + * + * +-----------------------------------------------------------+ + * | trace_id = txID[0:16] | + * | | + * | +-------------------+ +------------------+ +-------+ | + * | | tx.preflight | | tx.preclaim | | tx. | | + * | | stage=preflight |-->| stage=preclaim |-->| trans | | + * | | tx_type | | tx_type | | actor | | + * | | ter_result | | ter_result | | stage=| | + * | +-------------------+ +------------------+ | apply | | + * | stateless checks ledger-aware checks +-------+ | + * | (signature, fields) (sequence, fee) applies | + * +-----------------------------------------------------------+ + * + * Usage: + * @code + * #include + * using namespace telemetry; + * + * // preflight() / preclaim() use hashSpan with a full span name: + * auto span = SpanGuard::hashSpan( + * TraceCategory::Transactions, tx_apply_span::preflight, + * txID.data(), txID.kBytes); + * span.setAttribute(tx_apply_span::attr::stage, tx_apply_span::val::preflight); + * span.setAttribute(tx_apply_span::attr::terResult, transToken(ter).c_str()); + * @endcode + * + * @code + * // Transactor::operator() also uses hashSpan on the same txID so it + * // co-traces with preflight and preclaim under one trace_id: + * auto span = SpanGuard::hashSpan( + * TraceCategory::Transactions, tx_apply_span::transactor, + * txID.data(), txID.kBytes); + * span.setAttribute(tx_apply_span::attr::stage, tx_apply_span::val::apply); + * @endcode + */ + +#include + +namespace xrpl::telemetry::tx_apply_span { + +// ===== Span operation suffixes ============================================= + +namespace op { +/** + * "preflight" — stateless transaction checks (suffix form). + */ +inline constexpr auto preflight = makeStr("preflight"); +/** + * "preclaim" — ledger-aware checks before fee claim (suffix form). + */ +inline constexpr auto preclaim = makeStr("preclaim"); +/** + * "transactor" — the apply stage (suffix form, used with span()). + */ +inline constexpr auto transactor = makeStr("transactor"); +} // namespace op + +// ===== Full span names (tx.) =========================================== + +/** + * "tx.preflight" — full name for hashSpan() at the preflight stage. + */ +inline constexpr auto preflight = join(seg::tx, op::preflight); +/** + * "tx.preclaim" — full name for hashSpan() at the preclaim stage. + */ +inline constexpr auto preclaim = join(seg::tx, op::preclaim); +/** + * "tx.transactor" — full name for hashSpan() at the apply stage. Shares the + * txID-derived trace_id so it co-traces with tx.preflight and tx.preclaim. + */ +inline constexpr auto transactor = join(seg::tx, op::transactor); + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * "stage" — which apply-pipeline stage this span represents. Drives the + * collector spanmetrics `stage` dimension for per-stage RED metrics. + */ +inline constexpr auto stage = makeStr("stage"); +/** + * "tx_type" — transaction type name (e.g., "Payment", "OfferCreate"). + * Matches tx_span::attr::txType so both share the spanmetrics dimension. + */ +inline constexpr auto txType = makeStr("tx_type"); +/** + * "ter_result" — engine result code after the stage (e.g., "tesSUCCESS"). + */ +inline constexpr auto terResult = makeStr("ter_result"); +/** + * "applied" — whether the transaction was applied to the ledger (apply only). + */ +inline constexpr auto applied = makeStr("applied"); +} // namespace attr + +// ===== Attribute values (stage names) ====================================== + +namespace val { +/** + * "preflight" — value of the stage attribute on tx.preflight. + */ +inline constexpr auto preflight = makeStr("preflight"); +/** + * "preclaim" — value of the stage attribute on tx.preclaim. + */ +inline constexpr auto preclaim = makeStr("preclaim"); +/** + * "apply" — value of the stage attribute on tx.transactor. + */ +inline constexpr auto apply = makeStr("apply"); +} // namespace val + +} // namespace xrpl::telemetry::tx_apply_span diff --git a/sanitizers/suppressions/ubsan.supp b/sanitizers/suppressions/ubsan.supp index 7e3e02f8553..de2a66e64ae 100644 --- a/sanitizers/suppressions/ubsan.supp +++ b/sanitizers/suppressions/ubsan.supp @@ -102,6 +102,10 @@ undefined:nudb # Snappy compression library intentional overflows unsigned-integer-overflow:snappy.cc +# OpenTelemetry SDK intentional overflows in hash functions +unsigned-integer-overflow:opentelemetry/sdk/common/attributemap_hash.h +unsigned-integer-overflow:opentelemetry + # Abseil intentional overflows in hashing, RNG and time arithmetic. # Matched at library scope (like boost above): the wraparound is by design # across many absl files (hash mixing, raw_hash_set probing, duration math, diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index d1e54a515fe..7111bc5f349 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -7,6 +7,15 @@ #include #include +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#include +#endif // XRPL_ENABLE_TELEMETRY + #include #include #include @@ -291,6 +300,34 @@ Logs::format( break; } +#ifdef XRPL_ENABLE_TELEMETRY + // Inject OTel trace context when an active span exists on this thread. + // Checks the thread-local context value directly to avoid the heap + // allocation that GetSpan() performs on the no-span path. + { + auto context = opentelemetry::context::RuntimeContext::GetCurrent(); + auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey); + if (opentelemetry::nostd::holds_alternative< + opentelemetry::nostd::shared_ptr>(spanValue)) + { + auto span = opentelemetry::nostd::get< + opentelemetry::nostd::shared_ptr>(spanValue); + auto spanCtx = span->GetContext(); + if (spanCtx.IsValid()) + { + char traceId[32], spanId[16]; + spanCtx.trace_id().ToLowerBase16(opentelemetry::nostd::span{traceId}); + spanCtx.span_id().ToLowerBase16(opentelemetry::nostd::span{spanId}); + output += "trace_id="; + output.append(traceId, 32); + output += " span_id="; + output.append(spanId, 16); + output += ' '; + } + } + } +#endif // XRPL_ENABLE_TELEMETRY + output += message; // Limit the maximum length of the output diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp new file mode 100644 index 00000000000..ab171592720 --- /dev/null +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -0,0 +1,913 @@ +/** + * @file OTelCollector.cpp + * @brief OpenTelemetry Metrics SDK implementation of beast::insight::Collector. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake + * telemetry=ON). Maps beast::insight instruments to OTel SDK instruments + * created on the GLOBAL Meter published by the telemetry module. This class + * is a legacy shim: it no longer owns an export pipeline. The MeterProvider, + * PeriodicExportingMetricReader, OTLP exporter and histogram view all live in + * xrpl::telemetry::Telemetry. + * + * When XRPL_ENABLE_TELEMETRY is not defined, OTelCollector::New() returns + * a NullCollector so the build succeeds without OTel dependencies. + * + * Data flow: + * + * beast::insight callers + * | + * v + * OTelCounterImpl / OTelGaugeImpl / OTelEventImpl / OTelMeterImpl + * | | | | + * v v v v + * Counter ObservableGauge Histogram Counter + * | | | | + * +--------------------+----------------+--------------+ + * | + * v + * GLOBAL Meter (from metrics::Provider::GetMeterProvider()) + * | + * v + * telemetry-owned PeriodicMetricReader (1s) -> OTLP exporter -> Prometheus + */ + +#ifdef XRPL_ENABLE_TELEMETRY +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace beast::insight { + +namespace detail { + +namespace metrics_api = opentelemetry::metrics; + +class OTelCollectorImp; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::HookImpl. + * + * Stores a handler function that is invoked during each periodic + * metric collection cycle. This mirrors the StatsDHookImpl pattern + * where hooks are called at each 1-second timer tick, but here the + * invocation is triggered by the OTel PeriodicMetricReader's + * observable callback mechanism. + */ +class OTelHookImpl : public HookImpl +{ +public: + /** + * @param handler Callback invoked at each collection interval. + * @param impl Owning collector (prevents premature destruction). + */ + OTelHookImpl(HandlerType handler, std::shared_ptr impl); + + ~OTelHookImpl() override; + + OTelHookImpl& + operator=(OTelHookImpl const&) = delete; + + /** + * @brief Invoke the stored handler. + * + * Called by the collector during observable gauge callbacks to give + * metric producers a chance to update gauge values before export. + */ + void + callHandler(); + +private: + /** + * Owning collector. Prevents collector destruction while hook alive. + */ + std::shared_ptr impl_; + + /** + * User-supplied handler called at each collection interval. + */ + HandlerType handler_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::CounterImpl. + * + * Wraps an OTel Counter instrument. Each increment() call + * is forwarded directly to the OTel counter's Add() method. The + * PeriodicMetricReader collects and exports the accumulated delta. + * + * Thread safety: OTel Counter::Add() is thread-safe by specification. + */ +class OTelCounterImpl : public CounterImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended and + * dots replaced with underscores (e.g. "rpc_size"). + * @param meter OTel Meter used to create the counter instrument. + */ + OTelCounterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter); + + ~OTelCounterImpl() override = default; + + OTelCounterImpl& + operator=(OTelCounterImpl const&) = delete; + + /** + * @brief Add amount to the counter. + * @param amount Value to add (must be non-negative for OTel counters). + */ + void + increment(value_type amount) override; + +private: + /** + * OTel synchronous counter instrument. + */ + opentelemetry::nostd::unique_ptr> counter_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::EventImpl. + * + * Wraps an OTel Histogram instrument. Each notify() call + * records the duration in milliseconds. Uses explicit bucket boundaries + * matching the SpanMetrics connector configuration: + * [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms + * + * Thread safety: OTel Histogram::Record() is thread-safe by specification. + */ +class OTelEventImpl : public EventImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended and + * dots replaced with underscores (e.g. "rpc_size"). + * @param meter OTel Meter used to create the histogram instrument. + */ + OTelEventImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter); + + ~OTelEventImpl() override = default; + + OTelEventImpl& + operator=(OTelEventImpl const&) = delete; + + /** + * @brief Record a duration measurement. + * @param value Duration in milliseconds. + */ + void + notify(value_type const& value) override; + +private: + /** + * OTel histogram instrument for recording durations. + */ + opentelemetry::nostd::unique_ptr> histogram_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::GaugeImpl. + * + * Uses an atomic int64_t to store the current gauge value. The OTel SDK + * reads this value via an ObservableGauge async callback during each + * collection cycle. The set() and increment() methods update the + * atomic value without blocking the collection thread. + * + * Design note: OTel gauges are asynchronous (observable) instruments. + * The SDK calls a registered callback to read the value rather than + * accepting push-style updates. We bridge the beast::insight push-style + * API to OTel's pull-style API via the atomic variable. + * + * Thread safety: std::atomic operations are lock-free on all platforms. + */ +class OTelGaugeImpl : public GaugeImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended + * and dots replaced with underscores. + * @param meter OTel Meter used to create the observable gauge. + * @param collector Owning collector, used to invoke hooks before reads. + */ + OTelGaugeImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter, + std::shared_ptr const& collector); + + ~OTelGaugeImpl() override; + + /** + * @brief Set the gauge to an absolute value. + * @param value New gauge value. + */ + void + set(value_type value) override; + + /** + * @brief Increment (or decrement) the gauge by a signed amount. + * + * Clamps the result to [0, INT64_MAX] to match StatsDGaugeImpl + * behavior. + * + * @param amount Signed amount to add to the current value. + */ + void + increment(difference_type amount) override; + + /** + * @brief Return the current gauge value for the OTel callback. + * @return The most recently set/incremented value. + */ + int64_t + currentValue() const; + + OTelGaugeImpl& + operator=(OTelGaugeImpl const&) = delete; + + /** + * Static callback registered with the OTel SDK observable gauge. + */ + static void + gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state); + +private: + /** + * Current gauge value, updated atomically by set()/increment(). + */ + std::atomic value_{0}; + + /** + * OTel observable gauge handle (prevents deregistration). + */ + opentelemetry::nostd::shared_ptr gauge_; + + /** + * Owning collector, used to invoke hooks before reading gauge values. + */ + std::shared_ptr collector_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::MeterImpl. + * + * Wraps an OTel Counter instrument. Semantically identical + * to Counter but uses unsigned values. The OTel SDK accumulates deltas + * and exports them via the PeriodicMetricReader. + * + * Note: In StatsD, Meter used the non-standard "|m" type which was + * silently dropped by the OTel StatsD receiver. With native OTel, + * Meter values are properly captured as counter deltas. + * + * Thread safety: OTel Counter::Add() is thread-safe by specification. + */ +class OTelMeterImpl : public MeterImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended and + * dots replaced with underscores (e.g. "rpc_size"). + * @param meter OTel Meter used to create the counter instrument. + */ + OTelMeterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter); + + ~OTelMeterImpl() override = default; + + OTelMeterImpl& + operator=(OTelMeterImpl const&) = delete; + + /** + * @brief Add amount to the meter. + * @param amount Value to add (unsigned). + */ + void + increment(value_type amount) override; + +private: + /** + * OTel synchronous counter instrument (unsigned). + */ + opentelemetry::nostd::unique_ptr> counter_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief Main OTel Collector implementation (legacy shim). + * + * Obtains its Meter from the GLOBAL MeterProvider owned and published by the + * telemetry module (xrpl::telemetry::Telemetry), rather than building its own + * export pipeline. Implements all Collector::make_*() factory methods to + * create OTel-backed instrument wrappers on that shared Meter. + * + * The metrics pipeline (MeterProvider + PeriodicExportingMetricReader + OTLP + * HTTP exporter + histogram view) lives in the telemetry module. This class is + * a thin adapter kept for beast::insight callers during deprecation. + * + * Class diagram: + * + * +------------------+ +------------------+ + * | Collector (ABC) |<-----| OTelCollector | + * +------------------+ | (public header) | + * ^ +------------------+ + * | ^ + * +------------------+ | + * | OTelCollectorImp |-------------+ + * +------------------+ + * | - journal_ | + * | - prefix_ | +--------------------------+ + * | - otelMeter_ |---->| GLOBAL MeterProvider | + * | - hooks_[] | | (owned by Telemetry) | + * | - gauges_[] | | + PeriodicReader | + * +------------------+ | + OtlpHttpExporter | + * +--------------------------+ + * + * Lifecycle: + * 1. Constructor fetches the Meter from the global MeterProvider. + * 2. make_*() methods create instruments on that shared Meter. + * 3. The telemetry-owned PeriodicMetricReader collects every 1s, calling + * observable callbacks. + * 4. Observable callbacks invoke hooks, read gauge atomics. + * 5. Destructor only logs; the telemetry module owns pipeline teardown. + * + * Caveats: + * - Observable gauge callbacks run on the SDK's internal thread. Hook + * handlers must be thread-safe. + * - Metric names are formed as "prefix_name" with dots replaced by + * underscores to match StatsD->Prometheus naming conventions. + * - The OTel Prometheus exporter appends "_total" to counters. The + * metric names we register do NOT include this suffix — Prometheus + * adds it automatically. + * + * Example usage: + * @code + * auto collector = OTelCollector::New( + * "http://localhost:4318/v1/metrics", "xrpld", + * "node-1", "xrpld", "mainnet", journal); + * auto counter = collector->makeCounter("rpc.requests"); + * counter.increment(1); + * // Metric "rpc_requests" exported via OTLP every 1s. + * @endcode + */ +class OTelCollectorImp : public OTelCollector, public std::enable_shared_from_this +{ +public: + /** + * @brief Construct the OTel collector over the global MeterProvider. + * + * @param endpoint OTLP/HTTP metrics endpoint URL. Informational only: + * the global telemetry pipeline is authoritative for + * the actual export endpoint. Retained for logging and + * back-compat with the New() signature. + * @param prefix Prefix for all metric names. + * @param instanceId Value for the service.instance.id resource attribute. + * When empty, the attribute is omitted. + * @param serviceName Value for the service.name resource attribute. + * When empty, defaults to "xrpld". + * @param networkType Value for the xrpl.network.type resource attribute. + * When empty, the attribute is omitted. + * @param journal Journal for logging. + */ + OTelCollectorImp( + std::string const& endpoint, + std::string prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal); + + /** + * @brief Shut down the MeterProvider, flushing any pending exports. + */ + ~OTelCollectorImp() override; + + /** + * @name Collector interface implementation + */ + /** @{ */ + Hook + makeHook(HookImpl::HandlerType const& handler) override; + + Counter + makeCounter(std::string const& name) override; + + Event + makeEvent(std::string const& name) override; + + Gauge + makeGauge(std::string const& name) override; + + Meter + makeMeter(std::string const& name) override; + /** @} */ + + /** + * @name Hook management for observable callbacks + */ + /** @{ */ + + /** + * @brief Register a hook for periodic invocation. + * @param hook Pointer to the hook to register. + */ + void + addHook(OTelHookImpl* hook); + + /** + * @brief Unregister a hook. + * @param hook Pointer to the hook to unregister. + */ + void + removeHook(OTelHookImpl* hook); + + /** + * @brief Invoke all registered hooks. + * + * Called from observable gauge callbacks before reading gauge values, + * so that hook handlers have a chance to update metrics. + */ + void + callHooks(); + /** @} */ + + /** + * @name Gauge registration for observable callbacks + */ + /** @{ */ + + /** + * @brief Register a gauge for observable callback reading. + * @param gauge Pointer to the gauge to register. + */ + void + addGauge(OTelGaugeImpl* gauge); + + /** + * @brief Unregister a gauge. + * @param gauge Pointer to the gauge to unregister. + */ + void + removeGauge(OTelGaugeImpl* gauge); + /** @} */ + + /** + * @brief Format a metric name with the configured prefix. + * + * Replaces dots with underscores to match StatsD->Prometheus naming. + * Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age" + * -> "ledgermaster_validated_ledger_age" + * + * @param name Raw metric name from beast::insight callers. + * @return Fully-qualified metric name. + */ + static std::string + formatName(std::string const& name); + +private: + /** + * Journal for log output. + */ + Journal journal_; + + /** + * Prefix for all metric names (e.g., "xrpld"). + */ + std::string prefix_; + + /** + * OTel Meter used to create all instruments. + */ + opentelemetry::nostd::shared_ptr otelMeter_; + + /** + * Mutex protecting hook and gauge registration lists. + */ + std::mutex mutex_; + + /** + * Registered hooks called during observable callbacks. + */ + std::vector hooks_; + + /** + * Registered gauges read during observable callbacks. + */ + std::vector gauges_; + + /** + * @brief Debounce timestamp for callHooks(). + * + * Multiple gauge callbacks fire during the same collection cycle. + * This atomic tracks the last time hooks were invoked (ms since epoch). + * Hooks are called at most once per 500ms window to avoid redundant + * invocations while still ensuring fresh values each collection cycle. + */ + std::atomic lastHookCallMs_{0}; +}; + +//============================================================================== +// Implementation +//============================================================================== + +//------------------------------------------------------------------------------ +// OTelHookImpl +//------------------------------------------------------------------------------ + +OTelHookImpl::OTelHookImpl(HandlerType handler, std::shared_ptr impl) + : impl_(std::move(impl)), handler_(std::move(handler)) +{ + impl_->addHook(this); +} + +OTelHookImpl::~OTelHookImpl() +{ + impl_->removeHook(this); +} + +void +OTelHookImpl::callHandler() +{ + handler_(); +} + +//------------------------------------------------------------------------------ +// OTelCounterImpl +//------------------------------------------------------------------------------ + +OTelCounterImpl::OTelCounterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter) + : counter_(meter->CreateUInt64Counter(name)) +{ +} + +void +OTelCounterImpl::increment(value_type amount) +{ + // OTel counters require non-negative values. beast::insight CounterImpl + // uses int64_t, so clamp negative values to 0 and cast to uint64_t. + if (amount > 0) + counter_->Add(static_cast(amount)); +} + +//------------------------------------------------------------------------------ +// OTelEventImpl +//------------------------------------------------------------------------------ + +OTelEventImpl::OTelEventImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter) + : histogram_(meter->CreateDoubleHistogram(name, "Duration in ms", "ms")) +{ +} + +void +OTelEventImpl::notify(value_type const& value) +{ + histogram_->Record(static_cast(value.count()), opentelemetry::context::Context{}); +} + +//------------------------------------------------------------------------------ +// OTelGaugeImpl +//------------------------------------------------------------------------------ + +OTelGaugeImpl::OTelGaugeImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter, + std::shared_ptr const& collector) + : gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector) +{ + collector_->addGauge(this); + gauge_->AddCallback(gaugeCallback, this); +} + +void +OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state) +{ + auto* self = static_cast(state); + self->collector_->callHooks(); + if (auto intResult = opentelemetry::nostd::get_if< + opentelemetry::nostd::shared_ptr>>( + &result)) + { + (*intResult)->Observe(self->currentValue()); + } +} + +OTelGaugeImpl::~OTelGaugeImpl() +{ + // RemoveCallback must run before this object is destroyed so the SDK + // collection thread cannot invoke gaugeCallback on a dangling `this`. + // The SDK's ObservableRegistry guards its callback list and the Observe() + // pass with the same mutex, so RemoveCallback cannot return while a + // callback for this instrument is in flight — removal is synchronous. + gauge_->RemoveCallback(gaugeCallback, this); + collector_->removeGauge(this); +} + +void +OTelGaugeImpl::set(value_type value) +{ + value_.store(static_cast(value), std::memory_order_relaxed); +} + +void +OTelGaugeImpl::increment(difference_type amount) +{ + // Use compare-exchange loop to safely clamp to [0, MAX]. + int64_t current = value_.load(std::memory_order_relaxed); + int64_t desired = 0; + do + { + desired = current + amount; + // Clamp to 0 on underflow. + desired = std::max(desired, int64_t{0}); + } while (!value_.compare_exchange_weak(current, desired, std::memory_order_relaxed)); +} + +int64_t +OTelGaugeImpl::currentValue() const +{ + return value_.load(std::memory_order_relaxed); +} + +//------------------------------------------------------------------------------ +// OTelMeterImpl +//------------------------------------------------------------------------------ + +OTelMeterImpl::OTelMeterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter) + : counter_(meter->CreateUInt64Counter(name)) +{ +} + +void +OTelMeterImpl::increment(value_type amount) +{ + counter_->Add(amount); +} + +//------------------------------------------------------------------------------ +// OTelCollectorImp +//------------------------------------------------------------------------------ + +OTelCollectorImp::OTelCollectorImp( + std::string const& endpoint, + std::string prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal) + : journal_(journal), prefix_(std::move(prefix)) +{ + // instanceId/serviceName/networkType are retained on the New() signature + // for back-compat but no longer used here: the telemetry module owns the + // resource attributes for the shared metrics pipeline. + (void)instanceId; + (void)serviceName; + (void)networkType; + + if (journal_.info()) + { + // endpoint is informational: the global telemetry pipeline owns the + // real exporter. It is logged here for back-compat and diagnostics. + journal_.info() << "OTelCollector starting: endpoint=" << endpoint << " prefix=" << prefix_; + } + + // Fetch the Meter from the GLOBAL MeterProvider. The telemetry module + // (xrpl::telemetry::Telemetry) builds the metrics pipeline (exporter, + // periodic reader, histogram view, resource attributes) and registers it + // via metrics::Provider::SetMeterProvider() during start(). beast metrics + // ride that shared pipeline, so both direct-API and beast-sourced metrics + // export under one resource identity. + // + // The name/version literals MUST match the telemetry module's kMeterName + // ("xrpld") and kMeterVersion ("1.0.0"). They are written as literals (not + // referenced from Telemetry.h) because beast/insight sits below the + // telemetry module in the layering and cannot include its header. + otelMeter_ = metrics_api::Provider::GetMeterProvider()->GetMeter( + std::string{"xrpld"}, std::string{"1.0.0"}); + + if (journal_.info()) + { + journal_.info() << "OTelCollector started successfully"; + } +} + +OTelCollectorImp::~OTelCollectorImp() +{ + if (journal_.info()) + { + journal_.info() << "OTelCollector shutting down"; + } + // No pipeline teardown here: the telemetry module owns the global + // MeterProvider lifecycle (ForceFlush/Shutdown happen in Telemetry::stop()). + if (journal_.info()) + { + journal_.info() << "OTelCollector stopped"; + } +} + +Hook +OTelCollectorImp::makeHook(HookImpl::HandlerType const& handler) +{ + return Hook(std::make_shared(handler, shared_from_this())); +} + +Counter +OTelCollectorImp::makeCounter(std::string const& name) +{ + return Counter(std::make_shared(formatName(name), otelMeter_)); +} + +Event +OTelCollectorImp::makeEvent(std::string const& name) +{ + return Event(std::make_shared(formatName(name), otelMeter_)); +} + +Gauge +OTelCollectorImp::makeGauge(std::string const& name) +{ + return Gauge(std::make_shared(formatName(name), otelMeter_, shared_from_this())); +} + +Meter +OTelCollectorImp::makeMeter(std::string const& name) +{ + return Meter(std::make_shared(formatName(name), otelMeter_)); +} + +void +OTelCollectorImp::addHook(OTelHookImpl* hook) +{ + std::scoped_lock const lock(mutex_); + hooks_.push_back(hook); +} + +void +OTelCollectorImp::removeHook(OTelHookImpl* hook) +{ + std::scoped_lock const lock(mutex_); + std::erase(hooks_, hook); +} + +void +OTelCollectorImp::callHooks() +{ + // Debounce: hooks run at most once per 500ms. Multiple gauge callbacks + // fire during the same collection cycle — only the first one triggers + // hooks. Subsequent callbacks within the window read already-updated + // gauge values. + auto now = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + auto last = lastHookCallMs_.load(std::memory_order_acquire); + if (now - last < 500) + return; + if (!lastHookCallMs_.compare_exchange_strong(last, now, std::memory_order_acq_rel)) + return; // Another thread won the race. + + // Copy the hook list under the lock, then invoke handlers outside it. + // A handler may drop the last reference to an OTelHookImpl, whose + // destructor calls removeHook() and re-acquires mutex_; invoking + // handlers while holding the (non-recursive) lock would deadlock. + std::vector hooks; + { + std::scoped_lock const lock(mutex_); + hooks = hooks_; + } + for (auto* hook : hooks) + hook->callHandler(); +} + +void +OTelCollectorImp::addGauge(OTelGaugeImpl* gauge) +{ + std::scoped_lock const lock(mutex_); + gauges_.push_back(gauge); +} + +void +OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge) +{ + std::scoped_lock const lock(mutex_); + std::erase(gauges_, gauge); +} + +std::string +OTelCollectorImp::formatName(std::string const& name) +{ + // Produce a clean, lowercase, Prometheus-compatible metric name. + // No prefix — the OTel resource (service.name) identifies the service. + // Dots and spaces become underscores; everything lowercased. + std::string result; + result.reserve(name.size()); + for (char const c : name) + { + if (c == '.' || c == ' ') + { + result += '_'; + } + else + { + result += static_cast(std::tolower(static_cast(c))); + } + } + return result; +} + +} // namespace detail + +//------------------------------------------------------------------------------ + +std::shared_ptr +OTelCollector::New( + std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal) +{ + return std::make_shared( + endpoint, prefix, instanceId, serviceName, networkType, journal); +} + +} // namespace beast::insight + +#else // !XRPL_ENABLE_TELEMETRY + +// When telemetry is disabled at compile time, OTelCollector::New() +// returns a NullCollector so callers do not need conditional logic. + +#include +#include +#include +#include + +namespace beast::insight { + +std::shared_ptr +OTelCollector::New( + std::string const& /* endpoint */, + std::string const& /* prefix */, + std::string const& /* instanceId */, + std::string const& /* serviceName */, + std::string const& /* networkType */, + Journal /* journal */) +{ + return NullCollector::make(); +} + +} // namespace beast::insight + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/beast/insight/StatsDCollector.cpp b/src/libxrpl/beast/insight/StatsDCollector.cpp index 3cff5d93b5c..9a14393a5e8 100644 --- a/src/libxrpl/beast/insight/StatsDCollector.cpp +++ b/src/libxrpl/beast/insight/StatsDCollector.cpp @@ -166,7 +166,7 @@ class StatsDGaugeImpl : public GaugeImpl, public StatsDMetricBase std::string name_; GaugeImpl::value_type lastValue_{0}; GaugeImpl::value_type value_{0}; - bool dirty_{false}; + bool dirty_{true}; }; //------------------------------------------------------------------------------ @@ -572,6 +572,9 @@ StatsDEventImpl::doNotify(EventImpl::value_type const& value) StatsDGaugeImpl::StatsDGaugeImpl(std::string name, std::shared_ptr impl) : impl_(std::move(impl)), name_(std::move(name)) { + // Start dirty so the initial value (0) is emitted on the first flush. + // Without this, gauges whose value never changes from 0 would never + // appear in downstream metric stores (e.g. Prometheus via StatsD). impl_->add(*this); } diff --git a/src/libxrpl/telemetry/NullTelemetry.cpp b/src/libxrpl/telemetry/NullTelemetry.cpp new file mode 100644 index 00000000000..92cc7303efd --- /dev/null +++ b/src/libxrpl/telemetry/NullTelemetry.cpp @@ -0,0 +1,149 @@ +/** + * No-op implementation of the Telemetry interface. + * + * Always compiled (regardless of XRPL_ENABLE_TELEMETRY). Provides the + * makeTelemetry() factory when telemetry is compiled out (#ifndef), which + * unconditionally returns a NullTelemetry that does nothing. + * + * When XRPL_ENABLE_TELEMETRY IS defined, the OTel virtual methods + * (getTracer, startSpan) return noop tracers/spans. The makeTelemetry() + * factory in this file is not used in that case -- Telemetry.cpp provides + * its own factory that can return the real TelemetryImpl. + */ + +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#include + +#include +#endif + +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +/** + * No-op Telemetry that returns immediately from every method. + * + * Used as the sole implementation when XRPL_ENABLE_TELEMETRY is not + * defined, or as a fallback when it is defined but enabled=0. + */ +class NullTelemetry : public Telemetry +{ + /** + * Retained configuration (unused, kept for diagnostic access). + */ + Setup const setup_; + +public: + explicit NullTelemetry(Setup setup) : setup_{std::move(setup)} + { + } + + void + start() override + { + Telemetry::setInstance(this); + } + + void + stop() override + { + Telemetry::setInstance(nullptr); + } + + [[nodiscard]] bool + isEnabled() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceRpc() const override + { + return false; + } + + [[nodiscard]] bool + shouldTracePeer() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceLedger() const override + { + return false; + } + + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + return setup_.consensusTraceStrategy; + } + +#ifdef XRPL_ENABLE_TELEMETRY + opentelemetry::nostd::shared_ptr + getTracer(std::string_view) override + { + static auto noopTracer = opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::NoopTracer()); + return noopTracer; + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view, opentelemetry::trace::SpanKind) override + { + return opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::NoopSpan(nullptr)); + } + + opentelemetry::nostd::shared_ptr + startSpan( + std::string_view, + opentelemetry::context::Context const&, + opentelemetry::trace::SpanKind) override + { + return opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::NoopSpan(nullptr)); + } +#endif +}; + +} // namespace + +/** + * Factory used when XRPL_ENABLE_TELEMETRY is not defined. + * Unconditionally returns a NullTelemetry instance. + */ +#ifndef XRPL_ENABLE_TELEMETRY +std::unique_ptr +makeTelemetry(Telemetry::Setup const& setup, beast::Journal) +{ + return std::make_unique(setup); +} +#endif + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/Redaction.cpp b/src/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 00000000000..7164c4dee62 --- /dev/null +++ b/src/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,31 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +std::string +redactAccount(std::string_view addr) +{ + // Empty in, empty out: nothing to hash and no token to emit. + if (addr.empty()) + return {}; + + // sha512Half yields a uint256; to_string renders it as uppercase hex. + // Keep the first 16 chars (64 bits) — enough to correlate spans while + // staying non-reversible — and lowercase them for a stable token. + auto const digest = sha512Half(Slice(addr.data(), addr.size())); + std::string token = to_string(digest).substr(0, 16); + std::ranges::transform( + token, token.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return token; +} + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/SpanGuard.cpp b/src/libxrpl/telemetry/SpanGuard.cpp new file mode 100644 index 00000000000..c16e48b4d98 --- /dev/null +++ b/src/libxrpl/telemetry/SpanGuard.cpp @@ -0,0 +1,596 @@ +/** + * Pimpl implementation for SpanGuard and SpanContext. + * + * All OpenTelemetry SDK types are confined to this translation unit. + * The public SpanGuard.h header contains only standard-library types + * and forward-declares the Impl struct. + * + * Static factory methods access the global Telemetry instance via + * Telemetry::getInstance(), check whether the requested TraceCategory + * is enabled, and return either an active guard with a real Span+Scope + * or a null guard whose methods are all no-ops. + * + * The Impl struct holds the OTel Span (shared_ptr) and an optional + * Scope. Scope is non-movable, but since Impl lives behind a + * unique_ptr, SpanGuard's move constructor simply transfers the + * pointer — no double-Scope issues. A scoped guard holds the Scope; + * detached() produces an Impl with no Scope (nullopt) so the guard + * carries no thread-local binding and is safe to move across threads. + * + * @see SpanGuard (SpanGuard.h), Telemetry (Telemetry.h), + * FilteringSpanProcessor (Telemetry.cpp) + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace otel_trace = opentelemetry::trace; + +// ===== SpanContext::Impl =================================================== + +struct SpanContext::Impl +{ + opentelemetry::context::Context ctx; + + explicit Impl(opentelemetry::context::Context c) : ctx(std::move(c)) + { + } +}; + +SpanContext::SpanContext(std::shared_ptr impl) : impl_(std::move(impl)) +{ +} + +bool +SpanContext::isValid() const +{ + return impl_ != nullptr; +} + +// ===== SpanGuard::Impl ==================================================== + +struct SpanGuard::Impl +{ + /** + * The OTel span being guarded. Set to nullptr after discard() or + * once detached() moves it into a new guard, so ~Impl skips End(). + */ + opentelemetry::nostd::shared_ptr span; + + /** + * Scope that activates span on the current thread's context stack. + * nullopt for a detached guard, which holds the span with no + * thread-local binding and is therefore safe to move across threads. + */ + std::optional scope; + + /** + * Construct a scoped guard: the span is pushed onto this thread's + * active-context stack for the lifetime of the guard. + * @param s The span to guard. + */ + explicit Impl(opentelemetry::nostd::shared_ptr s) : span(std::move(s)) + { + scope.emplace(span); + } + + /** + * Tag type selecting the scope-less (detached) constructor. + */ + struct Detached + { + }; + + /** + * Construct a detached guard: the span is held with NO thread-local + * Scope, so the guard carries no context-stack binding and is safe + * to move to and destroy on another thread. + * @param s The span to guard. + */ + Impl(opentelemetry::nostd::shared_ptr s, Detached) : span(std::move(s)) + { + } + + ~Impl() + { + if (span) + span->End(); + } + + Impl(Impl const&) = delete; + Impl& + operator=(Impl const&) = delete; + Impl(Impl&&) = delete; + Impl& + operator=(Impl&&) = delete; +}; + +// ===== SpanGuard core lifecycle ============================================ + +SpanGuard::SpanGuard() = default; +SpanGuard::~SpanGuard() = default; +SpanGuard::SpanGuard(SpanGuard&&) noexcept = default; + +SpanGuard::SpanGuard(std::unique_ptr impl) : impl_(std::move(impl)) +{ +} + +SpanGuard:: +operator bool() const +{ + return impl_ != nullptr; +} + +// ===== Static factory methods ============================================== + +/** + * Check whether the given TraceCategory is enabled on the Telemetry instance. + * @return true if the category's shouldTrace*() flag is on. + */ +static bool +isCategoryEnabled(Telemetry const& tel, TraceCategory cat) +{ + switch (cat) + { + case TraceCategory::Rpc: + return tel.shouldTraceRpc(); + case TraceCategory::Transactions: + return tel.shouldTraceTransactions(); + case TraceCategory::Consensus: + return tel.shouldTraceConsensus(); + case TraceCategory::Peer: + return tel.shouldTracePeer(); + case TraceCategory::Ledger: + return tel.shouldTraceLedger(); + } + return false; // unreachable, silences compiler warning +} + +namespace { + +// Span-link attribute marking a "follows-from" (causal, non-parent) link, +// emitted by both linkedSpan() overloads. Custom xrpl attribute — not part +// of the OTel semantic conventions, so defined here rather than pulled from +// . +constexpr char const* kLinkTypeKey = "link_type"; +constexpr char const* kLinkTypeFollowsFrom = "follows_from"; + +// Map a TraceCategory to an OTel SpanKind so Tempo's service-graph / +// RED metrics see the correct direction. RPC spans are emitted at the +// server entry point (handler dispatch), Peer spans at inbound-message +// receipt. Transactions / Consensus / Ledger are internal processing +// and keep the default kInternal. +otel_trace::SpanKind +categoryToSpanKind(TraceCategory cat) +{ + switch (cat) + { + case TraceCategory::Rpc: + return otel_trace::SpanKind::kServer; + case TraceCategory::Peer: + return otel_trace::SpanKind::kConsumer; + case TraceCategory::Transactions: + case TraceCategory::Consensus: + case TraceCategory::Ledger: + return otel_trace::SpanKind::kInternal; + } + return otel_trace::SpanKind::kInternal; // unreachable +} + +} // namespace + +SpanGuard +SpanGuard::span(TraceCategory cat, std::string_view prefix, std::string_view name) +{ + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + std::string fullName; + fullName.reserve(prefix.size() + 1 + name.size()); + fullName.append(prefix).append(1, '.').append(name); + return SpanGuard(std::make_unique(tel->startSpan(fullName, categoryToSpanKind(cat)))); +} + +SpanGuard +SpanGuard::rootSpan(TraceCategory cat, std::string_view prefix, std::string_view name) +{ + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + std::string fullName; + fullName.reserve(prefix.size() + 1 + name.size()); + fullName.append(prefix).append(1, '.').append(name); + // Force a fresh trace root: do NOT inherit this thread's active span. + auto rootCtx = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + return SpanGuard( + std::make_unique(tel->startSpan(fullName, rootCtx, categoryToSpanKind(cat)))); +} + +// ===== Child / linked span creation ======================================== + +SpanGuard +SpanGuard::childSpan(std::string_view name) const +{ + if (!impl_) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + auto ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + return SpanGuard(std::make_unique(tel->startSpan(name, ctx))); +} + +SpanGuard +SpanGuard::childSpan(std::string_view name, SpanContext const& parentCtx) +{ + if (!parentCtx.isValid()) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + return SpanGuard(std::make_unique(tel->startSpan(name, parentCtx.impl_->ctx))); +} + +SpanGuard +SpanGuard::linkedSpan(std::string_view name) const +{ + if (!impl_) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + + auto tracer = tel->getTracer(); + auto spanCtx = impl_->span->GetContext(); + + // Mark as root span so it starts a new trace sub-tree rather than + // inheriting the current thread's active span as parent. + otel_trace::StartSpanOptions opts; + opts.parent = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + + // LCOV_EXCL_START + return SpanGuard( + std::make_unique(tracer->StartSpan( + std::string(name), {}, {{spanCtx, {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, opts))); + // LCOV_EXCL_STOP +} + +SpanGuard +SpanGuard::linkedSpan(std::string_view name, SpanContext const& linkCtx) +{ + if (!linkCtx.isValid()) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + + auto tracer = tel->getTracer(); + + // Extract the span from the captured context to get its SpanContext. + auto linkSpan = otel_trace::GetSpan(linkCtx.impl_->ctx); + if (!linkSpan || !linkSpan->GetContext().IsValid()) + return {}; + + // Mark as root span so it starts a new trace sub-tree rather than + // inheriting the current thread's active span as parent. + otel_trace::StartSpanOptions opts; + opts.parent = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + + // LCOV_EXCL_START + return SpanGuard( + std::make_unique(tracer->StartSpan( + std::string(name), + {}, + {{linkSpan->GetContext(), {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, + opts))); + // LCOV_EXCL_STOP +} + +SpanGuard +SpanGuard::detached() && +{ + if (!impl_) + return {}; + // Take the span out; the old Impl.span is now null so ~Impl won't End(). + auto s = std::move(impl_->span); + // Resetting the old Impl destroys its Scope HERE, on the origin thread, + // popping this thread's context stack correctly. The returned guard holds + // the span with no Scope, so it is safe to move to another thread. + impl_.reset(); + return SpanGuard(std::make_unique(std::move(s), Impl::Detached{})); +} + +// ===== Hash-derived span (category-gated) ================================== + +SpanGuard +SpanGuard::hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + SpanContext const* const followsFrom) +{ + if (hashSize < 16) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + + otel_trace::TraceId const traceId( + opentelemetry::nostd::span(hashData, 16)); + + auto const rval = defaultPrng()(); + std::uint8_t spanIdBytes[8]; + std::memcpy(spanIdBytes, &rval, sizeof(spanIdBytes)); + otel_trace::SpanId const spanId( + opentelemetry::nostd::span(spanIdBytes, 8)); + + otel_trace::SpanContext const syntheticCtx( + traceId, spanId, otel_trace::TraceFlags(1), /* remote = */ false); + + auto parentCtx = opentelemetry::context::Context{}.SetValue( + otel_trace::kSpanKey, + opentelemetry::nostd::shared_ptr( + new otel_trace::DefaultSpan(syntheticCtx))); + + if (followsFrom != nullptr && followsFrom->isValid()) + { + auto linkSpan = otel_trace::GetSpan(followsFrom->impl_->ctx); + if (linkSpan && linkSpan->GetContext().IsValid()) + { + auto tracer = tel->getTracer("xrpld"); + otel_trace::StartSpanOptions opts; + opts.parent = parentCtx; + opts.kind = categoryToSpanKind(cat); + return SpanGuard( + std::make_unique(tracer->StartSpan( + std::string(name), + {}, + {{linkSpan->GetContext(), {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, + opts))); + } + } + + return SpanGuard(std::make_unique(tel->startSpan(std::string(name), parentCtx))); +} + +SpanGuard +SpanGuard::hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + std::uint8_t const* const parentSpanId, + std::size_t const parentSpanSize, + std::uint8_t const traceFlags) +{ + if (hashSize < 16 || parentSpanSize != 8) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + + otel_trace::TraceId const traceId( + opentelemetry::nostd::span(hashData, 16)); + + otel_trace::SpanId const parentSpan( + opentelemetry::nostd::span(parentSpanId, 8)); + + otel_trace::SpanContext const combinedCtx( + traceId, + parentSpan, + otel_trace::TraceFlags(traceFlags), + /* remote = */ true); + + auto parentCtx = opentelemetry::context::Context{}.SetValue( + otel_trace::kSpanKey, + opentelemetry::nostd::shared_ptr( + new otel_trace::DefaultSpan(combinedCtx))); + + return SpanGuard(std::make_unique(tel->startSpan(std::string(name), parentCtx))); +} + +// ===== Context capture ===================================================== + +SpanContext +SpanGuard::captureContext() const +{ + if (!impl_) + return {}; + auto ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + return SpanContext(std::make_shared(std::move(ctx))); +} + +TraceBytes +SpanGuard::getTraceBytes() const +{ + if (!impl_ || !impl_->span) + return {}; + + auto const& spanCtx = impl_->span->GetContext(); + if (!spanCtx.IsValid()) + return {}; + + TraceBytes result; + auto const& tid = spanCtx.trace_id(); + std::memcpy(result.traceId.data(), tid.Id().data(), 16); + auto const& sid = spanCtx.span_id(); + std::memcpy(result.spanId.data(), sid.Id().data(), 8); + result.traceFlags = spanCtx.trace_flags().flags(); + result.valid = true; + return result; +} + +void +SpanGuard::injectCurrentContextToProtobuf(protocol::TraceContext& proto) +{ + auto ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + injectToProtobuf(ctx, proto); +} + +// ===== Attribute setters =================================================== + +void +SpanGuard::setAttribute(std::string_view key, std::string_view value) +{ + if (impl_) + { + impl_->span->SetAttribute( + opentelemetry::nostd::string_view(key.data(), key.size()), + opentelemetry::nostd::string_view(value.data(), value.size())); + } +} + +void +SpanGuard::setAttribute(std::string_view key, char const* value) +{ + setAttribute(key, std::string_view(value)); +} + +void +SpanGuard::setAttribute(std::string_view key, std::int64_t value) +{ + if (impl_) + impl_->span->SetAttribute(opentelemetry::nostd::string_view(key.data(), key.size()), value); +} + +void +SpanGuard::setAttribute(std::string_view key, double value) +{ + if (impl_) + impl_->span->SetAttribute(opentelemetry::nostd::string_view(key.data(), key.size()), value); +} + +void +SpanGuard::setAttribute(std::string_view key, bool value) +{ + if (impl_) + impl_->span->SetAttribute(opentelemetry::nostd::string_view(key.data(), key.size()), value); +} + +// ===== Status / events ===================================================== + +void +SpanGuard::setOk() +{ + if (impl_) + impl_->span->SetStatus(otel_trace::StatusCode::kOk); +} + +void +SpanGuard::setError(std::string_view description) +{ + if (impl_) + impl_->span->SetStatus(otel_trace::StatusCode::kError, std::string(description)); +} + +void +SpanGuard::addEvent(std::string_view name) +{ + if (impl_) + impl_->span->AddEvent(std::string(name)); +} + +void +SpanGuard::addEvent(std::string_view name, std::initializer_list attrs) +{ + if (!impl_) + return; + // OTel's AddEvent template requires a key-value-iterable; a plain + // std::vector> doesn't satisfy is_key_value_iterable. + // Wrap in nostd::span over the vector's storage so the SDK accepts it. + std::vector> + + otelAttrs; + otelAttrs.reserve(attrs.size()); + for (auto const& [k, v] : attrs) + { + otelAttrs.emplace_back( + opentelemetry::nostd::string_view{k.data(), k.size()}, + opentelemetry::common::AttributeValue{ + opentelemetry::nostd::string_view{v.data(), v.size()}}); + } + impl_->span->AddEvent( + std::string(name), + opentelemetry::nostd::span const>{otelAttrs.data(), otelAttrs.size()}); +} + +void +SpanGuard::recordException(std::exception const& e) +{ + if (!impl_) + return; + namespace semconv_exc = opentelemetry::semconv::exception; + // Event name "exception" and the attribute keys follow the OTel semantic + // conventions; the keys come from semconv constants rather than literals. + impl_->span->AddEvent( + "exception", + {{semconv_exc::kExceptionType, typeid(e).name()}, + {semconv_exc::kExceptionMessage, std::string(e.what())}}); + impl_->span->SetStatus(otel_trace::StatusCode::kError, e.what()); +} + +void +SpanGuard::discard() +{ + if (impl_) + { + { + // DiscardScope owns the flag's whole lifetime: it sets the flag, + // and clears it on scope exit — even if End() were to throw. The + // SDK invokes FilteringSpanProcessor::OnEnd() synchronously from + // End() on this thread, so the flag is observed while still set. + // Today every valid guard wraps a recording span (head sampling is + // 1.0), so OnEnd() always runs — but scoping set/clear keeps the + // flag leak-proof if a later phase can hand back a non-recording + // span (e.g. honoring a non-sampled remote parent during + // propagation), so it can never spill onto the next span. + DiscardScope const discardScope; + impl_->span->End(); + } + impl_->span = nullptr; // prevent ~Impl from calling End() again + impl_.reset(); + } +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/telemetry/Telemetry.cpp b/src/libxrpl/telemetry/Telemetry.cpp new file mode 100644 index 00000000000..9e877c3598d --- /dev/null +++ b/src/libxrpl/telemetry/Telemetry.cpp @@ -0,0 +1,642 @@ +/** + * OpenTelemetry SDK implementation of the Telemetry interface. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake + * telemetry=ON). Contains: + * + * - FilteringSpanProcessor: decorator that drops spans marked with + * kDiscardedAttr before they enter the batch export queue. + * - TelemetryImpl: configures the OTel SDK with an OTLP/HTTP exporter, + * FilteringSpanProcessor wrapping a batch span processor, + * trace-ID-ratio sampler, and resource attributes. + * - NullTelemetryOtel: no-op fallback used when telemetry is compiled in + * but disabled at runtime (enabled=0 in config). + * - makeTelemetry(): factory that selects the appropriate implementation. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +namespace trace_api = opentelemetry::trace; +namespace trace_sdk = opentelemetry::sdk::trace; +namespace metrics_api = opentelemetry::metrics; +namespace metrics_sdk = opentelemetry::sdk::metrics; +namespace otlp_http = opentelemetry::exporter::otlp; +namespace resource = opentelemetry::sdk::resource; + +/** + * SpanProcessor decorator that drops discarded spans. + * + * Wraps a delegate processor (typically BatchSpanProcessor). In OnEnd(), + * calls DiscardScope::isActive(). If the calling thread is inside a + * DiscardScope (entered by SpanGuard::discard()), the span is silently + * dropped — never entering the batch queue, never sent over the network, + * never stored. + * + * Uses a thread-local flag rather than inspecting Recordable attributes + * because the Recordable type varies by exporter (SpanData for simple + * exporters, OtlpRecordable for OTLP) and none expose a uniform getter. + * The flag is safe because Span::End() calls OnEnd() synchronously on + * the same thread. + * + * All other methods delegate directly to the wrapped processor. + * + * Dependency diagram: + * + * +---------------------------+ + * | FilteringSpanProcessor | + * +---------------------------+ + * | - delegate_ : unique_ptr | + * | | + * +---------------------------+ + * | wraps + * +---------+-----------+ + * | BatchSpanProcessor | + * +---------------------+ + * + * @note Thread safety: OnEnd() may be called concurrently from multiple + * threads. The discard flag behind DiscardScope is thread-local, so each + * thread's discard state is independent — no synchronization needed. + */ +class FilteringSpanProcessor : public trace_sdk::SpanProcessor +{ + std::unique_ptr delegate_; + +public: + explicit FilteringSpanProcessor(std::unique_ptr delegate) + : delegate_(std::move(delegate)) + { + } + + std::unique_ptr + MakeRecordable() noexcept override + { + return delegate_->MakeRecordable(); + } + + void + OnStart( + trace_sdk::Recordable& span, + opentelemetry::trace::SpanContext const& parentContext) noexcept override + { + delegate_->OnStart(span, parentContext); + } + + void + OnEnd(std::unique_ptr&& span) noexcept override + { + if (DiscardScope::isActive()) + { + // SpanGuard::discard() is inside a DiscardScope on this thread, + // which it entered just before calling Span::End() — and End() + // invokes OnEnd() synchronously. Drop the span. + return; + } + delegate_->OnEnd(std::move(span)); + } + + bool + ForceFlush( + std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept override + { + return delegate_->ForceFlush(timeout); + } + + bool + Shutdown(std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept override + { + return delegate_->Shutdown(timeout); + } +}; + +/** + * No-op implementation used when XRPL_ENABLE_TELEMETRY is defined but + * setup.enabled is false at runtime. + * + * Lives in the anonymous namespace so there is no ODR conflict with the + * NullTelemetry in NullTelemetry.cpp. + */ +class NullTelemetryOtel : public Telemetry +{ + /** + * Retained configuration (unused, kept for diagnostic access). + */ + Setup const setup_; + +public: + explicit NullTelemetryOtel(Setup setup) : setup_(std::move(setup)) + { + } + + void + start() override + { + Telemetry::setInstance(this); + } + + void + stop() override + { + Telemetry::setInstance(nullptr); + } + + [[nodiscard]] bool + isEnabled() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceRpc() const override + { + return false; + } + + [[nodiscard]] bool + shouldTracePeer() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceLedger() const override + { + return false; + } + + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + return setup_.consensusTraceStrategy; + } + + opentelemetry::nostd::shared_ptr + getTracer(std::string_view) override + { + static auto noopTracer = + opentelemetry::nostd::shared_ptr(new trace_api::NoopTracer()); + return noopTracer; + } + + opentelemetry::nostd::shared_ptr + getMeter(std::string_view name) override + { + // Serve a meter from a process-wide noop provider, mirroring the + // noop tracer above. Instruments created from it are inert. + static auto noopProvider = opentelemetry::nostd::shared_ptr( + new metrics_api::NoopMeterProvider()); + return noopProvider->GetMeter(std::string(name), std::string(kMeterVersion)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view, trace_api::SpanKind) override + { + return opentelemetry::nostd::shared_ptr(new trace_api::NoopSpan(nullptr)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view, opentelemetry::context::Context const&, trace_api::SpanKind) + override + { + return opentelemetry::nostd::shared_ptr(new trace_api::NoopSpan(nullptr)); + } +}; + +/** + * Full OTel SDK implementation that exports trace spans via OTLP/HTTP. + * + * Configures an OTLP/HTTP exporter, batch span processor, + * TraceIdRatioBasedSampler, and resource attributes on start(). + */ +class TelemetryImpl : public Telemetry +{ + /** + * Configuration from the [telemetry] config section. + * Non-const so setServiceInstanceId() can update the instance ID + * before start() creates the OTel resource. + */ + Setup setup_; + + /** + * Journal used for log output during start/stop. + */ + beast::Journal const journal_; + + /** + * The SDK TracerProvider that owns the export pipeline. + * + * Held as std::shared_ptr so we can call ForceFlush() on shutdown. + * Wrapped in a nostd::shared_ptr when registered as the global provider. + */ + std::shared_ptr sdkProvider_; + + /** + * The SDK MeterProvider that owns the metric export pipeline. + * + * Symmetric with sdkProvider_ on the tracing side. Held as + * std::shared_ptr so we can ForceFlush() on shutdown; wrapped in a + * nostd::shared_ptr when registered as the global meter provider. + */ + std::shared_ptr meterProvider_; + +public: + TelemetryImpl(Setup setup, beast::Journal journal) : setup_(std::move(setup)), journal_(journal) + { + // Build the metrics pipeline NOW, in the constructor, so the global + // MeterProvider is published before any subsystem is constructed. + // beast::insight instruments are created eagerly in subsystem + // constructors (e.g. LedgerMaster, NetworkOPs, ServerHandler), which + // run during ApplicationImp's member-init list — long before start(). + // opentelemetry-cpp has no proxy MeterProvider, so an instrument + // created before SetMeterProvider() binds to the noop provider forever. + // Tracing does not have this problem because getTracer() is called + // fresh at each span creation (runtime, after start()). + // + // The metrics resource uses setup_.serviceInstanceId as provided by + // config. A later setServiceInstanceId() (node-key fallback) cannot + // change this immutable resource, so operators relying on the node-key + // identity should set [telemetry] service_instance_id explicitly. + initMetrics(); + } + + void + setServiceInstanceId(std::string const& id) override + { + setup_.serviceInstanceId = id; + } + + void + start() override + { + JLOG(journal_.info()) << "Telemetry starting: endpoint=" << setup_.exporterEndpoint + << " sampling=" << setup_.samplingRatio; + + // Configure OTLP HTTP exporter + otlp_http::OtlpHttpExporterOptions exporterOpts; + exporterOpts.url = setup_.exporterEndpoint; + if (setup_.useTls) + { + exporterOpts.ssl_ca_cert_path = setup_.tlsCertPath; + // Present a client cert for mutual TLS. When both paths are + // empty the connection falls back to one-way (server) TLS. + exporterOpts.ssl_client_cert_path = setup_.tlsClientCertPath; + exporterOpts.ssl_client_key_path = setup_.tlsClientKeyPath; + } + + auto exporter = otlp_http::OtlpHttpExporterFactory::Create(exporterOpts); + + // Configure batch processor + trace_sdk::BatchSpanProcessorOptions processorOpts; + processorOpts.max_queue_size = setup_.maxQueueSize; + processorOpts.schedule_delay_millis = std::chrono::milliseconds(setup_.batchDelay); + processorOpts.max_export_batch_size = setup_.batchSize; + + auto batchProcessor = + trace_sdk::BatchSpanProcessorFactory::Create(std::move(exporter), processorOpts); + + // Wrap batch processor with filtering processor that drops spans + // marked with kDiscardedAttr (via SpanGuard::discard()). + auto processor = std::make_unique(std::move(batchProcessor)); + + // Configure resource attributes + auto resourceAttrs = resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, + {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, + {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, + {std::string(attr::networkId), + static_cast(setup_.networkId)}, // LCOV_EXCL_LINE + {std::string(attr::networkType), setup_.networkType}, // LCOV_EXCL_LINE + }); + + // Configure sampler. Head sampling is fixed at 1.0 (sample everything); + // setup_.samplingRatio is not config-driven. Wrap the ratio sampler in a + // ParentBasedSampler so spans with a remote parent honor the upstream + // sampled flag — this keeps keep/drop decisions coherent for a single + // distributed trace spanning multiple nodes. Volume reduction is left to + // the collector's tail sampling. + auto rootSampler = + std::make_shared(setup_.samplingRatio); + auto sampler = trace_sdk::ParentBasedSamplerFactory::Create(std::move(rootSampler)); + + // Create TracerProvider + sdkProvider_ = trace_sdk::TracerProviderFactory::Create( + std::move(processor), resourceAttrs, std::move(sampler)); + + // Set as global provider + trace_api::Provider::SetTracerProvider( + opentelemetry::nostd::shared_ptr(sdkProvider_)); + + // The metrics pipeline (meterProvider_) was already built and published + // in the constructor via initMetrics(), before any subsystem could + // create a beast::insight instrument. See initMetrics(). + + // Register as the global Telemetry instance so SpanGuard factory + // methods can access it without callers passing a reference. + Telemetry::setInstance(this); + + JLOG(journal_.info()) << "Telemetry started successfully"; + } + + /** + * Build and publish the metrics pipeline (MeterProvider + periodic + * reader + OTLP exporter + histogram view). + * + * Called from the constructor, NOT start(), so the global MeterProvider + * exists before subsystems construct their beast::insight instruments + * during ApplicationImp's member-init list. The metrics resource uses + * setup_.serviceInstanceId from config; it is immutable once the provider + * is built, so a later node-key setServiceInstanceId() does not affect it. + */ + void + initMetrics() + { + // Derive the metrics endpoint from the trace endpoint by swapping + // the trailing "/v1/traces" path for "/v1/metrics". Any other URL + // shape is used as-is. + std::string metricsEndpoint = setup_.exporterEndpoint; + constexpr std::string_view tracesPath{"/v1/traces"}; + if (metricsEndpoint.ends_with(tracesPath)) + { + metricsEndpoint.replace( + metricsEndpoint.size() - tracesPath.size(), tracesPath.size(), "/v1/metrics"); + } + + // Configure OTLP HTTP metric exporter, honoring the same TLS + // options as the trace exporter. + otlp_http::OtlpHttpMetricExporterOptions metricExporterOpts; + metricExporterOpts.url = metricsEndpoint; + if (setup_.useTls) + { + metricExporterOpts.ssl_ca_cert_path = setup_.tlsCertPath; + metricExporterOpts.ssl_client_cert_path = setup_.tlsClientCertPath; + metricExporterOpts.ssl_client_key_path = setup_.tlsClientKeyPath; + } + + auto metricExporter = otlp_http::OtlpHttpMetricExporterFactory::Create(metricExporterOpts); + + // Configure periodic metric reader (1-second export interval, + // matching the beast OTelCollector path). + metrics_sdk::PeriodicExportingMetricReaderOptions readerOpts; + readerOpts.export_interval_millis = std::chrono::milliseconds(1000); + readerOpts.export_timeout_millis = std::chrono::milliseconds(500); + + auto reader = metrics_sdk::PeriodicExportingMetricReaderFactory::Create( + std::move(metricExporter), readerOpts); + + // Metrics resource: same attributes as the tracer resource so metrics + // and traces share one identity. Built here (not shared with start()) + // because start() runs later; serviceInstanceId comes from config. + auto resourceAttrs = resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, + {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, + {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, + {std::string(attr::networkId), static_cast(setup_.networkId)}, + {std::string(attr::networkType), setup_.networkType}, + }); + + // Create MeterProvider with the shared resource, then attach reader. + meterProvider_ = metrics_sdk::MeterProviderFactory::Create( + std::make_unique(), resourceAttrs); + meterProvider_->AddMetricReader(std::move(reader)); + + // Histogram view: SpanMetrics-compatible bucket boundaries (ms) so + // histogram instruments align with the collector's SpanMetrics. The + // view is created with an EMPTY name so it applies the buckets WITHOUT + // renaming instruments — a non-empty view name would collapse every + // matching histogram (ios_latency, rpc_size, rpc_time, pathfind_*) + // into a single series under that one name. + auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create( + metrics_sdk::InstrumentType::kHistogram, "*", "ms"); + // Meter selector MUST match the meter name used by getMeter() and the + // beast OTelCollector (kMeterName = "xrpld"); otherwise this histogram + // view never applies and duration histograms fall back to the SDK + // default boundaries instead of these SpanMetrics-aligned buckets. + auto meterSelector = + metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", ""); + auto histogramConfig = std::make_shared(); + histogramConfig->boundaries_ = + std::vector{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0}; + auto histogramView = metrics_sdk::ViewFactory::Create( + "", // empty name: keep each instrument's own name, only set buckets + "SpanMetrics-compatible histogram buckets", + metrics_sdk::AggregationType::kHistogram, + histogramConfig); + + meterProvider_->AddView( + std::move(histogramSelector), std::move(meterSelector), std::move(histogramView)); + + // Publish as the global meter provider so developers (and the beast + // OTelCollector shim) reach the same pipeline. + metrics_api::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(meterProvider_)); + } + + void + stop() override + { + JLOG(journal_.info()) << "Telemetry stopping"; + + // Unregister global instance before tearing down the pipeline. + Telemetry::setInstance(nullptr); + + if (sdkProvider_) + { + // Force flush with timeout to avoid blocking indefinitely + // when the OTLP endpoint is unreachable. + sdkProvider_->ForceFlush(std::chrono::milliseconds(5000)); + // TODO: sdkProvider_ is not thread-safe. This reset() races with + // getTracer() if any thread is still calling startSpan(). + // Currently safe because Application::stop() shuts down + // serverHandler_, overlay_, and jobQueue_ before calling + // telemetry_->stop() — so no callers should remain. If the + // shutdown order ever changes, add an std::atomic stopped_ + // flag checked in getTracer() to make this robust. + sdkProvider_.reset(); + trace_api::Provider::SetTracerProvider( + opentelemetry::nostd::shared_ptr( + new trace_api::NoopTracerProvider())); + } + + if (meterProvider_) + { + // Mirror the tracer teardown: bounded flush, drop our provider, + // then restore a noop global provider. Same shutdown-order + // caveat as sdkProvider_ above applies to getMeter() callers. + meterProvider_->ForceFlush(std::chrono::milliseconds(5000)); + meterProvider_.reset(); + metrics_api::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr( + new metrics_api::NoopMeterProvider())); + } + JLOG(journal_.info()) << "Telemetry stopped"; + } + + [[nodiscard]] bool + isEnabled() const override + { + return true; + } + + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return setup_.traceTransactions; + } + + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return setup_.traceConsensus; + } + + [[nodiscard]] bool + shouldTraceRpc() const override + { + return setup_.traceRpc; + } + + [[nodiscard]] bool + shouldTracePeer() const override + { + return setup_.tracePeer; + } + + [[nodiscard]] bool + shouldTraceLedger() const override + { + return setup_.traceLedger; + } + + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + return setup_.consensusTraceStrategy; + } + + opentelemetry::nostd::shared_ptr + getTracer(std::string_view name = kTracerName) override + { + if (!sdkProvider_) + { + return trace_api::Provider::GetTracerProvider()->GetTracer(std::string(name)); + } + return sdkProvider_->GetTracer(std::string(name)); + } + + opentelemetry::nostd::shared_ptr + getMeter(std::string_view name = kMeterName) override + { + if (!meterProvider_) + { + return metrics_api::Provider::GetMeterProvider()->GetMeter( + std::string(name), std::string(kMeterVersion)); + } + return meterProvider_->GetMeter(std::string(name), std::string(kMeterVersion)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view name, trace_api::SpanKind kind) override + { + auto tracer = getTracer(); + trace_api::StartSpanOptions opts; + opts.kind = kind; + return tracer->StartSpan(std::string(name), opts); + } + + opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::context::Context const& parentContext, + trace_api::SpanKind kind) override + { + auto tracer = getTracer(); + trace_api::StartSpanOptions opts; + opts.kind = kind; + opts.parent = parentContext; + return tracer->StartSpan(std::string(name), opts); + } +}; + +} // namespace + +std::unique_ptr +makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal) +{ + if (setup.enabled) + { + return std::make_unique(setup, journal); + } + return std::make_unique(setup); +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/telemetry/TelemetryConfig.cpp b/src/libxrpl/telemetry/TelemetryConfig.cpp new file mode 100644 index 00000000000..d122393e51a --- /dev/null +++ b/src/libxrpl/telemetry/TelemetryConfig.cpp @@ -0,0 +1,161 @@ +/** + * Parser for the [telemetry] section of xrpld.cfg. + * + * Reads configuration values from the config file and populates a + * Telemetry::Setup struct. All options have sensible defaults so the + * section can be minimal or omitted entirely. + * + * See cfg/xrpld-example.cfg for the full list of available options. + */ + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +/** + * Config key names for the [telemetry] section. + * + * Each must match the corresponding option documented in + * cfg/xrpld-example.cfg verbatim. Defined as `char const*` so they + * pass to Section::valueOr() (which takes `std::string const&`) + * without an explicit conversion, exactly as a literal would. + */ +namespace key { +constexpr char const* enabled = "enabled"; +constexpr char const* serviceName = "service_name"; +constexpr char const* serviceInstanceId = "service_instance_id"; +constexpr char const* endpoint = "endpoint"; +constexpr char const* useTls = "use_tls"; +constexpr char const* tlsCaCert = "tls_ca_cert"; +constexpr char const* tlsClientCert = "tls_client_cert"; +constexpr char const* tlsClientKey = "tls_client_key"; +constexpr char const* batchSize = "batch_size"; +constexpr char const* batchDelayMs = "batch_delay_ms"; +constexpr char const* maxQueueSize = "max_queue_size"; +constexpr char const* traceTransactions = "trace_transactions"; +constexpr char const* traceConsensus = "trace_consensus"; +constexpr char const* traceRpc = "trace_rpc"; +constexpr char const* tracePeer = "trace_peer"; +constexpr char const* traceLedger = "trace_ledger"; +} // namespace key + +/** + * Default values applied when a key is absent from the config. + * + * @note serviceName mirrors SystemParameters' systemName() ("xrpld") but + * is duplicated here as a literal: the telemetry module deliberately does + * not link xrpl.libxrpl.protocol, so including SystemParameters.h would + * introduce an undeclared cross-module dependency. + */ +namespace dflt { +constexpr char const* serviceName = "xrpld"; +constexpr char const* endpoint = "http://localhost:4318/v1/traces"; +constexpr std::uint32_t batchSize = 512u; +constexpr std::uint32_t batchDelayMs = 5000u; +constexpr std::uint32_t maxQueueSize = 2048u; +} // namespace dflt + +} // namespace + +/** + * Derive a human-readable network type label from the numeric network ID. + * + * Declared in Telemetry.h; shared by the trace and metric export paths so + * both stamp the same xrpl.network.type resource attribute value. + * + * @param networkId The network identifier from [network_id] config. + * @return "mainnet", "testnet", "devnet", or "unknown" for other values. + */ +std::string +networkTypeFromId(std::uint32_t networkId) +{ + switch (networkId) + { + case 0: + return "mainnet"; + case 1: + return "testnet"; + case 2: + return "devnet"; + default: + return "unknown"; + } +} + +Telemetry::Setup +makeTelemetrySetup( + Section const& section, + std::string const& nodePublicKey, + std::string const& version, + std::uint32_t networkId) +{ + Telemetry::Setup setup; + + setup.enabled = section.valueOr(key::enabled, 0) != 0; + setup.serviceName = section.valueOr(key::serviceName, dflt::serviceName); + setup.serviceVersion = version; + setup.serviceInstanceId = section.valueOr(key::serviceInstanceId, nodePublicKey); + + setup.exporterEndpoint = section.valueOr(key::endpoint, dflt::endpoint); + + setup.useTls = section.valueOr(key::useTls, 0) != 0; + setup.tlsCertPath = section.valueOr(key::tlsCaCert, ""); + setup.tlsClientCertPath = section.valueOr(key::tlsClientCert, ""); + setup.tlsClientKeyPath = section.valueOr(key::tlsClientKey, ""); + + // Mutual TLS needs both the client certificate and its private key. + // Supplying only one fails later with a cryptic SSL handshake error, so + // reject the partial configuration here with an actionable message. + if (setup.tlsClientCertPath.empty() != setup.tlsClientKeyPath.empty()) + { + Throw( + "[telemetry] tls_client_cert and tls_client_key must be set together " + "(set both for mutual TLS, or neither for one-way TLS)."); + } + + // Mutual TLS only takes effect when TLS is on. Certificate paths set with + // use_tls=0 would be silently ignored and the exporter would connect in + // plaintext, so reject that contradiction instead of failing open. + if (!setup.tlsClientCertPath.empty() && !setup.useTls) + { + Throw( + "[telemetry] tls_client_cert/tls_client_key require use_tls=1 " + "(set use_tls=1 to enable mutual TLS, or remove the cert paths)."); + } + + // Head sampling is intentionally fixed at 1.0 (sample everything) and is + // not read from config. A per-node ratio would let nodes make divergent + // keep/drop decisions for the same distributed trace, producing broken + // traces; volume reduction is delegated to the collector's tail sampling. + // setup.samplingRatio is a const member fixed at 1.0; nothing to parse. + + setup.batchSize = section.valueOr(key::batchSize, dflt::batchSize); + setup.batchDelay = std::chrono::milliseconds{ + section.valueOr(key::batchDelayMs, dflt::batchDelayMs)}; + setup.maxQueueSize = section.valueOr(key::maxQueueSize, dflt::maxQueueSize); + + setup.networkId = networkId; + setup.networkType = networkTypeFromId(networkId); + + setup.traceTransactions = section.valueOr(key::traceTransactions, 1) != 0; + setup.traceConsensus = section.valueOr(key::traceConsensus, 1) != 0; + setup.traceRpc = section.valueOr(key::traceRpc, 1) != 0; + setup.tracePeer = section.valueOr(key::tracePeer, 1) != 0; + setup.traceLedger = section.valueOr(key::traceLedger, 1) != 0; + + setup.consensusTraceStrategy = + section.valueOr("consensus_trace_strategy", "deterministic"); + + return setup; +} + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/tx/Transactor.cpp b/src/libxrpl/tx/Transactor.cpp index 4b562692d75..6a4eb6f3bb1 100644 --- a/src/libxrpl/tx/Transactor.cpp +++ b/src/libxrpl/tx/Transactor.cpp @@ -34,13 +34,16 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include #include #include @@ -1591,6 +1594,20 @@ Transactor::checkInvariants(TER result, XRPAmount fee) ApplyResult Transactor::operator()() { + // Derive the trace_id from the transaction id so this apply-stage span + // shares one trace with the preflight and preclaim spans (which also use + // hashSpan on the same id). + auto const txID = ctx_.tx.getTransactionID(); + auto span = telemetry::SpanGuard::hashSpan( + telemetry::TraceCategory::Transactions, + telemetry::tx_apply_span::transactor, + txID.data(), + txID.kBytes); + // "apply" — the third apply-pipeline stage, after preflight and preclaim. + span.setAttribute(telemetry::tx_apply_span::attr::stage, telemetry::tx_apply_span::val::apply); + if (auto const* fmt = TxFormats::getInstance().findByType(ctx_.tx.getTxnType())) + span.setAttribute(telemetry::tx_apply_span::attr::txType, fmt->getName().c_str()); + JLOG(j_.trace()) << "apply: " << ctx_.tx.getTransactionID(); // These global updates really should have been for every Transaction @@ -1714,6 +1731,14 @@ Transactor::operator()() JLOG(j_.trace()) << (applied ? "applied " : "not applied ") << transToken(result); + span.setAttribute(telemetry::tx_apply_span::attr::terResult, transToken(result).c_str()); + span.setAttribute(telemetry::tx_apply_span::attr::applied, applied); + // Mark the span as errored when the transaction was not applied or the + // engine result is not a success, so failed applies surface in span-status + // error counts alongside preflight and preclaim. + if (!applied || !isTesSuccess(result)) + span.setError(transToken(result)); + return {result, applied, metadata}; } diff --git a/src/libxrpl/tx/applySteps.cpp b/src/libxrpl/tx/applySteps.cpp index 5af4f621a79..f982f447e35 100644 --- a/src/libxrpl/tx/applySteps.cpp +++ b/src/libxrpl/tx/applySteps.cpp @@ -12,13 +12,16 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include #include #pragma push_macro("TRANSACTION") #undef TRANSACTION @@ -50,6 +53,49 @@ struct UnknownTxnType : std::exception } }; +/** + * Look up the human-readable transaction type name for span attributes. + * Returns nullptr if the type is unknown so the caller can skip the + * attribute rather than emit an empty value. + */ +char const* +txTypeName(TxType txnType) +{ + if (auto const* fmt = TxFormats::getInstance().findByType(txnType)) + return fmt->getName().c_str(); + return nullptr; +} + +/** + * Create a deterministic-trace span for an apply-pipeline stage. + * + * The trace_id is derived from txID[0:16] so the preflight, preclaim, and + * transactor spans of one transaction share a trace even though they run + * sequentially and often on different threads. Sets the stage, tx_type, and + * (after the stage runs) ter_result attributes that drive the collector + * spanmetrics dimensions. A no-op when telemetry is disabled. + * + * @param name Full span name (tx_apply_span::preflight / ::preclaim). + * @param stage Stage attribute value (tx_apply_span::val::*). + * @param tx The transaction supplying the id and type. + */ +[[nodiscard]] telemetry::SpanGuard +makeStageSpan(std::string_view name, std::string_view stage, STTx const& tx) +{ + auto const txID = tx.getTransactionID(); + auto span = telemetry::SpanGuard::hashSpan( + telemetry::TraceCategory::Transactions, name, txID.data(), txID.kBytes); + // Guard the type lookup behind the active check: preflight runs for every + // transaction, so findByType() must not run when tracing is off/disabled. + if (span) + { + span.setAttribute(telemetry::tx_apply_span::attr::stage, stage); + if (char const* typeName = txTypeName(tx.getTxnType())) + span.setAttribute(telemetry::tx_apply_span::attr::txType, typeName); + } + return span; +} + // Call a lambda with the concrete transaction type as a template parameter // throw an "UnknownTxnType" exception on error template @@ -131,86 +177,130 @@ consequencesHelper(PreflightContext const& ctx) static std::pair invokePreflight(PreflightContext const& ctx) { + // Trace the preflight stage. The span shares the transaction's + // deterministic trace_id so it correlates with preclaim and transactor. + auto span = makeStageSpan( + telemetry::tx_apply_span::preflight, telemetry::tx_apply_span::val::preflight, ctx.tx); try { - return withTxnType(ctx.rules, ctx.tx.getTxnType(), [&]() { + auto result = withTxnType(ctx.rules, ctx.tx.getTxnType(), [&]() { auto const tec = Transactor::invokePreflight(ctx); return std::make_pair( tec, isTesSuccess(tec) ? consequencesHelper(ctx) : TxConsequences{tec}); }); + if (span) + { + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(result.first).c_str()); + // Mark the span as errored when preflight rejects the transaction so + // failed stages surface in span-status error counts. + if (!isTesSuccess(result.first)) + span.setError(transToken(result.first)); + } + return result; } catch (UnknownTxnType const& e) { // Should never happen // LCOV_EXCL_START JLOG(ctx.j.fatal()) << "Unknown transaction type in preflight: " << e.txnType; + span.recordException(e); UNREACHABLE("xrpl::invokePreflight : unknown transaction type"); return {temUNKNOWN, TxConsequences{temUNKNOWN}}; // LCOV_EXCL_STOP } + catch (std::exception const& e) + { + // The caller's preflight() maps this to tefEXCEPTION. Record it on the + // span before unwinding so per-stage error counts include exceptions. + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(tefEXCEPTION).c_str()); + span.recordException(e); + throw; + } } static TER invokePreclaim(PreclaimContext const& ctx) { + // Trace the preclaim stage under the transaction's deterministic trace_id. + auto span = makeStageSpan( + telemetry::tx_apply_span::preclaim, telemetry::tx_apply_span::val::preclaim, ctx.tx); try { // use name hiding to accomplish compile-time polymorphism of static // class functions for Transactor and derived classes. - return withTxnType(ctx.view.rules(), ctx.tx.getTxnType(), [&]() -> TER { - // preclaim functionality is divided into two sections: - // 1. Up to and including the signature check: returns NotTEC. - // All transaction checks before and including checkSign - // MUST return NotTEC, or something more restrictive. - // Allowing tec results in these steps risks theft or - // destruction of funds, as a fee will be charged before the - // signature is checked. - // 2. After the signature check: returns TER. - - // If the transactor requires a valid account and the - // transaction doesn't list one, preflight will have already - // a flagged a failure. - auto const id = ctx.tx.getAccountID(sfAccount); - - if (id != beast::kZero) - { - if (NotTEC const preSigResult = [&]() -> NotTEC { - if (NotTEC const result = T::checkSeqProxy(ctx.view, ctx.tx, ctx.j)) - return result; - - if (NotTEC const result = T::checkPriorTxAndLastLedger(ctx)) - return result; - - if (NotTEC const result = T::checkSponsor(ctx.view, ctx.tx)) - return result; - - if (NotTEC const result = - Transactor::invokeCheckPermission(ctx.view, ctx.tx)) - return result; - - if (NotTEC const result = T::checkSign(ctx)) - return result; - - return tesSUCCESS; - }()) - return preSigResult; - - if (TER const result = T::checkFee(ctx, calculateBaseFee(ctx.view, ctx.tx))) - return result; - } - - return T::preclaim(ctx); - }); + TER const preclaimTer = + withTxnType(ctx.view.rules(), ctx.tx.getTxnType(), [&]() -> TER { + // preclaim functionality is divided into two sections: + // 1. Up to and including the signature check: returns NotTEC. + // All transaction checks before and including checkSign + // MUST return NotTEC, or something more restrictive. + // Allowing tec results in these steps risks theft or + // destruction of funds, as a fee will be charged before the + // signature is checked. + // 2. After the signature check: returns TER. + + // If the transactor requires a valid account and the + // transaction doesn't list one, preflight will have already + // a flagged a failure. + auto const id = ctx.tx.getAccountID(sfAccount); + + if (id != beast::kZero) + { + if (NotTEC const preSigResult = [&]() -> NotTEC { + if (NotTEC const result = T::checkSeqProxy(ctx.view, ctx.tx, ctx.j)) + return result; + + if (NotTEC const result = T::checkPriorTxAndLastLedger(ctx)) + return result; + + if (NotTEC const result = T::checkSponsor(ctx.view, ctx.tx)) + return result; + + if (NotTEC const result = + Transactor::invokeCheckPermission(ctx.view, ctx.tx)) + return result; + + if (NotTEC const result = T::checkSign(ctx)) + return result; + + return tesSUCCESS; + }()) + return preSigResult; + + if (TER const result = T::checkFee(ctx, calculateBaseFee(ctx.view, ctx.tx))) + return result; + } + + return T::preclaim(ctx); + }); + if (span) + { + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(preclaimTer).c_str()); + } + return preclaimTer; } catch (UnknownTxnType const& e) { // Should never happen // LCOV_EXCL_START JLOG(ctx.j.fatal()) << "Unknown transaction type in preclaim: " << e.txnType; + span.recordException(e); UNREACHABLE("xrpl::invokePreclaim : unknown transaction type"); return temUNKNOWN; // LCOV_EXCL_STOP } + catch (std::exception const& e) + { + // The caller's preclaim() maps this to tefEXCEPTION. Record it on the + // span before unwinding so per-stage error counts include exceptions. + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(tefEXCEPTION).c_str()); + span.recordException(e); + throw; + } } /** diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 79bffec9cb3..dba528886c9 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -16,6 +16,10 @@ #include #include +#ifdef XRPL_ENABLE_TELEMETRY +#include +#endif + #include #include #include @@ -24,6 +28,7 @@ #include #include #include +#include #include #include @@ -35,6 +40,7 @@ #include #include #include +#include #include #include @@ -685,6 +691,47 @@ struct Peer { } + // Telemetry hooks — no-ops in the simulator. The generic engine calls + // these at every phase transition / outcome resolution so the + // production adaptor (RCLConsensus::Adaptor) can record events on the + // round span; the simulator runs without telemetry. + void + onPhaseEvent(std::string_view, std::string_view) + { + } + + void + onOutcomeEvent(std::string_view) + { + } + + // The generic engine parents its phase spans under the round span via + // this context; the simulator runs without telemetry, so return an + // invalid context and no phase span is created. + static telemetry::SpanContext + roundSpanContext() + { + return {}; + } + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Provide telemetry access for the Consensus template. + * + * The test Peer adaptor uses a static disabled NullTelemetry instance + * so that all shouldTrace*() checks return false and no spans are + * created during simulation tests. It is static because the shared + * disabled instance does not depend on any per-peer state. + */ + static telemetry::Telemetry& + getTelemetry() + { + static auto tel = telemetry::makeTelemetry( + telemetry::Telemetry::Setup{}, beast::Journal{beast::Journal::getNullSink()}); + return *tel; + } +#endif + // Share a message by broadcasting to all connected peers template void diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index cafe72eff9b..743de4f043e 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -33,6 +33,7 @@ set(test_modules shamap tx protocol_autogen + telemetry ) if(NOT WIN32) list(APPEND test_modules net) @@ -63,4 +64,45 @@ if(verify_headers) verify_target_headers(xrpl_tests "${CMAKE_CURRENT_SOURCE_DIR}") endif() +# The telemetry test module exercises SpanGuard/Telemetry, which link against +# the OpenTelemetry SDK when the telemetry build option is enabled. +if(telemetry) + # The SpanGuardScope tests drive an InMemorySpanExporter to read exported + # spans (test-only; production never uses it). The Conan package ships + # libopentelemetry_exporter_in_memory.a but declares the component with no + # libs, so neither the umbrella target nor opentelemetry-cpp::exporter_in_memory + # puts the archive on the link line. Locate it by path and link it BEFORE + # the umbrella so its undefined SDK references resolve against the umbrella. + find_library( + OTEL_IN_MEMORY_EXPORTER_LIB + NAMES opentelemetry_exporter_in_memory + HINTS "${opentelemetry-cpp_PACKAGE_FOLDER_RELEASE}/lib" + REQUIRED + ) + target_link_libraries( + xrpl_tests + PRIVATE + "${OTEL_IN_MEMORY_EXPORTER_LIB}" + opentelemetry-cpp::opentelemetry-cpp + ) + # ValidationTracker lives in src/xrpld/ (not libxrpl), so we compile its + # implementation directly into the test binary and put src/ on the include + # path so its tests can reach headers. + target_include_directories(xrpl_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_sources( + xrpl_tests + PRIVATE + ${CMAKE_SOURCE_DIR}/src/xrpld/telemetry/detail/ValidationTracker.cpp + ) +else() + # MetricsRegistry lives in xrpld; compile its .cpp directly into the test + # target so the no-op path can be tested without linking all of xrpld. + # When telemetry=ON, XRPL_ENABLE_TELEMETRY is globally defined and the + # .cpp pulls in xrpld symbols we cannot satisfy here. + target_sources( + xrpl_tests + PRIVATE ${CMAKE_SOURCE_DIR}/src/xrpld/telemetry/MetricsRegistry.cpp + ) +endif() + gtest_discover_tests(xrpl_tests DISCOVERY_TIMEOUT 60) diff --git a/src/tests/libxrpl/helpers/TestServiceRegistry.h b/src/tests/libxrpl/helpers/TestServiceRegistry.h index 5475b54dc67..b11599310df 100644 --- a/src/tests/libxrpl/helpers/TestServiceRegistry.h +++ b/src/tests/libxrpl/helpers/TestServiceRegistry.h @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -337,6 +338,18 @@ class TestServiceRegistry : public ServiceRegistry throw std::logic_error("TestServiceRegistry::getPerfLog() not implemented"); } + telemetry::Telemetry& + getTelemetry() override + { + throw std::logic_error("TestServiceRegistry::getTelemetry() not implemented"); + } + + telemetry::MetricsRegistry* + getMetricsRegistry() override + { + return nullptr; + } + // Configuration and state bool isStopping() const override diff --git a/src/tests/libxrpl/telemetry/GetMeter.cpp b/src/tests/libxrpl/telemetry/GetMeter.cpp new file mode 100644 index 00000000000..902f40fcd99 --- /dev/null +++ b/src/tests/libxrpl/telemetry/GetMeter.cpp @@ -0,0 +1,112 @@ +/** + * @file GetMeter.cpp + * Unit tests for the direct OTel metrics API surface: + * - xrpl::telemetry::Telemetry::getMeter() on the disabled (noop) path. + * - The global MeterProvider contract that beast's OTelCollector shim + * relies on (GetMeterProvider()->GetMeter(kMeterName, kMeterVersion)). + * + * These tests deliberately exercise the global-provider level rather than a + * fully started TelemetryImpl: TelemetryImpl::start() spins up an OTLP HTTP + * exporter with background export threads and network connect attempts, which + * is unsuitable for a hermetic unit test. The design spec permits testing the + * metrics unlock at the provider level, which is the exact path the beast shim + * and getMeter() delegate to. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined; the metrics API and SDK + * headers exist only in that build. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +namespace metrics_api = opentelemetry::metrics; +namespace metrics_sdk = opentelemetry::sdk::metrics; + +// --------------------------------------------------------------- +// 1. Disabled path: Telemetry::getMeter() returns a usable noop meter. +// makeTelemetry() with enabled=false yields NullTelemetryOtel, whose +// getMeter() serves a meter from a process-wide NoopMeterProvider. The +// meter and any instrument built from it must be non-null, and Add() +// (positive and negative) must be inert but safe. +// --------------------------------------------------------------- +TEST(GetMeter, disabled_path_returns_usable_noop_meter) +{ + Telemetry::Setup setup; + setup.enabled = false; + + beast::Journal::Sink& sink = beast::Journal::getNullSink(); + beast::Journal const journal(sink); + auto telemetry = makeTelemetry(setup, journal); + ASSERT_NE(telemetry, nullptr); + + // getMeter() must return a concrete (non-null) meter, never a raw nullptr. + auto meter = telemetry->getMeter(); + ASSERT_TRUE(static_cast(meter)); + + // A synchronous UpDownCounter — one of the three instrument kinds the + // direct API unlocks — must be creatable and non-null. + auto upDown = meter->CreateInt64UpDownCounter("test_open_ledgers"); + ASSERT_TRUE(static_cast(upDown)); + + // Both increment and decrement paths must execute without error (inert on + // the noop instrument, but exercised here to prove both directions work). + upDown->Add(static_cast(1)); + upDown->Add(static_cast(-1)); +} + +// --------------------------------------------------------------- +// 2. Global-provider contract (the beast OTelCollector shim path). +// After a real SDK MeterProvider is registered via SetMeterProvider(), +// GetMeterProvider()->GetMeter(kMeterName, kMeterVersion) — the exact call +// the shim makes — must return a non-null meter, and an UpDownCounter built +// from it must be non-null and accept positive and negative Add(). +// --------------------------------------------------------------- +TEST(GetMeter, global_provider_meter_accepts_updown_counter) +{ + // Preserve and later restore the process-wide provider so this test does + // not leak state into other telemetry tests in the same binary. + auto const previous = metrics_api::Provider::GetMeterProvider(); + + // A views-less SDK MeterProvider with no reader is sufficient to prove the + // API contract: it hands out a real (non-noop) Meter that creates working + // instruments. No exporter/reader means no background threads or network. + std::shared_ptr const sdkProvider = + metrics_sdk::MeterProviderFactory::Create(); + metrics_api::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(sdkProvider)); + + // Fetch the meter exactly as the beast shim does: kMeterName + kMeterVersion + // from the global provider. + auto meter = metrics_api::Provider::GetMeterProvider()->GetMeter( + std::string(kMeterName), std::string(kMeterVersion)); + ASSERT_TRUE(static_cast(meter)); + + auto upDown = meter->CreateInt64UpDownCounter("test_in_flight_requests"); + ASSERT_TRUE(static_cast(upDown)); + + // Positive path: value goes up. + upDown->Add(static_cast(1)); + // Negative path: UpDownCounter permits decrement (unlike a plain Counter). + upDown->Add(static_cast(-1)); + + // Restore the previous global provider. + metrics_api::Provider::SetMeterProvider(previous); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp new file mode 100644 index 00000000000..a1000d4cae1 --- /dev/null +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -0,0 +1,359 @@ +/** + * GTest unit tests for MetricsRegistry (no-op / telemetry-disabled path). + * + * Tests cover: + * - Construction with telemetry disabled (no-op behavior). + * - start()/stop() lifecycle when disabled. + * - Synchronous instrument recording methods do not crash when disabled. + * - Double stop() is safe. + * - Destructor handles cleanup without crash. + * + * NOTE: These tests only exercise the no-op path (telemetry disabled). + * When XRPL_ENABLE_TELEMETRY is defined, MetricsRegistry.cpp pulls in + * xrpld symbols that cannot be linked into this standalone test binary, + * so the tests are compiled out. + */ + +// When telemetry is globally enabled, MetricsRegistry.cpp requires xrpld +// link dependencies we cannot satisfy in a standalone GTest binary. +#ifndef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include + +using namespace xrpl; + +namespace { + +/** + * Minimal mock ServiceRegistry for MetricsRegistry testing. + * + * Only the getMetricsRegistry() call is used in the tests; other methods + * are not invoked because the registry is disabled (enabled=false) so no + * gauge callbacks execute. + * + * All pure virtual methods throw to catch accidental calls during tests. + */ +class MockServiceRegistry : public ServiceRegistry +{ + [[noreturn]] static void + throwUnimplemented() + { + throw std::logic_error("MockServiceRegistry: method not implemented"); + } + +public: + // ServiceRegistry interface — stubs that should never be called. + CollectorManager& + getCollectorManager() override + { + throwUnimplemented(); + } + Family& + getNodeFamily() override + { + throwUnimplemented(); + } + TimeKeeper& + getTimeKeeper() override + { + throwUnimplemented(); + } + JobQueue& + getJobQueue() override + { + throwUnimplemented(); + } + NodeCache& + getTempNodeCache() override + { + throwUnimplemented(); + } + CachedSLEs& + getCachedSLEs() override + { + throwUnimplemented(); + } + NetworkIDService& + getNetworkIDService() override + { + throwUnimplemented(); + } + AmendmentTable& + getAmendmentTable() override + { + throwUnimplemented(); + } + HashRouter& + getHashRouter() override + { + throwUnimplemented(); + } + LoadFeeTrack& + getFeeTrack() override + { + throwUnimplemented(); + } + LoadManager& + getLoadManager() override + { + throwUnimplemented(); + } + RCLValidations& + getValidations() override + { + throwUnimplemented(); + } + ValidatorList& + getValidators() override + { + throwUnimplemented(); + } + ValidatorSite& + getValidatorSites() override + { + throwUnimplemented(); + } + ManifestCache& + getValidatorManifests() override + { + throwUnimplemented(); + } + ManifestCache& + getPublisherManifests() override + { + throwUnimplemented(); + } + Overlay& + getOverlay() override + { + throwUnimplemented(); + } + Cluster& + getCluster() override + { + throwUnimplemented(); + } + PeerReservationTable& + getPeerReservations() override + { + throwUnimplemented(); + } + Resource::Manager& + getResourceManager() override + { + throwUnimplemented(); + } + NodeStore::Database& + getNodeStore() override + { + throwUnimplemented(); + } + SHAMapStore& + getSHAMapStore() override + { + throwUnimplemented(); + } + RelationalDatabase& + getRelationalDatabase() override + { + throwUnimplemented(); + } + InboundLedgers& + getInboundLedgers() override + { + throwUnimplemented(); + } + InboundTransactions& + getInboundTransactions() override + { + throwUnimplemented(); + } + TaggedCache& + getAcceptedLedgerCache() override + { + throwUnimplemented(); + } + LedgerMaster& + getLedgerMaster() override + { + throwUnimplemented(); + } + LedgerCleaner& + getLedgerCleaner() override + { + throwUnimplemented(); + } + LedgerReplayer& + getLedgerReplayer() override + { + throwUnimplemented(); + } + PendingSaves& + getPendingSaves() override + { + throwUnimplemented(); + } + [[nodiscard]] OpenLedger& + getOpenLedger() override + { + throwUnimplemented(); + } + [[nodiscard]] OpenLedger const& + getOpenLedger() const override + { + throwUnimplemented(); + } + NetworkOPs& + getOPs() override + { + throwUnimplemented(); + } + OrderBookDB& + getOrderBookDB() override + { + throwUnimplemented(); + } + TransactionMaster& + getMasterTransaction() override + { + throwUnimplemented(); + } + TxQ& + getTxQ() override + { + throwUnimplemented(); + } + PathRequestManager& + getPathRequestManager() override + { + throwUnimplemented(); + } + ServerHandler& + getServerHandler() override + { + throwUnimplemented(); + } + perf::PerfLog& + getPerfLog() override + { + throwUnimplemented(); + } + telemetry::Telemetry& + getTelemetry() override + { + throwUnimplemented(); + } + telemetry::MetricsRegistry* + getMetricsRegistry() override + { + return nullptr; + } + [[nodiscard]] bool + isStopping() const override + { + return false; + } + beast::Journal + getJournal(std::string const&) override + { + return beast::Journal(beast::Journal::getNullSink()); + } + boost::asio::io_context& + getIOContext() override + { + throwUnimplemented(); + } + Logs& + getLogs() override + { + throwUnimplemented(); + } + [[nodiscard]] std::optional const& + getTrapTxID() const override + { + static std::optional const empty; + return empty; + } + DatabaseCon& + getWalletDB() override + { + throwUnimplemented(); + } + Application& + getApp() override + { + throwUnimplemented(); + } +}; + +/** + * Test fixture that provides a MockServiceRegistry and null Journal. + */ +class MetricsRegistryTest : public ::testing::Test +{ +protected: + MockServiceRegistry mockApp_; + beast::Journal j_{beast::Journal::getNullSink()}; +}; + +} // namespace + +TEST_F(MetricsRegistryTest, disabled_construction) +{ + // Construct with enabled=false; should be a no-op. + telemetry::MetricsRegistry const registry(false, mockApp_, j_); + EXPECT_FALSE(registry.isEnabled()); +} + +TEST_F(MetricsRegistryTest, disabled_start_stop) +{ + telemetry::MetricsRegistry registry(false, mockApp_, j_); + + // start() and stop() should be no-ops when disabled. + registry.start("http://localhost:4318/v1/metrics"); + registry.stop(); + + // Double stop should be safe. + registry.stop(); +} + +TEST_F(MetricsRegistryTest, disabled_recording_methods) +{ + telemetry::MetricsRegistry registry(false, mockApp_, j_); + registry.start("http://localhost:4318/v1/metrics"); + + // All recording methods should be no-ops (not crash). + registry.recordRpcStarted("server_info"); + registry.recordRpcFinished("server_info", 1000); + registry.recordRpcErrored("ledger", 500); + registry.recordJobQueued("ledgerData"); + registry.recordJobStarted("ledgerData", 200); + registry.recordJobFinished("ledgerData", 3000); + + registry.stop(); +} + +TEST_F(MetricsRegistryTest, destructor_calls_stop) +{ + { + // Let the destructor handle cleanup. + telemetry::MetricsRegistry registry(false, mockApp_, j_); + registry.start("http://localhost:4318/v1/metrics"); + } + // If we get here without crash, the destructor handled stop. +} + +#endif // !XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/Redaction.cpp b/src/tests/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 00000000000..2ef6ed377b7 --- /dev/null +++ b/src/tests/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,44 @@ +#include + +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +// Empty input must produce empty output (edge / negative path). +TEST(Redaction, empty_input_returns_empty) +{ + EXPECT_EQ(redactAccount(""), ""); +} + +// Success path: assert the EXACT expected token. The value is the first +// 16 chars of sha512Half(addr) rendered as lowercase hex, computed once +// and hard-coded here so any change to the hashing recipe is caught. +TEST(Redaction, known_account_exact_hash) +{ + EXPECT_EQ(redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), "a513895f49311f54"); +} + +// Structural invariants: length, lowercase-hex alphabet, and stability. +TEST(Redaction, token_is_16_lowercase_hex_and_stable) +{ + auto const token = redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"); + EXPECT_EQ(token.size(), 16u); + EXPECT_TRUE(std::ranges::all_of(token, [](unsigned char c) { + return std::isdigit(c) || (c >= 'a' && c <= 'f'); + })); + // Deterministic: hashing the same input twice yields the same token. + EXPECT_EQ(token, redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")); +} + +// Distinct inputs must produce distinct tokens (no accidental constant). +TEST(Redaction, distinct_inputs_distinct_tokens) +{ + EXPECT_NE( + redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), + redactAccount("rDifferentAccount1234567890abcdefgh")); +} diff --git a/src/tests/libxrpl/telemetry/SpanGuardFactory.cpp b/src/tests/libxrpl/telemetry/SpanGuardFactory.cpp new file mode 100644 index 00000000000..0d3802df8b5 --- /dev/null +++ b/src/tests/libxrpl/telemetry/SpanGuardFactory.cpp @@ -0,0 +1,114 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +TEST(SpanGuardFactory, null_guard_methods_are_safe) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "nonexistent"); + EXPECT_FALSE(span); + + span.setAttribute("key", "value"); + span.setAttribute("int_key", static_cast(42)); + span.setAttribute("bool_key", true); + span.setOk(); + span.setError("test"); + span.addEvent("event"); +} + +TEST(SpanGuardFactory, category_span_returns_null_when_disabled) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "test"); + EXPECT_FALSE(span); + + // Attribute keys use the underscore convention for span attributes (the + // dotted xrpl.. form is reserved for resource attributes). The + // canonical constants live in the xrpld-level *SpanNames.h headers, which a + // libxrpl test cannot include, so the keys are written as literals here. + span.setAttribute("command", "test"); + span.setAttribute("rpc_status", "success"); +} + +TEST(SpanGuardFactory, child_span_null_when_no_parent) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "parent"); + auto child = span.childSpan("child.test"); + EXPECT_FALSE(child); +} + +TEST(SpanGuardFactory, linked_span_null_when_no_context) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "source"); + auto linked = span.linkedSpan("linked.test"); + EXPECT_FALSE(linked); +} + +TEST(SpanGuardFactory, capture_context_returns_invalid_on_null) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "ctx"); + auto ctx = span.captureContext(); + EXPECT_FALSE(ctx.isValid()); +} + +TEST(SpanGuardFactory, move_construction_transfers_ownership) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "move"); + auto moved = std::move(span); + EXPECT_FALSE(span); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + moved.setAttribute("key", "value"); +} + +TEST(SpanGuardFactory, record_exception_safe_on_null) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc.command", "test"); + try + { + throw std::runtime_error("test error"); + } + catch (std::exception const& e) + { + span.recordException(e); + } +} + +TEST(SpanGuardFactory, discard_safe_on_null) +{ + auto span = SpanGuard::span(TraceCategory::Transactions, "tx", "process"); + span.discard(); + EXPECT_FALSE(span); +} + +TEST(SpanGuardFactory, consensus_close_time_attributes) +{ + // Verify the consensus attribute pattern compiles and doesn't crash with a + // null SpanGuard. Attribute keys/values use the underscore convention; the + // canonical consensus::span constants are defined in the xrpld-level + // ConsensusSpanNames.h, which a libxrpl test cannot include, so the keys are + // written as literals here. + { + auto span = telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, "accept.apply"); + span.setAttribute("ledger_seq", static_cast(42)); + span.setAttribute("close_time", static_cast(780000000)); + span.setAttribute("close_time_correct", true); + span.setAttribute("close_resolution_ms", static_cast(30000)); + span.setAttribute("consensus_state", std::string("finished")); + span.setAttribute("proposing", true); + span.setAttribute("round_time_ms", static_cast(3500)); + } + { + auto span = telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, "accept.apply"); + span.setAttribute("close_time_correct", false); + span.setAttribute("consensus_state", std::string("moved_on")); + } +} diff --git a/src/tests/libxrpl/telemetry/SpanGuardScope.cpp b/src/tests/libxrpl/telemetry/SpanGuardScope.cpp new file mode 100644 index 00000000000..7c3cdd88ba1 --- /dev/null +++ b/src/tests/libxrpl/telemetry/SpanGuardScope.cpp @@ -0,0 +1,447 @@ +// Tests for SpanGuard::rootSpan() and SpanGuard::detached(). +// +// These verify the cross-thread scope-leak fix at the trace level using an +// in-memory span exporter: rootSpan() must start a fresh trace root (ignoring +// the ambient active span), and detached() must strip the thread-local Scope +// on the origin thread so the guard can be moved to and ended on another +// thread without corrupting the origin thread's context stack. +// +// The whole file is telemetry-only: when XRPL_ENABLE_TELEMETRY is not defined +// SpanGuard is a no-op stub and the OpenTelemetry SDK headers are unavailable, +// so the translation unit compiles empty. + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { +namespace { + +namespace otel_sdk_trace = opentelemetry::sdk::trace; +namespace otel_memory = opentelemetry::exporter::memory; + +/** + * In-memory Telemetry backing for SpanGuard scope tests. + * + * Reports every trace category as enabled and creates spans through an SDK + * TracerProvider whose SimpleSpanProcessor forwards ended spans to an + * InMemorySpanExporter, so a test can read the exact exported SpanData + * (trace id, span id, parent id, name). + * + * Inheritance: + * + * +-----------+ + * | Telemetry | (abstract interface) + * +-----+-----+ + * | + * +-----+---------+ + * | TestTelemetry | in-memory exporter pipeline + * +---------------+ + * + * @note Test-only. Install with Telemetry::setInstance() and clear it in + * teardown. The exporter buffer is drained by InMemorySpanData::GetSpans(), + * so each GetSpans() call returns only spans exported since the previous one. + */ +class TestTelemetry : public Telemetry +{ +public: + /** + * Build the SDK export pipeline and keep the exporter's span buffer. + */ + TestTelemetry() + { + // Factory populates spanData_ with the exporter's shared buffer. + auto exporter = otel_memory::InMemorySpanExporterFactory::Create(spanData_); + auto processor = otel_sdk_trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); + provider_ = otel_sdk_trace::TracerProviderFactory::Create( + std::move(processor), + opentelemetry::sdk::resource::Resource::Create({}), + otel_sdk_trace::AlwaysOnSamplerFactory::Create()); + } + + /** + * @return The exporter's span buffer (drained by GetSpans()). + */ + std::shared_ptr + spanData() const + { + return spanData_; + } + + void + start() override + { + } + void + stop() override + { + } + + [[nodiscard]] bool + isEnabled() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceRpc() const override + { + return true; + } + [[nodiscard]] bool + shouldTracePeer() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceLedger() const override + { + return true; + } + + /** + * @return A fixed strategy label; the scope tests do not exercise + * deterministic trace-id correlation, so any stable value works. + */ + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + static std::string const kStrategy{"none"}; + return kStrategy; + } + + opentelemetry::nostd::shared_ptr + getTracer(std::string_view name) override + { + return provider_->GetTracer(std::string(name)); + } + + /** + * @return A meter from a noop provider; the scope tests exercise only + * tracing, so metrics instruments are inert. + */ + opentelemetry::nostd::shared_ptr + getMeter(std::string_view name) override + { + static auto noopProvider = + opentelemetry::nostd::shared_ptr( + new opentelemetry::metrics::NoopMeterProvider()); + return noopProvider->GetMeter(std::string(name), std::string(kMeterVersion)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view name, opentelemetry::trace::SpanKind kind) override + { + opentelemetry::trace::StartSpanOptions opts; + opts.kind = kind; + return getTracer(kTracerName)->StartSpan(std::string(name), opts); + } + + opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::context::Context const& parentContext, + opentelemetry::trace::SpanKind kind) override + { + opentelemetry::trace::StartSpanOptions opts; + opts.kind = kind; + opts.parent = parentContext; + return getTracer(kTracerName)->StartSpan(std::string(name), opts); + } + +private: + /** + * SDK provider owning the export pipeline. + */ + std::shared_ptr provider_; + + /** + * Shared buffer that receives ended spans from the exporter. + */ + std::shared_ptr spanData_; +}; + +/** + * @return The span's name as a std::string for equality checks. + */ +std::string +nameOf(otel_sdk_trace::SpanData const& span) +{ + auto view = span.GetName(); + return std::string(view.data(), view.size()); +} + +/** + * @return Pointer to the first exported span with the given name, or null. + */ +otel_sdk_trace::SpanData* +findSpan(std::vector> const& spans, std::string_view name) +{ + for (auto const& span : spans) + { + if (nameOf(*span) == name) + return span.get(); + } + return nullptr; +} + +/** + * @return Number of exported spans with the given name. + */ +std::size_t +countSpans( + std::vector> const& spans, + std::string_view name) +{ + std::size_t count = 0; + for (auto const& span : spans) + { + if (nameOf(*span) == name) + ++count; + } + return count; +} + +/** + * Installs a TestTelemetry as the global instance for each test and clears it + * afterwards so the singleton never dangles between cases. + */ +class SpanGuardScopeTest : public ::testing::Test +{ +protected: + void + SetUp() override + { + telemetry_ = std::make_unique(); + Telemetry::setInstance(telemetry_.get()); + } + + void + TearDown() override + { + Telemetry::setInstance(nullptr); + telemetry_.reset(); + } + + /** + * @return The exporter's span buffer for the active TestTelemetry. + */ + std::shared_ptr + spanData() const + { + return telemetry_->spanData(); + } + + /** + * The in-memory Telemetry installed for the duration of a test. + */ + std::unique_ptr telemetry_; +}; + +// rootSpan() must ignore the ambient active span and start a brand-new trace. +TEST_F(SpanGuardScopeTest, rootSpan_ignores_ambient_parent) +{ + { + // Ambient span becomes the active span on this thread. + auto ambient = SpanGuard::span(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(ambient)); + + // rootSpan must NOT inherit the ambient span as its parent. + auto root = SpanGuard::rootSpan(TraceCategory::Peer, "peer", "validation.receive"); + ASSERT_TRUE(static_cast(root)); + } // root ends first, then ambient -- both on this thread. + + auto spans = spanData()->GetSpans(); + auto* ambient = findSpan(spans, "rpc.command"); + auto* root = findSpan(spans, "peer.validation.receive"); + ASSERT_NE(ambient, nullptr); + ASSERT_NE(root, nullptr); + + // The ambient span is itself a root (first span, no parent). + EXPECT_FALSE(ambient->GetParentSpanId().IsValid()); + EXPECT_TRUE(ambient->GetTraceId().IsValid()); + + // The rootSpan has NO parent and lives in a DIFFERENT trace than ambient. + EXPECT_FALSE(root->GetParentSpanId().IsValid()); + EXPECT_TRUE(root->GetTraceId().IsValid()); + EXPECT_NE(root->GetTraceId(), ambient->GetTraceId()); +} + +// detached() pops the Scope on the origin thread, so ending the guard on +// another thread leaves the origin thread's context stack clean. +TEST_F(SpanGuardScopeTest, detached_does_not_corrupt_origin_stack) +{ + // Scoped guard is active on this (origin) thread. + auto guard = SpanGuard::span(TraceCategory::Ledger, "ledger", "build"); + ASSERT_TRUE(static_cast(guard)); + + // Strip the thread-local Scope here on the origin thread. + auto detached = std::move(guard).detached(); + ASSERT_TRUE(static_cast(detached)); + + // End the detached span on a worker thread. + std::thread worker([d = std::move(detached)]() mutable {}); + worker.join(); + + // Back on the origin thread: a new span must be a fresh root, proving the + // origin stack top was restored by detached() (not left holding + // ledger.build). + { + auto after = SpanGuard::span(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(after)); + } + + auto spans = spanData()->GetSpans(); + auto* build = findSpan(spans, "ledger.build"); + auto* after = findSpan(spans, "rpc.command"); + ASSERT_NE(build, nullptr); + ASSERT_NE(after, nullptr); + + // 'after' is a new root: zero parent and a different trace than + // ledger.build. + EXPECT_FALSE(after->GetParentSpanId().IsValid()); + EXPECT_NE(after->GetTraceId(), build->GetTraceId()); +} + +// A detached span is ended exactly once, by the worker thread that owns it. +TEST_F(SpanGuardScopeTest, detached_span_ends_once) +{ + auto guard = SpanGuard::span(TraceCategory::Ledger, "ledger", "build"); + auto detached = std::move(guard).detached(); + ASSERT_TRUE(static_cast(detached)); + + // Before the worker ends it, the span is still open: nothing exported yet. + auto before = spanData()->GetSpans(); + EXPECT_EQ(countSpans(before, "ledger.build"), 0u); + + // Ending happens once, on the worker thread, when the guard is destroyed. + { + std::thread worker([d = std::move(detached)]() mutable {}); + worker.join(); + } + + auto after = spanData()->GetSpans(); + EXPECT_EQ(countSpans(after, "ledger.build"), 1u); +} + +// rootSpan().detached() is the combination used at RPC coroutine entry points +// (e.g. RipplePathFind) that hold a span across a coro yield: the span must be +// a fresh root AND must not leave a Scope on the worker's context stack. +TEST_F(SpanGuardScopeTest, root_then_detached_is_root_and_leaves_stack_clean) +{ + { + // Ambient span active on this (origin) thread. + auto ambient = SpanGuard::span(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(ambient)); + + // Fresh root, then detach the Scope on the origin thread. + auto req = SpanGuard::rootSpan(TraceCategory::Rpc, "pathfind", "request").detached(); + ASSERT_TRUE(static_cast(req)); + + // End the detached root span on a worker thread (mimics coro resume). + std::thread worker([r = std::move(req)]() mutable {}); + worker.join(); + + // Origin stack must be clean: a new span here is still parented by the + // ambient span (proving the detached root did not linger on the stack). + { + auto child = SpanGuard::span(TraceCategory::Rpc, "rpc", "sub"); + ASSERT_TRUE(static_cast(child)); + } + } + + auto spans = spanData()->GetSpans(); + auto* ambient = findSpan(spans, "rpc.command"); + auto* req = findSpan(spans, "pathfind.request"); + auto* child = findSpan(spans, "rpc.sub"); + ASSERT_NE(ambient, nullptr); + ASSERT_NE(req, nullptr); + ASSERT_NE(child, nullptr); + + // The rooted-then-detached span has NO parent and a DIFFERENT trace. + EXPECT_FALSE(req->GetParentSpanId().IsValid()); + EXPECT_NE(req->GetTraceId(), ambient->GetTraceId()); + + // The detached root did not corrupt the origin stack: 'child' still nests + // under the ambient span (same trace, parent = ambient), NOT under req. + EXPECT_EQ(child->GetParentSpanId(), ambient->GetSpanId()); + EXPECT_EQ(child->GetTraceId(), ambient->GetTraceId()); +} + +// Negative control (documents the bug detached() fixes): a scoped guard NOT +// detached, moved to and destroyed on a worker thread, pops the WRONG thread's +// stack, so a later span on the origin thread inherits the stale span. The +// whole scenario runs on a dedicated origin thread so the corrupted +// thread-local stack cannot leak into other tests. +TEST_F(SpanGuardScopeTest, non_detached_cross_thread_corrupts_origin_stack) +{ + std::thread origin([&] { + auto guard = SpanGuard::span(TraceCategory::Ledger, "ledger", "build"); + { + // ~SpanGuard on the worker ends the span and Detaches on the + // WORKER stack -- the origin stack is never popped. + std::thread worker([d = std::move(guard)]() mutable {}); + worker.join(); + } + // Origin stack still holds the stale ledger.build as active, so this + // span inherits it as parent instead of being a root. + auto after = SpanGuard::span(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(after)); + }); + origin.join(); + + auto spans = spanData()->GetSpans(); + auto* build = findSpan(spans, "ledger.build"); + auto* after = findSpan(spans, "rpc.command"); + ASSERT_NE(build, nullptr); + ASSERT_NE(after, nullptr); + + // BUG: 'after' inherited the stale span -- non-zero parent, same trace. + EXPECT_TRUE(after->GetParentSpanId().IsValid()); + EXPECT_EQ(after->GetParentSpanId(), build->GetSpanId()); + EXPECT_EQ(after->GetTraceId(), build->GetTraceId()); +} + +} // namespace +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp new file mode 100644 index 00000000000..bfa0b851495 --- /dev/null +++ b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp @@ -0,0 +1,146 @@ +#include +#include +#include + +#include + +#include + +using namespace xrpl; + +TEST(TelemetryConfig, setup_defaults) +{ + telemetry::Telemetry::Setup const s; + EXPECT_FALSE(s.enabled); + EXPECT_EQ(s.serviceName, "xrpld"); + EXPECT_TRUE(s.serviceVersion.empty()); + EXPECT_TRUE(s.serviceInstanceId.empty()); + EXPECT_EQ(s.exporterEndpoint, "http://localhost:4318/v1/traces"); + EXPECT_FALSE(s.useTls); + EXPECT_TRUE(s.tlsCertPath.empty()); + EXPECT_DOUBLE_EQ(s.samplingRatio, 1.0); + EXPECT_EQ(s.batchSize, 512u); + EXPECT_EQ(s.batchDelay, std::chrono::milliseconds{5000}); + EXPECT_EQ(s.maxQueueSize, 2048u); + EXPECT_EQ(s.networkId, 0u); + EXPECT_EQ(s.networkType, "mainnet"); + EXPECT_TRUE(s.traceTransactions); + EXPECT_TRUE(s.traceConsensus); + EXPECT_TRUE(s.traceRpc); + EXPECT_TRUE(s.tracePeer); + EXPECT_TRUE(s.traceLedger); +} + +TEST(TelemetryConfig, parse_empty_section) +{ + Section const section; + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + + EXPECT_FALSE(setup.enabled); + EXPECT_EQ(setup.serviceName, "xrpld"); + EXPECT_EQ(setup.serviceVersion, "2.0.0"); + EXPECT_EQ(setup.serviceInstanceId, "nHUtest123"); + EXPECT_DOUBLE_EQ(setup.samplingRatio, 1.0); + EXPECT_TRUE(setup.traceRpc); + EXPECT_TRUE(setup.traceTransactions); + EXPECT_TRUE(setup.traceConsensus); + EXPECT_TRUE(setup.tracePeer); + EXPECT_TRUE(setup.traceLedger); +} + +TEST(TelemetryConfig, parse_full_section) +{ + Section section; + section.set("enabled", "1"); + section.set("service_name", "my-rippled"); + section.set("service_instance_id", "custom-id"); + section.set("exporter", "otlp_http"); + section.set("endpoint", "http://collector:4318/v1/traces"); + section.set("use_tls", "1"); + section.set("tls_ca_cert", "/etc/ssl/ca.pem"); + section.set("batch_size", "256"); + section.set("batch_delay_ms", "3000"); + section.set("max_queue_size", "4096"); + section.set("trace_transactions", "0"); + section.set("trace_consensus", "0"); + section.set("trace_rpc", "1"); + section.set("trace_peer", "1"); + section.set("trace_ledger", "0"); + + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 1); + + EXPECT_TRUE(setup.enabled); + EXPECT_EQ(setup.serviceName, "my-rippled"); + EXPECT_EQ(setup.serviceInstanceId, "custom-id"); + EXPECT_EQ(setup.exporterEndpoint, "http://collector:4318/v1/traces"); + EXPECT_TRUE(setup.useTls); + EXPECT_EQ(setup.tlsCertPath, "/etc/ssl/ca.pem"); + EXPECT_EQ(setup.batchSize, 256u); + EXPECT_EQ(setup.batchDelay, std::chrono::milliseconds{3000}); + EXPECT_EQ(setup.maxQueueSize, 4096u); + EXPECT_FALSE(setup.traceTransactions); + EXPECT_FALSE(setup.traceConsensus); + EXPECT_TRUE(setup.traceRpc); + EXPECT_TRUE(setup.tracePeer); + EXPECT_FALSE(setup.traceLedger); +} + +TEST(TelemetryConfig, mtls_cert_and_key_both_set) +{ + Section section; + section.set("use_tls", "1"); + section.set("tls_client_cert", "/etc/ssl/client.pem"); + section.set("tls_client_key", "/etc/ssl/client.key"); + + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + EXPECT_EQ(setup.tlsClientCertPath, "/etc/ssl/client.pem"); + EXPECT_EQ(setup.tlsClientKeyPath, "/etc/ssl/client.key"); +} + +TEST(TelemetryConfig, mtls_cert_without_key_throws) +{ + Section section; + section.set("tls_client_cert", "/etc/ssl/client.pem"); + EXPECT_THROW( + telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error); +} + +TEST(TelemetryConfig, mtls_key_without_cert_throws) +{ + Section section; + section.set("tls_client_key", "/etc/ssl/client.key"); + EXPECT_THROW( + telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error); +} + +TEST(TelemetryConfig, mtls_neither_set_is_one_way_tls) +{ + Section section; + section.set("use_tls", "1"); + section.set("tls_ca_cert", "/etc/ssl/ca.pem"); + + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + EXPECT_TRUE(setup.tlsClientCertPath.empty()); + EXPECT_TRUE(setup.tlsClientKeyPath.empty()); +} + +TEST(TelemetryConfig, null_telemetry_factory) +{ + telemetry::Telemetry::Setup setup; + setup.enabled = false; + + beast::Journal::Sink& sink = beast::Journal::getNullSink(); + beast::Journal const j(sink); + auto tel = telemetry::makeTelemetry(setup, j); + EXPECT_TRUE(tel != nullptr); + EXPECT_FALSE(tel->isEnabled()); + EXPECT_FALSE(tel->shouldTraceRpc()); + EXPECT_FALSE(tel->shouldTraceTransactions()); + EXPECT_FALSE(tel->shouldTraceConsensus()); + EXPECT_FALSE(tel->shouldTracePeer()); + EXPECT_FALSE(tel->shouldTraceLedger()); + + // start/stop should be no-ops without crashing + tel->start(); + tel->stop(); +} diff --git a/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp new file mode 100644 index 00000000000..fdc6c443cb3 --- /dev/null +++ b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp @@ -0,0 +1,158 @@ +#include + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace trace = opentelemetry::trace; + +TEST(TraceContextPropagator, round_trip) +{ + std::uint8_t traceIdBuf[16] = { + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0a, + 0x0b, + 0x0c, + 0x0d, + 0x0e, + 0x0f, + 0x10}; + std::uint8_t spanIdBuf[8] = {0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x11, 0x22}; + + trace::TraceId const traceId( + opentelemetry::nostd::span(traceIdBuf, 16)); + trace::SpanId const spanId(opentelemetry::nostd::span(spanIdBuf, 8)); + trace::TraceFlags const flags(trace::TraceFlags::kIsSampled); + trace::SpanContext const spanCtx(traceId, spanId, flags, true); + + auto ctx = opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr(new trace::DefaultSpan(spanCtx))); + + protocol::TraceContext proto; + xrpl::telemetry::injectToProtobuf(ctx, proto); + + EXPECT_TRUE(proto.has_trace_id()); + EXPECT_EQ(proto.trace_id().size(), 16u); + EXPECT_TRUE(proto.has_span_id()); + EXPECT_EQ(proto.span_id().size(), 8u); + EXPECT_EQ(proto.trace_flags(), static_cast(trace::TraceFlags::kIsSampled)); + EXPECT_EQ(std::memcmp(proto.trace_id().data(), traceIdBuf, 16), 0); + EXPECT_EQ(std::memcmp(proto.span_id().data(), spanIdBuf, 8), 0); + + auto extractedCtx = xrpl::telemetry::extractFromProtobuf(proto); + auto extractedSpan = trace::GetSpan(extractedCtx); + ASSERT_NE(extractedSpan, nullptr); + + auto const& extracted = extractedSpan->GetContext(); + EXPECT_TRUE(extracted.IsValid()); + EXPECT_TRUE(extracted.IsRemote()); + EXPECT_EQ(extracted.trace_id(), traceId); + EXPECT_EQ(extracted.span_id(), spanId); + EXPECT_TRUE(extracted.trace_flags().IsSampled()); +} + +TEST(TraceContextPropagator, extract_empty_protobuf) +{ + protocol::TraceContext const proto; + auto ctx = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(ctx); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); +} + +TEST(TraceContextPropagator, extract_wrong_size_trace_id) +{ + protocol::TraceContext proto; + proto.set_trace_id(std::string(8, '\x01')); + proto.set_span_id(std::string(8, '\xaa')); + + auto ctx = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(ctx); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); +} + +TEST(TraceContextPropagator, extract_wrong_size_span_id) +{ + protocol::TraceContext proto; + proto.set_trace_id(std::string(16, '\x01')); + proto.set_span_id(std::string(4, '\xaa')); + + auto ctx = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(ctx); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); +} + +TEST(TraceContextPropagator, inject_invalid_span) +{ + auto ctx = opentelemetry::context::Context{}; + protocol::TraceContext proto; + xrpl::telemetry::injectToProtobuf(ctx, proto); + + EXPECT_FALSE(proto.has_trace_id()); + EXPECT_FALSE(proto.has_span_id()); +} + +TEST(TraceContextPropagator, flags_preservation) +{ + std::uint8_t const traceIdBuf[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + std::uint8_t const spanIdBuf[8] = {1, 2, 3, 4, 5, 6, 7, 8}; + + // Test with flags NOT sampled (flags = 0) + trace::TraceFlags const flags(0); + trace::SpanContext const spanCtx( + trace::TraceId(opentelemetry::nostd::span(traceIdBuf, 16)), + trace::SpanId(opentelemetry::nostd::span(spanIdBuf, 8)), + flags, + true); + + auto ctx = opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr(new trace::DefaultSpan(spanCtx))); + + protocol::TraceContext proto; + xrpl::telemetry::injectToProtobuf(ctx, proto); + EXPECT_EQ(proto.trace_flags(), 0u); + + auto extracted = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(extracted); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().trace_flags().IsSampled()); +} + +#else // XRPL_ENABLE_TELEMETRY not defined + +TEST(TraceContextPropagator, compiles_without_telemetry) +{ + SUCCEED(); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/TxApplySpanNames.cpp b/src/tests/libxrpl/telemetry/TxApplySpanNames.cpp new file mode 100644 index 00000000000..52c4bdab0e6 --- /dev/null +++ b/src/tests/libxrpl/telemetry/TxApplySpanNames.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#include + +/** + * Contract tests for the transaction apply-pipeline span constants. + * + * The span names and attribute keys in TxApplySpanNames.h are a cross-component + * contract: the collector spanmetrics connector aggregates on these exact + * strings (dimensions tx_type, ter_result, stage) and the Grafana + * transaction-overview dashboard queries them. A silent rename here would + * break per-stage metrics with no compile error, so these tests pin the + * literal values. They need no telemetry runtime and run in every build. + */ + +using namespace xrpl::telemetry; + +TEST(TxApplySpanNames, span_names_are_dot_qualified) +{ + // Full span names feed SpanGuard::hashSpan() in applySteps.cpp. + EXPECT_EQ(std::string_view(tx_apply_span::preflight), "tx.preflight"); + EXPECT_EQ(std::string_view(tx_apply_span::preclaim), "tx.preclaim"); +} + +TEST(TxApplySpanNames, operation_suffixes) +{ + // Suffix used with SpanGuard::span(cat, seg::tx, suffix) in Transactor.cpp. + EXPECT_EQ(std::string_view(tx_apply_span::op::preflight), "preflight"); + EXPECT_EQ(std::string_view(tx_apply_span::op::preclaim), "preclaim"); + EXPECT_EQ(std::string_view(tx_apply_span::op::transactor), "transactor"); +} + +TEST(TxApplySpanNames, attribute_keys_match_collector_dimensions) +{ + // These keys MUST match docker/telemetry/otel-collector-config.yaml + // spanmetrics dimensions and TxSpanNames.h (so both span sets aggregate + // under one dimension). + EXPECT_EQ(std::string_view(tx_apply_span::attr::stage), "stage"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::txType), "tx_type"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::terResult), "ter_result"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::applied), "applied"); +} + +TEST(TxApplySpanNames, stage_values_are_the_three_pipeline_stages) +{ + // The stage attribute carries exactly these three values; they become the + // spanmetrics `stage` dimension cardinality (3) and the dashboard filter. + EXPECT_EQ(std::string_view(tx_apply_span::val::preflight), "preflight"); + EXPECT_EQ(std::string_view(tx_apply_span::val::preclaim), "preclaim"); + EXPECT_EQ(std::string_view(tx_apply_span::val::apply), "apply"); +} diff --git a/src/tests/libxrpl/telemetry/ValidationTracker.cpp b/src/tests/libxrpl/telemetry/ValidationTracker.cpp new file mode 100644 index 00000000000..2542316e7a5 --- /dev/null +++ b/src/tests/libxrpl/telemetry/ValidationTracker.cpp @@ -0,0 +1,379 @@ +/** + * @file ValidationTracker.cpp + * Unit tests for xrpl::telemetry::ValidationTracker. + */ + +#include + +#include +#include + +#include + +#include +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +/** + * Helper to create a unique uint256 from an integer seed. + */ +static uint256 +makeHash(std::uint64_t n) +{ + return uint256(n); +} + +/** + * Test fixture providing a fresh ValidationTracker per test. + */ +class ValidationTrackerTest : public ::testing::Test +{ +protected: + ValidationTracker tracker_; +}; + +// --------------------------------------------------------------- +// 1. Normal agreement +// Record both our validation and network validation for the +// same hash, then reconcile after the grace period elapses. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, NormalAgreement) +{ + auto const hash = makeHash(1); + LedgerIndex const seq = 100; + + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + + // Immediately after recording, nothing is reconciled yet + // (grace period has not elapsed). + tracker_.reconcile(); + EXPECT_EQ(tracker_.totalValidationsSent(), 1u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); + + // Wait for the grace period (8 seconds) to elapse, then reconcile. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 1u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.agreements1h(), 1u); + EXPECT_EQ(tracker_.missed1h(), 0u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0); +} + +// --------------------------------------------------------------- +// 2. Missed validation +// Only the network validates; we never do. After grace period +// the event should be reconciled as a miss. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, MissedValidation) +{ + auto const hash = makeHash(2); + LedgerIndex const seq = 200; + + tracker_.recordNetworkValidation(hash, seq); + + // Wait for grace period then reconcile. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.agreements1h(), 0u); + EXPECT_EQ(tracker_.missed1h(), 1u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); +} + +// --------------------------------------------------------------- +// 3. Late repair +// Network validates first, grace period elapses (miss), then +// our validation arrives within the 5-minute repair window and +// the miss is flipped to an agreement. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, LateRepair) +{ + auto const hash = makeHash(3); + LedgerIndex const seq = 300; + + // Network validates, but we do not (yet). + tracker_.recordNetworkValidation(hash, seq); + + // Grace period elapses -- reconciled as a miss. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.missed1h(), 1u); + + // Late arrival of our validation (within repair window). + tracker_.recordOurValidation(hash, seq); + tracker_.reconcile(); + + // Miss should be repaired to agreement. + EXPECT_EQ(tracker_.totalAgreements(), 1u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.agreements1h(), 1u); + EXPECT_EQ(tracker_.missed1h(), 0u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0); +} + +// --------------------------------------------------------------- +// 4. Empty window returns 0% +// When no events have been recorded the percentage methods +// must return 0.0, not NaN or any other value. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, EmptyWindowReturnsZero) +{ + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); + EXPECT_DOUBLE_EQ(tracker_.agreementPct24h(), 0.0); + EXPECT_EQ(tracker_.agreements1h(), 0u); + EXPECT_EQ(tracker_.missed1h(), 0u); + EXPECT_EQ(tracker_.agreements24h(), 0u); + EXPECT_EQ(tracker_.missed24h(), 0u); + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 0u); + EXPECT_EQ(tracker_.totalMissedEver(), 0u); + EXPECT_EQ(tracker_.totalValidationsSent(), 0u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 0u); +} + +// --------------------------------------------------------------- +// 5. Grace period boundary +// Events recorded less than 8 seconds ago must NOT be +// reconciled. Verify that an immediate reconcile is a no-op. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, GracePeriodBoundary) +{ + auto const hash = makeHash(5); + LedgerIndex const seq = 500; + + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + + // Reconcile immediately -- grace period has not elapsed. + tracker_.reconcile(); + + // Nothing should be reconciled yet. + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.agreements1h(), 0u); + EXPECT_EQ(tracker_.missed1h(), 0u); + + // Lifetime send/check counters should still be incremented. + EXPECT_EQ(tracker_.totalValidationsSent(), 1u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); +} + +// --------------------------------------------------------------- +// 6. Max pending events -- trimming +// Add more than kMaxPendingEvents (1000) events. After +// reconciliation and a second reconcile pass the pending map +// should be trimmed. Lifetime totals must remain consistent. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, MaxPendingEventsTrimming) +{ + constexpr std::size_t kCount = 1100; + + for (std::size_t i = 0; i < kCount; ++i) + { + auto const hash = makeHash(i + 1); + auto const seq = static_cast(i + 1); + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + } + + EXPECT_EQ(tracker_.totalValidationsSent(), kCount); + EXPECT_EQ(tracker_.totalValidationsChecked(), kCount); + + // Wait for grace period so all events can be reconciled. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + // All events should be reconciled as agreements. + EXPECT_EQ(tracker_.totalAgreements(), kCount); + EXPECT_EQ(tracker_.totalMissed(), 0u); + + // Reconcile again to trigger pending eviction / trimming. + // The pending map should be trimmed, but totals remain correct. + tracker_.reconcile(); + EXPECT_EQ(tracker_.totalAgreements(), kCount); + EXPECT_EQ(tracker_.totalMissed(), 0u); +} + +// --------------------------------------------------------------- +// 7. Multiple distinct ledgers -- mixed results +// Record a mix of agreements and misses to verify that window +// counts and percentages are computed correctly. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, MixedAgreementsAndMisses) +{ + // 3 agreements: both sides validate. + for (int i = 1; i <= 3; ++i) + { + auto const hash = makeHash(static_cast(i)); + tracker_.recordOurValidation(hash, static_cast(i)); + tracker_.recordNetworkValidation(hash, static_cast(i)); + } + + // 2 misses: only network validates. + for (int i = 4; i <= 5; ++i) + { + auto const hash = makeHash(static_cast(i)); + tracker_.recordNetworkValidation(hash, static_cast(i)); + } + + // Wait for grace period then reconcile. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 3u); + EXPECT_EQ(tracker_.totalMissed(), 2u); + EXPECT_EQ(tracker_.agreements1h(), 3u); + EXPECT_EQ(tracker_.missed1h(), 2u); + + // 3 out of 5 = 60% + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 60.0); +} + +// --------------------------------------------------------------- +// 8. Duplicate recording for same hash +// Recording the same hash multiple times should not create +// duplicate pending entries or double-count totals beyond the +// per-call increments. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, DuplicateRecordingSameHash) +{ + auto const hash = makeHash(42); + LedgerIndex const seq = 42; + + // Record our validation twice for the same hash. + tracker_.recordOurValidation(hash, seq); + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + + // Each call increments the lifetime counter. + EXPECT_EQ(tracker_.totalValidationsSent(), 2u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); + + // But only one pending event exists, so only one agreement. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 1u); + EXPECT_EQ(tracker_.totalMissed(), 0u); +} + +// --------------------------------------------------------------- +// 9. Only-we-validated scenario +// We validate but the network does not. After grace period +// this should be a miss (not an agreement). +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, OnlyWeValidated) +{ + auto const hash = makeHash(99); + LedgerIndex const seq = 99; + + tracker_.recordOurValidation(hash, seq); + + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.missed1h(), 1u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); +} + +// --------------------------------------------------------------- +// 10. Gross miss tally is monotonic across a late repair +// The gross lifetime tallies (totalAgreementsEver/totalMissedEver) +// back the monotonic Prometheus _total counters. A late repair must +// move the NET totals (miss -> agreement) but must NOT move the gross +// tallies: a miss already counted stays counted, and the repair does +// not add a second (agreement) count for the same ledger. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, GrossMissedNeverDecrementsOnRepair) +{ + auto const hash = makeHash(10); + LedgerIndex const seq = 1000; + + // Network validates, we do not (yet). + tracker_.recordNetworkValidation(hash, seq); + + // Grace period elapses -- reconciled as a miss. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + // Net and gross both show exactly one initial miss, zero agreements. + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.totalMissedEver(), 1u); + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 0u); + + // Late arrival of our validation repairs the miss to an agreement. + tracker_.recordOurValidation(hash, seq); + tracker_.reconcile(); + + // Net totals reflect the repair... + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.totalAgreements(), 1u); + // ...but the gross tallies are frozen at first classification: the miss + // stays counted and no agreement was added (repair path excluded). + EXPECT_EQ(tracker_.totalMissedEver(), 1u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 0u); +} + +// --------------------------------------------------------------- +// 11. Gross tallies count initial classification only (additive) +// With a mix of initial agreements and misses the gross tallies equal +// the net totals. A subsequent repair shifts the net totals but leaves +// the gross tallies unchanged, and the gross sum equals the number of +// reconciled ledgers (the additive invariant the _total counters rely on). +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, GrossAgreementsCountInitialOnly) +{ + // 3 initial agreements: both sides validate. + for (int i = 1; i <= 3; ++i) + { + auto const h = makeHash(static_cast(i)); + tracker_.recordOurValidation(h, static_cast(i)); + tracker_.recordNetworkValidation(h, static_cast(i)); + } + + // 2 initial misses: only network validates. + for (int i = 4; i <= 5; ++i) + { + auto const h = makeHash(static_cast(i)); + tracker_.recordNetworkValidation(h, static_cast(i)); + } + + // Grace period elapses -- all five reconciled at first classification. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + // Before any repair, gross equals net. + EXPECT_EQ(tracker_.totalAgreements(), 3u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 3u); + EXPECT_EQ(tracker_.totalMissed(), 2u); + EXPECT_EQ(tracker_.totalMissedEver(), 2u); + + // Repair one of the misses (hash 4) within the repair window. + tracker_.recordOurValidation(makeHash(4), 4); + tracker_.reconcile(); + + // Net totals shift by the repair... + EXPECT_EQ(tracker_.totalAgreements(), 4u); + EXPECT_EQ(tracker_.totalMissed(), 1u); + // ...gross tallies stay at the initial classification. + EXPECT_EQ(tracker_.totalAgreementsEver(), 3u); + EXPECT_EQ(tracker_.totalMissedEver(), 2u); + + // Additive invariant: gross agree + gross miss == ledgers reconciled. + EXPECT_EQ(tracker_.totalAgreementsEver() + tracker_.totalMissedEver(), 5u); +} diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 4abf77f5786..5d1e61971d4 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -18,9 +18,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -62,6 +64,9 @@ #include #include #include +#include +#include +#include #include @@ -80,6 +85,7 @@ #include #include #include +#include #include #include #include @@ -225,6 +231,14 @@ RCLConsensus::Adaptor::share(RCLCxTx const& tx) void RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) { + // Child of the round span via its captured context (roundSpan_ is detached, + // so it is no longer the thread's ambient parent). + auto span = telemetry::SpanGuard::childSpan( + telemetry::consensus::span::proposalSend, roundSpanContext_); + span.setAttribute( + telemetry::consensus::span::attr::round, static_cast(proposal.proposeSeq())); + span.setAttribute(telemetry::consensus::span::attr::isBowOut, proposal.isBowOut()); + JLOG(j_.trace()) << (proposal.isBowOut() ? "We bow out: " : "We propose: ") << xrpl::to_string(proposal.prevLedger()) << " -> " << xrpl::to_string(proposal.position()); @@ -261,6 +275,11 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) app_.getHashRouter().addSuppression(suppression); + // Inject the current thread's active span context (e.g. the + // consensus round span from Phase 4) so receiving peers can link + // their proposal.receive span as a child of this trace. + telemetry::SpanGuard::injectCurrentContextToProtobuf(*prop.mutable_trace_context()); + app_.getOverlay().broadcast(prop); } @@ -327,6 +346,21 @@ RCLConsensus::Adaptor::onClose( NetClock::time_point const& closeTime, ConsensusMode mode) -> Result { + namespace cs = telemetry::consensus::span; + + // Child of the round span via its captured context (roundSpan_ is detached, + // so it is no longer the thread's ambient parent). + auto span = telemetry::SpanGuard::childSpan(cs::ledgerClose, roundSpanContext_); + span.setAttribute(cs::attr::ledgerSeq, static_cast(ledger.ledger->header().seq) + 1); + span.setAttribute(cs::attr::mode, toDisplayString(mode).c_str()); + span.setAttribute( + cs::attr::txCountOpen, static_cast(app_.getOpenLedger().current()->txCount())); + span.setAttribute( + cs::attr::closeTimeResolutionMs, + static_cast( + std::chrono::duration_cast(ledger.closeTimeResolution()) + .count())); + bool const wrongLCL = mode == ConsensusMode::WrongLedger; bool const proposing = mode == ConsensusMode::Proposing; @@ -422,7 +456,15 @@ RCLConsensus::Adaptor::onForceAccept( ConsensusMode const& mode, json::Value&& consensusJson) { - doAccept(result, prevLedger, closeResolution, rawCloseTimes, mode, std::move(consensusJson)); + auto acceptSpan = makeAcceptSpan(result); + doAccept( + result, + prevLedger, + closeResolution, + rawCloseTimes, + mode, + std::move(consensusJson), + std::move(acceptSpan)); } void @@ -435,22 +477,76 @@ RCLConsensus::Adaptor::onAccept( json::Value&& consensusJson, bool const validating) { + auto acceptSpan = makeAcceptSpan(result); + app_.getJobQueue().addJob( JtAccept, "AcceptLedger", // NOLINTNEXTLINE(cppcoreguidelines-misleading-capture-default-by-value) - [=, this, cj = std::move(consensusJson)]() mutable { + [=, this, cj = std::move(consensusJson), sp = std::move(acceptSpan)]() mutable { // Note that no lock is held or acquired during this job. // This is because generic Consensus guarantees that once a ledger // is accepted, the consensus results and capture by reference state // will not change until startRound is called (which happens via // endConsensus). RclConsensusLogger clog("onAccept", validating, j_); - this->doAccept(result, prevLedger, closeResolution, rawCloseTimes, mode, std::move(cj)); + this->doAccept( + result, + prevLedger, + closeResolution, + rawCloseTimes, + mode, + std::move(cj), + std::move(sp)); this->app_.getOPs().endConsensus(clog.ss()); }); } +std::shared_ptr +RCLConsensus::Adaptor::makeAcceptSpan(Result const& result) +{ + namespace cs = telemetry::consensus::span; + + auto span = std::make_shared( + telemetry::SpanGuard::childSpan(cs::accept, roundSpanContext_)); + span->setAttribute(cs::attr::proposers, static_cast(result.proposers)); + span->setAttribute( + cs::attr::roundTimeMs, static_cast(result.roundTime.read().count())); + span->setAttribute(cs::attr::quorum, static_cast(app_.getValidators().quorum())); + span->setAttribute(cs::attr::disputesCount, static_cast(result.disputes.size())); + char const* stateStr = [&] { + switch (result.state) + { + case ConsensusState::Yes: + return "yes"; + case ConsensusState::MovedOn: + return "moved_on"; + case ConsensusState::Expired: + return "expired"; + default: + return "no"; + } + }(); + span->setAttribute(cs::attr::consensusState, stateStr); + + // Capture the accept span's context so createValidationSpan() — which + // runs on the jtACCEPT worker thread — can link the validation.send + // span to the accept span (matching the design diagram and the + // "validation follows acceptance" causal model). + if (*span) + { + acceptSpanContext_ = span->captureContext(); + // The accept span is moved into the JtAccept worker (onAccept) and + // ended there. Detach its Scope now, on this thread, AFTER the capture + // above, so it is not popped on the wrong thread. accept.apply parents + // via acceptSpanContext_ (captured above), so no ambient child is + // orphaned on either the sync (onForceAccept) or async (onAccept) path. + // Rebuild the shared_ptr from the detached rvalue (move-assign deleted). + span = std::make_shared(std::move(*span).detached()); + } + return span; +} + void RCLConsensus::Adaptor::doAccept( Result const& result, @@ -458,8 +554,11 @@ RCLConsensus::Adaptor::doAccept( NetClock::duration closeResolution, ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, - json::Value&& consensusJson) + json::Value&& consensusJson, + std::shared_ptr acceptSpan) { + namespace cs = telemetry::consensus::span; + prevProposers_ = result.proposers; prevRoundTime_ = result.roundTime.read(); @@ -486,6 +585,54 @@ RCLConsensus::Adaptor::doAccept( closeTimeCorrect = true; } + // Parent accept.apply via the captured accept context (acceptSpanContext_) + // rather than the accept guard's ambient Scope: the accept span is detached + // (its Scope no longer sits on this thread), so an explicit context is used + // for both the sync (onForceAccept) and async (onAccept) paths. Falls back + // to the round context if the accept span was null. + auto doAcceptSpan = acceptSpanContext_.isValid() + ? telemetry::SpanGuard::childSpan(cs::acceptApply, acceptSpanContext_) + : telemetry::SpanGuard::childSpan(cs::acceptApply, roundSpanContext_); + doAcceptSpan.setAttribute(cs::attr::ledgerSeq, static_cast(prevLedger.seq()) + 1); + doAcceptSpan.setAttribute( + cs::attr::closeTime, static_cast(consensusCloseTime.time_since_epoch().count())); + doAcceptSpan.setAttribute(cs::attr::closeTimeCorrect, closeTimeCorrect); + doAcceptSpan.setAttribute( + cs::attr::closeResolutionMs, + static_cast( + std::chrono::duration_cast(closeResolution).count())); + doAcceptSpan.setAttribute( + cs::attr::consensusState, + consensusFail ? std::string_view{cs::val::movedOn} : std::string_view{cs::val::finished}); + doAcceptSpan.setAttribute(cs::attr::proposing, proposing); + doAcceptSpan.setAttribute( + cs::attr::roundTimeMs, static_cast(result.roundTime.read().count())); + doAcceptSpan.setAttribute( + cs::attr::parentCloseTime, + static_cast(prevLedger.closeTime().time_since_epoch().count())); + doAcceptSpan.setAttribute( + cs::attr::closeTimeSelf, + static_cast(rawCloseTimes.self.time_since_epoch().count())); + doAcceptSpan.setAttribute( + cs::attr::closeTimeVoteBins, static_cast(rawCloseTimes.peers.size())); + doAcceptSpan.setAttribute( + cs::attr::disputesResolvedCount, static_cast(result.disputes.size())); + { + auto const prevRes = prevLedger.closeTimeResolution(); + auto const dir = [&]() -> std::string { + if (closeResolution > prevRes) + { + return "increased"; + } + if (closeResolution < prevRes) + { + return "decreased"; + } + return "unchanged"; + }(); + doAcceptSpan.setAttribute(cs::attr::resolutionDirection, std::move(dir)); + } + JLOG(j_.debug()) << "Report: Prop=" << (proposing ? "yes" : "no") << " val=" << (validating_ ? "yes" : "no") << " corLCL=" << (haveCorrectLCL ? "yes" : "no") @@ -503,12 +650,16 @@ RCLConsensus::Adaptor::doAccept( JLOG(j_.debug()) << "Building canonical tx set: " << retriableTxs.key(); + int64_t txCount = 0; for (auto const& item : *result.txns.map) { try { retriableTxs.insert(std::make_shared(SerialIter{item.slice()})); JLOG(j_.debug()) << " Tx: " << item.key(); + ++txCount; + auto const txHash = to_string(item.key()); + doAcceptSpan.addEvent(cs::event::txIncluded, {{cs::attr::txId, txHash}}); } catch (std::exception const& ex) { @@ -516,6 +667,7 @@ RCLConsensus::Adaptor::doAccept( JLOG(j_.warn()) << " Tx: " << item.key() << " throws: " << ex.what(); } } + doAcceptSpan.setAttribute(cs::attr::txCount, txCount); auto built = buildLCL( prevLedger, @@ -587,6 +739,10 @@ RCLConsensus::Adaptor::doAccept( // See if we can accept a ledger as fully-validated ledgerMaster_.consensusBuilt(built.ledger, result.txns.id(), std::move(consensusJson)); + // Record ledger close for OTel dashboard parity counter. + if (auto* mr = app_.getMetricsRegistry()) + mr->incrementLedgersClosed(); + //------------------------------------------------------------------------- { // Apply disputed transactions that didn't get in @@ -807,6 +963,19 @@ RCLConsensus::Adaptor::buildLCL( void RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns, bool proposing) { + auto valSpan = createValidationSpan(); + if (valSpan) + { + namespace cs = telemetry::consensus::span; + valSpan->setAttribute(cs::attr::ledgerSeq, static_cast(ledger.seq())); + valSpan->setAttribute( + cs::attr::ledgerHash, to_string(ledger.ledger->header().hash).c_str()); + valSpan->setAttribute(cs::attr::proposing, proposing); + // proposing implies a full validation (vfFullValidation is set on + // the STValidation only when proposing — see below). + valSpan->setAttribute(cs::attr::fullValidation, proposing); + } + using namespace std::chrono_literals; auto validationTime = app_.getTimeKeeper().closeTime(); @@ -814,6 +983,13 @@ RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns, validationTime = lastValidationTime_ + 1s; lastValidationTime_ = validationTime; + if (valSpan) + { + valSpan->setAttribute( + telemetry::consensus::span::attr::validationSignTime, + static_cast(validationTime.time_since_epoch().count())); + } + if (!validatorKeys_.keys) { JLOG(j_.warn()) << "RCLConsensus::Adaptor::validate: ValidatorKeys " @@ -885,15 +1061,41 @@ RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns, // Broadcast to all our peers: protocol::TMValidation val; val.set_validation(serialized.data(), serialized.size()); + // Inject the current thread's active span context so receiving + // peers can link their validation.receive span as a child. + // + // The trace_context appended below is outside the signature on + // `serialized`, so it is not covered by validation authenticity. + // Downstream consumers treat it as advisory only. A signature-covered + // trace context is a possible future enhancement. + telemetry::SpanGuard::injectCurrentContextToProtobuf(*val.mutable_trace_context()); app_.getOverlay().broadcast(val); // Publish to all our subscribers: app_.getOPs().pubValidation(v); + + // Record validation sent for OTel dashboard parity counter. + if (auto* mr = app_.getMetricsRegistry()) + { + mr->incrementValidationsSent(); + // Record our validation for the agreement tracker so it can + // compare against network-validated ledgers. + mr->getValidationTracker().recordOurValidation(ledger.id(), ledger.seq()); + } } void RCLConsensus::Adaptor::onModeChange(ConsensusMode before, ConsensusMode after) { + namespace cs = telemetry::consensus::span; + + // Child of the round span via its captured context (roundSpan_ is detached, + // so it is no longer the thread's ambient parent). A mode change outside a + // round leaves roundSpanContext_ invalid, yielding a null guard (no-op). + auto span = telemetry::SpanGuard::childSpan(cs::modeChange, roundSpanContext_); + span.setAttribute(cs::attr::modeOld, toDisplayString(before).c_str()); + span.setAttribute(cs::attr::modeNew, toDisplayString(after).c_str()); + JLOG(j_.info()) << "Consensus mode change before=" << to_string(before) << ", after=" << to_string(after); @@ -1018,6 +1220,8 @@ RCLConsensus::Adaptor::preStartRound(RCLCxLedger const& prevLgr, hash_setsetAttribute(cs::attr::ledgerId, to_string(prevLgr.id()).c_str()); + roundSpan_->setAttribute(cs::attr::ledgerSeq, static_cast(prevLgr.seq()) + 1); + roundSpan_->setAttribute(cs::attr::mode, toDisplayString(mode_.load()).c_str()); + roundSpan_->setAttribute(cs::attr::traceStrategy, strategy.c_str()); + roundSpan_->setAttribute(cs::attr::roundId, static_cast(prevLgr.seq()) + 1); + roundSpan_->setAttribute(cs::attr::previousLedgerSeq, static_cast(prevLgr.seq())); + roundSpan_->setAttribute(cs::attr::previousProposers, static_cast(prevProposers_)); + roundSpan_->setAttribute( + cs::attr::previousRoundTimeMs, static_cast(prevRoundTime_.load().count())); + roundSpan_->setAttribute(cs::attr::consensusPhase, cs::val::phaseOpen); + + roundSpan_->addEvent(cs::event::phaseOpen); + + roundSpanContext_ = roundSpan_->captureContext(); + + // roundSpanContext_ (captured above) is the durable handle that child + // spans on other threads link to. The guard itself is reset() on a + // different worker than it was emplaced on, so detach its Scope now -- + // AFTER the context capture -- to avoid a wrong-thread Scope pop. Re-emplace + // the detached guard in place (SpanGuard move-assignment is deleted). + roundSpan_.emplace(std::move(*roundSpan_).detached()); +} + +std::optional +RCLConsensus::Adaptor::createValidationSpan() +{ + namespace cs = telemetry::consensus::span; + + // Prefer linking to the accept span (matches the design diagram and + // the "validation follows acceptance" causal model). Fall back to the + // round span only if the accept context isn't yet captured (e.g. + // tracing started after onAccept, or makeAcceptSpan returned a null + // guard). + if (acceptSpanContext_.isValid()) + { + return telemetry::SpanGuard::linkedSpan(cs::validationSend, acceptSpanContext_); + } + + if (!roundSpanContext_.isValid()) + { + return std::nullopt; + } + + return telemetry::SpanGuard::linkedSpan(cs::validationSend, roundSpanContext_); +} + +void +RCLConsensus::Adaptor::onPhaseEvent(std::string_view eventName, std::string_view phaseLabel) +{ + namespace cs = telemetry::consensus::span; + + if (!roundSpan_ || !*roundSpan_) + { + return; + } + + roundSpan_->addEvent(eventName); + if (!phaseLabel.empty()) + { + roundSpan_->setAttribute(cs::attr::consensusPhase, phaseLabel); + } +} + +void +RCLConsensus::Adaptor::onOutcomeEvent(std::string_view eventName) +{ + if (!roundSpan_ || !*roundSpan_) + { + return; + } + roundSpan_->addEvent(eventName); +} + void RCLConsensus::startRound( NetClock::time_point const& now, diff --git a/src/xrpld/app/consensus/RCLConsensus.h b/src/xrpld/app/consensus/RCLConsensus.h index 4ffe18a7a89..39a584194cf 100644 --- a/src/xrpld/app/consensus/RCLConsensus.h +++ b/src/xrpld/app/consensus/RCLConsensus.h @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -34,6 +35,7 @@ #include #include #include +#include #include namespace xrpl { @@ -86,6 +88,60 @@ class RCLConsensus RCLCensorshipDetector censorshipDetector_; NegativeUNLVote nUnlVote_; + /** + * Span for the current consensus round. + * + * Created in preStartRound(), ended (via reset()) when the next + * round begins. When consensusTraceStrategy is "deterministic", + * the trace_id is derived from previousLedger.id() so that all + * validators in the same round share the same trace_id. + * + * Stored detached: its Scope is stripped right after roundSpanContext_ + * is captured, because it is emplaced and reset on different job + * workers. Child spans link via roundSpanContext_, not the ambient span. + */ + std::optional roundSpan_; + + /** + * SpanContext snapshot of the current round span. + * + * Captured in startRoundTracing() as a lightweight value-type copy + * so that createValidationSpan() — which runs on the jtACCEPT + * worker thread — can build span links without accessing roundSpan_ + * across threads. + */ + telemetry::SpanContext roundSpanContext_; + + /** + * SpanContext of the prior round, captured before roundSpanContext_ + * is overwritten by the next round's startRoundTracing(). + * + * Used as a follows-from link target on the new round's span so + * consecutive rounds (each with its own deterministic trace-id) + * remain navigable in trace UIs as a connected sequence. + */ + telemetry::SpanContext prevRoundSpanContext_; + + /** + * SpanContext snapshot of the current round's accept span. + * + * Captured in makeAcceptSpan() and consumed by createValidationSpan() + * on the jtACCEPT worker thread so the validation.send span can be + * follows-from linked to consensus.accept (matching the design doc + * and span hierarchy diagram). Reset on each startRoundTracing() + * to prevent a stale prior-round context from being linked. + * + * Thread safety: same model as roundSpanContext_. The write in + * makeAcceptSpan happens on the main consensus thread under + * mutex_, and the read in createValidationSpan happens on the + * jtACCEPT worker thread that was posted by makeAcceptSpan. + * The job queue dispatch establishes a happens-before relation, + * so no atomic synchronization is needed. The next reset (in + * startRoundTracing) only runs after endConsensus → startRound, + * which by Consensus design only fires after doAccept completes. + */ + telemetry::SpanContext acceptSpanContext_; + public: using Ledger_t = RCLCxLedger; using NodeID_t = NodeID; @@ -178,6 +234,75 @@ class RCLConsensus return parms_; } + /** + * Set up the consensus round span and link it to the previous round. + * + * Saves the previous round's context for span-link construction, + * ends the old round span, and creates a new "consensus.round" span. + * Depending on the configured trace strategy the trace_id is either + * deterministic (derived from prevLgr hash) or random. + * + * @param prevLgr The ledger that will be the prior ledger for the + * new round. + */ + void + startRoundTracing(RCLCxLedger const& prevLgr); + + /** + * Create the "consensus.validation.send" span linked to the round. + * + * @return An engaged optional SpanGuard if tracing is active, + * std::nullopt otherwise. + */ + std::optional + createValidationSpan(); + + /** + * Record a phase-transition event on the active round span. + * + * Called from the engine at each phase boundary + * (open/establish/accepted/recovery) so the round span carries a + * complete timeline of state changes. Also updates the + * `consensus_phase` attribute to the current phase name. + * + * @param eventName Event name (e.g. "phase.establish"). + * @param phaseLabel String value for the consensus_phase attr + * ("open"/"establish"/"accepted"). Empty to skip + * the attribute update (e.g. for "recovery"). + */ + void + onPhaseEvent(std::string_view eventName, std::string_view phaseLabel); + + /** + * Record a checkConsensus outcome event on the round span. + * + * Called from the engine at the establish→accepted transition so + * the path that drove acceptance (Yes / MovedOn / Expired) is + * queryable from the round-level trace. + * + * @param eventName Event name (e.g. "outcome.yes"). + */ + void + onOutcomeEvent(std::string_view eventName); + + /** + * Captured context of the current round span. + * + * The generic engine uses this to parent the phase spans it owns + * (consensus.phase.open, consensus.establish) under the round span + * without touching roundSpan_ across threads. roundSpan_ is detached + * after this context is captured, so it is no longer the thread's + * ambient parent; child spans must link via this context. Returns an + * invalid context before the round span is created. + * + * @return The round span's captured context, or an invalid context. + */ + telemetry::SpanContext + roundSpanContext() const + { + return roundSpanContext_; + } + private: //--------------------------------------------------------------------- // The following members implement the generic Consensus requirements @@ -363,9 +488,19 @@ class RCLConsensus void notify(protocol::NodeEvent ne, RCLCxLedger const& ledger, bool haveCorrectLCL); + /** + * Create a consensus.accept span as a child of the round span. + * Returned via shared_ptr so it can be captured into the + * jtACCEPT lambda and live until doAccept completes. + */ + std::shared_ptr + makeAcceptSpan(Result const& result); + /** * Accept a new ledger based on the given transactions. * + * @param acceptSpan Parent span created by makeAcceptSpan(); + * accept.apply is created as its child. * @ref onAccept */ void @@ -375,7 +510,8 @@ class RCLConsensus NetClock::duration closeResolution, ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, - json::Value&& consensusJson); + json::Value&& consensusJson, + std::shared_ptr acceptSpan); /** * Build the new last closed ledger. diff --git a/src/xrpld/app/ledger/InboundLedger.h b/src/xrpld/app/ledger/InboundLedger.h index d8a9ddf46b8..f5009d4b219 100644 --- a/src/xrpld/app/ledger/InboundLedger.h +++ b/src/xrpld/app/ledger/InboundLedger.h @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -193,6 +195,18 @@ class InboundLedger final : public TimeoutCounter, receivedData_; bool receiveDispatched_{false}; std::unique_ptr peerSet_; + + /** + * Spans the acquire lifecycle: started in init(), finalized in done() + * with the outcome (complete/failed), timeout count, and peer count. + * Gives operators visibility into back-fill / fork-recovery cost, which + * previously emitted no span or metric. + * Stored detached: emplaced by the acquiring thread, reset on a + * JtLedgerData worker. detached() strips the thread-local Scope so the + * guard can be destroyed on the worker without corrupting the origin + * thread's context stack. + */ + std::optional acquireSpan_; }; } // namespace xrpl diff --git a/src/xrpld/app/ledger/LedgerHistory.cpp b/src/xrpld/app/ledger/LedgerHistory.cpp index 8734faa1fb2..d94f114d33c 100644 --- a/src/xrpld/app/ledger/LedgerHistory.cpp +++ b/src/xrpld/app/ledger/LedgerHistory.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #include #include @@ -323,11 +325,19 @@ LedgerHistory::handleMismatch( auto builtLedger = getLedgerByHash(built); auto validLedger = getLedgerByHash(valid); + // Records the classified mismatch reason as a labeled OTel counter so + // fork diagnosis is a queryable time series, not just a log grep. + auto recordReason = [this](std::string_view reason) { + if (auto* mr = app_.getMetricsRegistry()) + mr->incrementLedgerHistoryMismatch(reason); + }; + if (!builtLedger || !validLedger) { JLOG(j_.error()) << "MISMATCH cannot be analyzed:" << " builtLedger: " << to_string(built) << " -> " << builtLedger << " validLedger: " << to_string(valid) << " -> " << validLedger; + recordReason("unknown"); return; } @@ -349,6 +359,7 @@ LedgerHistory::handleMismatch( if (builtLedger->header().parentHash != validLedger->header().parentHash) { JLOG(j_.error()) << "MISMATCH on prior ledger"; + recordReason("prior_ledger"); return; } @@ -356,9 +367,16 @@ LedgerHistory::handleMismatch( if (builtLedger->header().closeTime != validLedger->header().closeTime) { JLOG(j_.error()) << "MISMATCH on close time"; + recordReason("close_time"); return; } + // Tracks whether the mismatch reason has already been recorded. The + // consensus tx-set hash disagreement is the root cause, so it is counted + // once here; the tx-level comparison below still logs its diagnostics but + // must not record a second reason for the same mismatch event. + bool reasonRecorded = false; + if (builtConsensusHash && validatedConsensusHash) { if (builtConsensusHash != validatedConsensusHash) @@ -366,6 +384,8 @@ LedgerHistory::handleMismatch( JLOG(j_.error()) << "MISMATCH on consensus transaction set " << " built: " << to_string(*builtConsensusHash) << " validated: " << to_string(*validatedConsensusHash); + recordReason("consensus_txset"); + reasonRecorded = true; } else { @@ -381,11 +401,15 @@ LedgerHistory::handleMismatch( if (builtTx == validTx) { JLOG(j_.error()) << "MISMATCH with same " << builtTx.size() << " transactions"; + if (!reasonRecorded) + recordReason("same_txset_diff_result"); } else { JLOG(j_.error()) << "MISMATCH with " << builtTx.size() << " built and " << validTx.size() << " valid transactions."; + if (!reasonRecorded) + recordReason("different_txset"); } JLOG(j_.error()) << "built\n" << getJson({*builtLedger, {}}); diff --git a/src/xrpld/app/ledger/detail/BuildLedger.cpp b/src/xrpld/app/ledger/detail/BuildLedger.cpp index e9c01c71339..d0e8967c099 100644 --- a/src/xrpld/app/ledger/detail/BuildLedger.cpp +++ b/src/xrpld/app/ledger/detail/BuildLedger.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,9 +19,13 @@ #include #include // IWYU pragma: keep #include +#include +#include #include +#include #include +#include #include #include #include @@ -44,6 +49,9 @@ buildLedgerImpl( beast::Journal j, ApplyTxs&& applyTxs) { + using namespace telemetry; + auto buildSpan = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::build); + auto built = std::make_shared(*parent, closeTime); if (built->isFlagLedger()) @@ -77,6 +85,14 @@ buildLedgerImpl( built->header().seq < kXrpLedgerEarliestFees || built->read(keylet::feeSettings()), "xrpl::buildLedgerImpl : valid ledger fees"); built->setAccepted(closeTime, closeResolution, closeTimeCorrect); + buildSpan.setAttribute(ledger_span::attr::ledgerSeq, static_cast(built->header().seq)); + buildSpan.setAttribute( + ledger_span::attr::closeTime, static_cast(closeTime.time_since_epoch().count())); + buildSpan.setAttribute(ledger_span::attr::closeTimeCorrect, closeTimeCorrect); + buildSpan.setAttribute( + ledger_span::attr::closeResolutionMs, + static_cast( + std::chrono::duration_cast(closeResolution).count())); return built; } @@ -101,6 +117,9 @@ applyTransactions( OpenView& view, beast::Journal j) { + using namespace telemetry; + auto applySpan = SpanGuard::span(TraceCategory::Transactions, seg::tx, ledger_span::op::apply); + bool certainRetry = true; std::size_t count = 0; @@ -167,6 +186,8 @@ applyTransactions( // If there are any transactions left, we must have // tried them in at least one final pass XRPL_ASSERT(txns.empty() || !certainRetry, "xrpl::applyTransactions : retry transactions"); + applySpan.setAttribute(ledger_span::attr::txCount, static_cast(count)); + applySpan.setAttribute(ledger_span::attr::txFailed, static_cast(failed.size())); return count; } diff --git a/src/xrpld/app/ledger/detail/InboundLedger.cpp b/src/xrpld/app/ledger/detail/InboundLedger.cpp index 627a5d574f4..84f77e5d955 100644 --- a/src/xrpld/app/ledger/detail/InboundLedger.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedger.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,8 @@ #include #include #include +#include +#include #include @@ -46,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +99,37 @@ InboundLedger::init(ScopedLockType& collectionLock) ScopedLockType sl(mtx_); collectionLock.unlock(); + // Span the acquire lifecycle so back-fill / fork-recovery cost is + // observable. Finalized in done() with the outcome and timeout count. + { + using namespace telemetry; + // Detach the Scope: this guard is emplaced here but reset() on a + // JtLedgerData worker thread. Detaching strips the thread-local Scope + // so its destruction there does not corrupt this thread's context stack. + acquireSpan_.emplace( + SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::acquire) + .detached()); + if (*acquireSpan_) + { + acquireSpan_->setAttribute(ledger_span::attr::ledgerSeq, static_cast(seq_)); + // Map the acquire reason to its span-attribute value. A switch + // keeps the mapping flat and exhaustive over the Reason enum. + std::string_view reasonVal = ledger_span::val::generic; + switch (reason_) + { + case Reason::HISTORY: + reasonVal = ledger_span::val::history; + break; + case Reason::CONSENSUS: + reasonVal = ledger_span::val::consensus; + break; + case Reason::GENERIC: + break; + } + acquireSpan_->setAttribute(ledger_span::attr::acquireReason, reasonVal); + } + } + tryDB(app_.getNodeFamily().db()); if (failed_) return; @@ -418,6 +453,21 @@ InboundLedger::done() signaled_ = true; touch(); + // Finalize the acquire span with the outcome, timeout count, and peer + // count, then end it (reset) so its duration is exported. + if (acquireSpan_ && *acquireSpan_) + { + using namespace telemetry; + acquireSpan_->setAttribute( + ledger_span::attr::outcome, + failed_ ? std::string_view(ledger_span::val::failed) + : std::string_view(ledger_span::val::complete)); + acquireSpan_->setAttribute(ledger_span::attr::timeouts, static_cast(timeouts_)); + acquireSpan_->setAttribute( + ledger_span::attr::peerCount, static_cast(getPeerCount())); + } + acquireSpan_.reset(); + JLOG(journal_.debug()) << "Acquire " << hash_ << (failed_ ? " fail " : " ") << ((timeouts_ == 0) ? std::string() diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index 2bd83b0f18b..bc5b517127c 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -56,6 +58,8 @@ #include #include #include +#include +#include #include @@ -292,6 +296,11 @@ LedgerMaster::setValidLedger(std::shared_ptr const& l) (void)maxLedgerDifference_; validLedgerSeq_ = l->header().seq; + // Record the network-validated ledger for the agreement tracker so it + // can compare against our own validations. + if (auto* mr = app_.getMetricsRegistry()) + mr->getValidationTracker().recordNetworkValidation(l->header().hash, l->header().seq); + app_.getOPs().updateLocalTx(*l); app_.getSHAMapStore().onLedgerClosed(getValidatedLedger()); ledgerHistory_.validatedLedger(l, consensusHash); @@ -450,6 +459,10 @@ LedgerMaster::fixIndex(LedgerIndex ledgerIndex, LedgerHash const& ledgerHash) bool LedgerMaster::storeLedger(std::shared_ptr ledger) { + using namespace telemetry; + auto span = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::store); + span.setAttribute(ledger_span::attr::ledgerSeq, static_cast(ledger->header().seq)); + bool const validated = ledger->header().validated; // Returns true if we already had the ledger return ledgerHistory_.insert(ledger, validated); @@ -970,6 +983,11 @@ LedgerMaster::checkAccept(std::shared_ptr const& ledger) return; } + using namespace telemetry; + auto valSpan = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::validate); + valSpan.setAttribute(ledger_span::attr::ledgerSeq, static_cast(ledger->header().seq)); + valSpan.setAttribute(ledger_span::attr::validations, static_cast(tvc)); + JLOG(journal_.info()) << "Advancing accepted ledger to " << ledger->header().seq << " with >= " << minVal << " validations"; diff --git a/src/xrpld/app/ledger/detail/LedgerSpanNames.h b/src/xrpld/app/ledger/detail/LedgerSpanNames.h new file mode 100644 index 00000000000..4de4d248245 --- /dev/null +++ b/src/xrpld/app/ledger/detail/LedgerSpanNames.h @@ -0,0 +1,76 @@ +#pragma once + +/** + * Compile-time span name constants for ledger tracing. + * + * Used by BuildLedger and LedgerMaster for ledger lifecycle spans. + * Built on StaticStr/join() from SpanNames.h. + * + * Span hierarchy: + * + * ledger.build (BuildLedger — ledger construction) + * ledger.store (LedgerMaster — ledger storage) + * ledger.validate (LedgerMaster — ledger validation acceptance) + * ledger.acquire (InboundLedger — fetch a missing ledger from peers) + * tx.apply (BuildLedger — transaction application) + */ + +#include + +namespace xrpl::telemetry::ledger_span { + +// ===== Span operation suffixes =============================================== + +namespace op { +inline constexpr auto build = makeStr("build"); +inline constexpr auto store = makeStr("store"); +inline constexpr auto validate = makeStr("validate"); +inline constexpr auto apply = makeStr("apply"); +inline constexpr auto acquire = makeStr("acquire"); +} // namespace op + +// ===== Attribute keys ======================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). + */ +using ::xrpl::telemetry::attr::closeResolutionMs; +using ::xrpl::telemetry::attr::closeTime; +using ::xrpl::telemetry::attr::closeTimeCorrect; +using ::xrpl::telemetry::attr::ledgerHash; +using ::xrpl::telemetry::attr::ledgerSeq; + +/** + * Domain-owned bare attrs. + */ +inline constexpr auto txCount = makeStr("tx_count"); +inline constexpr auto txFailed = makeStr("tx_failed"); +inline constexpr auto validations = makeStr("validations"); + +/** + * ledger.acquire attrs (InboundLedger fetch lifecycle). + */ +inline constexpr auto acquireReason = makeStr("acquire_reason"); +inline constexpr auto timeouts = makeStr("timeouts"); +inline constexpr auto peerCount = makeStr("peer_count"); +inline constexpr auto outcome = makeStr("outcome"); +} // namespace attr + +// ===== Attribute values ====================================================== + +namespace val { +/** + * ledger.acquire outcome values. + */ +inline constexpr auto complete = makeStr("complete"); +inline constexpr auto failed = makeStr("failed"); +/** + * ledger.acquire reason values (mirror InboundLedger::Reason). + */ +inline constexpr auto history = makeStr("history"); +inline constexpr auto consensus = makeStr("consensus"); +inline constexpr auto generic = makeStr("generic"); +} // namespace val + +} // namespace xrpl::telemetry::ledger_span diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index d329475874b..f6fd5896954 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -81,10 +82,12 @@ #include #include // IWYU pragma: keep #include +#include #include #include #include // IWYU pragma: keep #include +#include #include #include #include @@ -98,6 +101,7 @@ #include #include #include +#include #include #include @@ -149,6 +153,7 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal_; beast::IOLatencyProbe probe_; std::atomic lastSample_; + std::atomic firstSample_; public: IOLatencySampler( @@ -156,7 +161,7 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal, std::chrono::milliseconds interval, boost::asio::io_context& ios) - : event_(std::move(ev)), journal_(journal), probe_(interval, ios) + : event_(std::move(ev)), journal_(journal), probe_(interval, ios), firstSample_(true) { } @@ -175,7 +180,10 @@ class ApplicationImp : public Application, public BasicApp lastSample_ = lastSample; - if (lastSample >= 10ms) + // Always emit the first sample so the metric is registered in + // downstream stores (Prometheus via StatsD). After that, only + // report latency >= 10 ms to avoid flooding with sub-ms values. + if (firstSample_.exchange(false) || lastSample >= 10ms) event_.notify(lastSample); if (lastSample >= 500ms) { @@ -212,6 +220,12 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal_; std::unique_ptr perfLog_; + std::unique_ptr telemetry_; + /** + * OTel metrics registry for gap-fill metrics (counters, histograms, + * observable gauges). Created after telemetry_ during setup(). + */ + std::unique_ptr metricsRegistry_; Application::MutexType masterMutex_; // Required by the SHAMapStore @@ -321,9 +335,24 @@ class ApplicationImp : public Application, public BasicApp *this, logs_->journal("PerfLog"), [this] { signalStop("PerfLog"); })) + , telemetry_( + telemetry::makeTelemetry( + telemetry::makeTelemetrySetup( + config_->section("telemetry"), + "", // Updated later via setServiceInstanceId() + BuildInfo::getVersionString(), + config_->networkId), + logs_->journal("Telemetry"))) + , txMaster_(*this) , collectorManager_(makeCollectorManager( config_->section(Sections::kInsight), + // Fall back to [telemetry] service_name so metrics inherit the + // same service.name as traces when [insight] omits it. Network + // type is derived from [network_id] via the shared telemetry + // helper, keeping metrics and traces on one network label. + config_->section("telemetry").valueOr("service_name", ""), + telemetry::networkTypeFromId(config_->networkId), logs_->journal("Collector"))) , jobQueue_( std::make_unique( @@ -367,7 +396,7 @@ class ApplicationImp : public Application, public BasicApp stopwatch(), logs_->journal("TaggedCache")) , cachedSLEs_( - "Cached SLEs", + "Cached_SLEs", 0, std::chrono::minutes(1), stopwatch(), @@ -649,6 +678,18 @@ class ApplicationImp : public Application, public BasicApp return *perfLog_; } + telemetry::Telemetry& + getTelemetry() override + { + return *telemetry_; + } + + telemetry::MetricsRegistry* + getMetricsRegistry() override + { + return metricsRegistry_.get(); + } + NodeCache& getTempNodeCache() override { @@ -1297,6 +1338,19 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) nodeIdentity_ = getNodeIdentity(*this, cmdline); + // Now that the node identity is known, inject it into the telemetry + // resource attributes — but only if the user didn't already set a + // custom service_instance_id in [telemetry]. The Telemetry object + // was constructed with an empty serviceInstanceId because + // nodeIdentity_ is not available in the member initializer list. + if (!config_->section("telemetry").exists("service_instance_id")) + telemetry_->setServiceInstanceId(toBase58(TokenType::NodePublic, nodeIdentity_->first)); + + // Create the OTel MetricsRegistry for gap-fill metrics (counters, + // histograms, observable gauges). It is started later in start(). + metricsRegistry_ = std::make_unique( + telemetry_->isEnabled(), *this, logs_->journal("MetricsRegistry")); + if (!cluster_->load(config().section(Sections::kClusterNodes))) { JLOG(journal_.fatal()) << "Invalid entry in cluster configuration."; @@ -1497,6 +1551,32 @@ ApplicationImp::start(bool withTimers) setEntropyTimer(); } + // Start telemetry before the other services so the tracer is live and + // their startup/early activity can be traced. (The metrics MeterProvider + // is already published in the Telemetry constructor — before subsystems + // create their beast::insight instruments during ApplicationImp init — so + // start() ordering does not affect metrics; only tracing benefits here.) + telemetry_->start(); + + // Start the metrics pipeline right after telemetry, before subsystems + // begin recording; the endpoint uses the same base URL with the + // /v1/metrics path. + if (metricsRegistry_) + { + auto const& section = config_->section("telemetry"); + std::string endpoint = "http://localhost:4318/v1/metrics"; + set(endpoint, "metrics_endpoint", section); + + // Pass the service_instance_id so the MeterProvider Resource + // carries it, giving Prometheus an service_instance_id label. + std::string instanceId; + set(instanceId, "service_instance_id", section); + if (instanceId.empty() && nodeIdentity_) + instanceId = toBase58(TokenType::NodePublic, nodeIdentity_->first); + + metricsRegistry_->start(endpoint, instanceId); + } + io_latency_sampler_.start(); resolver_->start(); loadManager_->start(); @@ -1586,6 +1666,19 @@ ApplicationImp::run() // The order of these stop calls is delicate. // Re-ordering them risks undefined behavior. loadManager_->stop(); + + // Detach MetricsRegistry observable-gauge callbacks BEFORE stopping + // any service the callbacks read from. The callbacks run on the OTel + // reader thread and touch nodeStore_, overlay_, networkOPs_, + // ledgerMaster, inboundLedgers, etc. A final tick that fires after + // one of those services has shut down would dereference dangling + // state. detachCallbacks() flips an atomic flag every callback + // acquire-loads at its entry, so subsequent ticks become no-ops. + // The final provider teardown still happens in metricsRegistry_->stop() + // farther down. + if (metricsRegistry_) + metricsRegistry_->detachCallbacks(); + shaMapStore_->stop(); jobQueue_->stop(); if (overlay_) @@ -1599,6 +1692,15 @@ ApplicationImp::run() ledgerCleaner_->stop(); nodeStore_->stop(); perfLog_->stop(); + // Stop metrics pipeline before telemetry — gauge callbacks reference + // Application services that may be shutting down. + if (metricsRegistry_) + metricsRegistry_->stop(); + // Telemetry must stop last among trace-producing components. + // serverHandler_, overlay_, and jobQueue_ are already stopped above, + // so no threads should be calling startSpan() at this point. + // See TODO in TelemetryImpl::stop() re: thread-safety of sdkProvider_. + telemetry_->stop(); JLOG(journal_.info()) << "Done."; } diff --git a/src/xrpld/app/main/CollectorManager.cpp b/src/xrpld/app/main/CollectorManager.cpp index 9e1278607f6..84dcddb6698 100644 --- a/src/xrpld/app/main/CollectorManager.cpp +++ b/src/xrpld/app/main/CollectorManager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,12 @@ class CollectorManagerImp : public CollectorManager std::unique_ptr groups_; // NOLINTEND(readability-identifier-naming) - CollectorManagerImp(Section const& params, beast::Journal journal) : journal_(journal) + CollectorManagerImp( + Section const& params, + std::string const& serviceName, + std::string const& networkType, + beast::Journal journal) + : journal_(journal) { std::string const& server = get(params, Keys::kServer); @@ -36,6 +42,33 @@ class CollectorManagerImp : public CollectorManager collector_ = beast::insight::StatsDCollector::make(address, prefix, journal); } + // LCOV_EXCL_START -- OTel collector path is not exercised in unit tests + else if (server == "otel") + { + // Read OTLP metrics endpoint from [insight] section. + // Default to the standard OTLP/HTTP metrics path on localhost. + std::string endpoint = get(params, "endpoint"); + if (endpoint.empty()) + endpoint = "http://localhost:4318/v1/metrics"; + std::string const& prefix(get(params, "prefix")); + + // Read service_instance_id, same key as the [telemetry] + // section uses, so multi-node deployments can distinguish + // metric sources via the service_instance_id Prometheus label. + std::string const instanceId = get(params, "service_instance_id"); + + // service.name from [insight] (falls back to the value the + // caller derived from [telemetry]); network type derived from + // [network_id]. Both mirror the trace exporter so metrics and + // traces carry the same service and network identity. + std::string serviceNameCfg = get(params, "service_name"); + if (serviceNameCfg.empty()) + serviceNameCfg = serviceName; + + collector_ = beast::insight::OTelCollector::New( + endpoint, prefix, instanceId, serviceNameCfg, networkType, journal); + } + // LCOV_EXCL_STOP else { collector_ = beast::insight::NullCollector::make(); @@ -62,9 +95,13 @@ class CollectorManagerImp : public CollectorManager //------------------------------------------------------------------------------ std::unique_ptr -makeCollectorManager(Section const& params, beast::Journal journal) +makeCollectorManager( + Section const& params, + std::string const& serviceName, + std::string const& networkType, + beast::Journal journal) { - return std::make_unique(params, journal); + return std::make_unique(params, serviceName, networkType, journal); } } // namespace xrpl diff --git a/src/xrpld/app/main/CollectorManager.h b/src/xrpld/app/main/CollectorManager.h index fac72b4de1e..2539b7ecc8a 100644 --- a/src/xrpld/app/main/CollectorManager.h +++ b/src/xrpld/app/main/CollectorManager.h @@ -25,7 +25,21 @@ class CollectorManager group(std::string const& name) = 0; }; +/** + * Construct the collector manager. + * + * @param params The [insight] config section. + * @param serviceName service.name resource attribute for OTel metrics + * (empty -> the collector defaults it to "xrpld"). + * @param networkType xrpl.network.type resource attribute for OTel + * metrics (e.g. "mainnet"), derived from [network_id]. + * @param journal Journal for logging. + */ std::unique_ptr -makeCollectorManager(Section const& params, beast::Journal journal); +makeCollectorManager( + Section const& params, + std::string const& serviceName, + std::string const& networkType, + beast::Journal journal); } // namespace xrpl diff --git a/src/xrpld/app/main/GRPCServer.cpp b/src/xrpld/app/main/GRPCServer.cpp index 1146cbdc08c..566362bb0f6 100644 --- a/src/xrpld/app/main/GRPCServer.cpp +++ b/src/xrpld/app/main/GRPCServer.cpp @@ -2,6 +2,7 @@ #include // IWYU pragma: keep #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -49,6 +51,7 @@ #include #include #include +#include #include #include @@ -94,7 +97,8 @@ GRPCServerImpl::CallData::CallData( Forward forward, RPC::Condition requiredCondition, Resource::Charge loadType, - std::vector const& secureGatewayIPs) + std::vector const& secureGatewayIPs, + std::string_view name) : service_(service) , cq_(cq) , finished_(false) @@ -106,6 +110,7 @@ GRPCServerImpl::CallData::CallData( , requiredCondition_(requiredCondition) , loadType_(std::move(loadType)) , secureGatewayIPs_(secureGatewayIPs) + , name_(name) { // Bind a listener. When a request is received, "this" will be returned // from CompletionQueue::Next @@ -125,7 +130,8 @@ GRPCServerImpl::CallData::clone() forward_, requiredCondition_, loadType_, - secureGatewayIPs_); + secureGatewayIPs_, + name_); } template @@ -163,12 +169,19 @@ template void GRPCServerImpl::CallData::process(std::shared_ptr coro) { + using namespace telemetry; + auto span = SpanGuard::span(TraceCategory::Rpc, grpc_span::prefix::grpc, name_); + span.setAttribute(grpc_span::attr::method, name_); + try { auto usage = getUsage(); bool const isUnlimited = clientIsUnlimited(); if (!isUnlimited && usage.disconnect(app_.getJournal("gRPCServer"))) { + span.setAttribute( + grpc_span::attr::grpcStatus, grpc_span::val::error); // LCOV_EXCL_LINE + span.setError(grpc_span::val::resourceExhausted); // LCOV_EXCL_LINE grpc::Status const status{ grpc::StatusCode::RESOURCE_EXHAUSTED, "usage balance exceeds threshold"}; responder_.FinishWithError(status, this); @@ -179,6 +192,11 @@ GRPCServerImpl::CallData::process(std::shared_ptr #include #include +#include #include #include #include @@ -196,6 +197,11 @@ class GRPCServerImpl final std::vector const& secureGatewayIPs_; + /** + * Human-readable name for telemetry spans (e.g. "GetLedger"). + */ + std::string_view name_; + public: ~CallData() override = default; @@ -211,7 +217,8 @@ class GRPCServerImpl final Forward forward, RPC::Condition requiredCondition, Resource::Charge loadType, - std::vector const& secureGatewayIPs); + std::vector const& secureGatewayIPs, + std::string_view name = ""); CallData(CallData const&) = delete; diff --git a/src/xrpld/app/main/GrpcSpanNames.h b/src/xrpld/app/main/GrpcSpanNames.h new file mode 100644 index 00000000000..2eb2f44ca2f --- /dev/null +++ b/src/xrpld/app/main/GrpcSpanNames.h @@ -0,0 +1,66 @@ +#pragma once + +/** + * Compile-time span name constants for the gRPC subsystem. + * + * All span prefixes, operation names, and attribute keys used by gRPC + * tracing call sites are defined here. Built on the StaticStr/join() + * primitives from . + * + * Span hierarchy: + * + * +-------------------------------------------------------+ + * | grpc. (e.g. grpc.GetLedger) | + * | CallData::process(coro) | + * | attrs: method, grpc_role, grpc_status | + * +-------------------------------------------------------+ + * + * Unlike the HTTP/WS RPC path, gRPC has a flat single-span structure + * per request since each CallData handles exactly one RPC method. + * The method name is embedded in the span name (rather than only as + * an attribute) so dashboards can break out per-method latency and + * error rates without needing TraceQL attribute filters. + */ + +#include + +namespace xrpl::telemetry::grpc_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "grpc" — root prefix for gRPC transport spans. The full span name is + * formed at the call site as `grpc.` (see GRPCServer.cpp). + */ +inline constexpr auto grpc = makeStr("grpc"); +} // namespace prefix + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * "method" — gRPC method name (e.g. GetLedger). + */ +inline constexpr auto method = makeStr("method"); +/** + * "grpc_role" — Domain-qualified: collides with rpc_role. + */ +inline constexpr auto grpcRole = makeStr("grpc_role"); +/** + * "grpc_status" — Domain-qualified: avoids OTel reserved span status. + */ +inline constexpr auto grpcStatus = makeStr("grpc_status"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +using telemetry::attr_val::error; +using telemetry::attr_val::success; +inline constexpr auto admin = makeStr("admin"); +inline constexpr auto user = makeStr("user"); +inline constexpr auto resourceExhausted = makeStr("resource_exhausted"); +} // namespace val + +} // namespace xrpl::telemetry::grpc_span diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 4b0091dff6f..94a604c9f83 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -33,6 +33,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -114,6 +118,7 @@ #include #include #include +#include #include #include @@ -169,9 +174,17 @@ class NetworkOPsImp final : public NetworkOPs FailHard const failType; bool applied = false; TER result; - - TransactionStatus(std::shared_ptr t, bool a, bool l, FailHard f) - : transaction(std::move(t)), admin(a), local(l), failType(f) + std::shared_ptr span; ///< Keeps the tx.process + ///< span alive until the + ///< batch processes this. + + TransactionStatus( + std::shared_ptr t, + bool a, + bool l, + FailHard f, + std::shared_ptr s = nullptr) + : transaction(std::move(t)), admin(a), local(l), failType(f), span(std::move(s)) { XRPL_ASSERT( local || failType == FailHard::No, @@ -264,6 +277,21 @@ class NetworkOPsImp final : public NetworkOPs .start = start_, .initialSyncUs = initialSyncUs_}; } + + /** + * Time spent in the current operating mode so far. This is the same + * quantity reported as `server_state_duration_us` in json(): the + * elapsed time since the last state transition. Thread-safe. + * + * @return Duration since entering the current state, in microseconds. + */ + std::chrono::microseconds + currentStateDurationUs() const + { + std::scoped_lock const lock(mutex_); + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_); + } }; /** @@ -348,6 +376,9 @@ class NetworkOPsImp final : public NetworkOPs OperatingMode getOperatingMode() const override; + std::chrono::microseconds + getServerStateDurationUs() const override; + std::string strOperatingMode(OperatingMode const mode, bool const admin) const override; @@ -379,9 +410,15 @@ class NetworkOPsImp final : public NetworkOPs * @param transaction Transaction object. * @param bUnlimited Whether a privileged client connection submitted it. * @param failType fail_hard setting from transaction submission. + * @param span Optional tx.process span to keep alive across the + * batch boundary so its context propagates to peers. */ void - doTransactionSync(std::shared_ptr transaction, bool bUnlimited, FailHard failType); + doTransactionSync( + std::shared_ptr transaction, + bool bUnlimited, + FailHard failType, + std::shared_ptr span = nullptr); /** * For transactions not submitted by a locally connected client, fire and @@ -396,7 +433,8 @@ class NetworkOPsImp final : public NetworkOPs doTransactionAsync( std::shared_ptr transaction, bool bUnlimited, - FailHard failtype); + FailHard failtype, + std::shared_ptr span = nullptr); private: bool @@ -966,6 +1004,12 @@ NetworkOPsImp::getOperatingMode() const return mode_; } +std::chrono::microseconds +NetworkOPsImp::getServerStateDurationUs() const +{ + return accounting_.currentStateDurationUs(); +} + inline std::string NetworkOPsImp::strOperatingMode(bool const admin /* = false */) const { @@ -1364,6 +1408,24 @@ NetworkOPsImp::processTransaction( bool bLocal, FailHard failType) { + using namespace telemetry; + // Detached: this span is stored in TransactionStatus and applied on a + // batch worker thread, so it must not leave its Scope bound to this + // thread's context stack (that leak would adopt later work into this + // transaction's trace). + auto span = std::make_shared(txProcessSpan(transaction->getID()).detached()); + span->setAttribute(tx_span::attr::txHash, to_string(transaction->getID()).c_str()); + span->setAttribute(tx_span::attr::local, bLocal); + if (auto const& stx = transaction->getSTransaction()) + { + if (auto const* fmt = TxFormats::getInstance().findByType(stx->getTxnType())) + span->setAttribute(tx_span::attr::txType, fmt->getName().c_str()); + span->setAttribute( + tx_span::attr::fee, static_cast(stx->getFieldAmount(sfFee).xrp().drops())); + span->setAttribute( + tx_span::attr::sequence, static_cast(stx->getSeqProxy().value())); + } + auto ev = jobQueue_.makeLoadEvent(JtTxnProc, "ProcessTXN"); // preProcessTransaction can change our pointer @@ -1372,11 +1434,13 @@ NetworkOPsImp::processTransaction( if (bLocal) { - doTransactionSync(transaction, bUnlimited, failType); + span->setAttribute(tx_span::attr::path, tx_span::val::sync); + doTransactionSync(transaction, bUnlimited, failType, std::move(span)); } else { - doTransactionAsync(transaction, bUnlimited, failType); + span->setAttribute(tx_span::attr::path, tx_span::val::async); + doTransactionAsync(transaction, bUnlimited, failType, std::move(span)); } } @@ -1384,14 +1448,15 @@ void NetworkOPsImp::doTransactionAsync( std::shared_ptr transaction, bool bUnlimited, - FailHard failType) + FailHard failType, + std::shared_ptr span) { std::scoped_lock const lock(mutex_); if (transaction->getApplying()) return; - transactions_.emplace_back(transaction, bUnlimited, false, failType); + transactions_.emplace_back(transaction, bUnlimited, false, failType, std::move(span)); transaction->setApplying(); if (dispatchState_ == DispatchState::None) @@ -1407,13 +1472,14 @@ void NetworkOPsImp::doTransactionSync( std::shared_ptr transaction, bool bUnlimited, - FailHard failType) + FailHard failType, + std::shared_ptr span) { std::unique_lock lock(mutex_); if (!transaction->getApplying()) { - transactions_.emplace_back(transaction, bUnlimited, true, failType); + transactions_.emplace_back(transaction, bUnlimited, true, failType, std::move(span)); transaction->setApplying(); } @@ -1579,6 +1645,12 @@ NetworkOPsImp::apply(std::unique_lock& batchLock) auto newOL = registry_.get().getOpenLedger().current(); for (TransactionStatus const& e : transactions) { + if (e.span && *e.span) + { + e.span->setAttribute( + telemetry::tx_span::attr::terResult, transToken(e.result).c_str()); + e.span->setAttribute(telemetry::tx_span::attr::applied, e.applied); + } e.transaction->clearSubmitResult(); if (e.applied) @@ -1739,6 +1811,10 @@ NetworkOPsImp::apply(std::unique_lock& batchLock) tx.set_receivetimestamp( registry_.get().getTimeKeeper().now().time_since_epoch().count()); tx.set_deferred(e.result == terQUEUED); + // Inject the tx.process span's trace context so the + // receiving node can link its tx.receive span as a child. + if (e.span && *e.span) + telemetry::injectSpanContext(*e.span, *tx.mutable_trace_context()); // FIXME: This should be when we received it registry_.get().getOverlay().relay(e.transaction->getID(), tx, *toSkip); e.transaction->setBroadcast(); @@ -2566,6 +2642,10 @@ NetworkOPsImp::setMode(OperatingMode om) accounting_.mode(om); + // Record state change for OTel dashboard parity counter. + if (auto* mr = registry_.get().getMetricsRegistry()) + mr->incrementStateChanges(); + JLOG(journal_.info()) << "STATE->" << strOperatingMode(); pubServer(); } @@ -2574,6 +2654,8 @@ bool NetworkOPsImp::recvValidation(std::shared_ptr const& val, std::string const& source) { JLOG(journal_.trace()) << "recvValidation " << val->getLedgerHash() << " from " << source; + if (auto* mr = registry_.get().getMetricsRegistry()) + mr->incrementValidationsChecked(); std::unique_lock lock(validationsMutex_); BypassAccept bypassAccept = BypassAccept::No; diff --git a/src/xrpld/app/misc/detail/TxQ.cpp b/src/xrpld/app/misc/detail/TxQ.cpp index 041d2ade1e5..a62fdbd287e 100644 --- a/src/xrpld/app/misc/detail/TxQ.cpp +++ b/src/xrpld/app/misc/detail/TxQ.cpp @@ -2,8 +2,11 @@ #include #include +#include +#include #include +#include #include #include #include @@ -30,6 +33,7 @@ #include #include #include +#include #include #include @@ -541,6 +545,10 @@ TxQ::tryClearAccountQueueUpThruTx( FeeMetrics::Snapshot const& metricsSnapshot, beast::Journal j) { + using namespace telemetry; + [[maybe_unused]] auto span = SpanGuard::span( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::batchClear); + SeqProxy const tSeqProx{tx.getSeqProxy()}; XRPL_ASSERT( beginTxIter != accountIter->second.transactions.end(), @@ -612,7 +620,8 @@ TxQ::tryClearAccountQueueUpThruTx( if (txResult.applied) { // All of the queued transactions applied, so remove them from the - // queue. + // queue. `dist` queued txs preceded the current one in the batch. + span.setAttribute(txq_span::attr::numCleared, static_cast(dist)); endTxIter = erase(accountIter->second, beginTxIter, endTxIter); // If `tx` is replacing a queued tx, delete that one, too. if (endTxIter != accountIter->second.transactions.end() && endTxIter->first == tSeqProx) @@ -743,6 +752,16 @@ TxQ::apply( ApplyFlags flags, beast::Journal j) { + using namespace telemetry; + auto span = + SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::enqueue); + span.setAttribute(txq_span::attr::txHash, to_string(tx->getTransactionID()).c_str()); + if (auto const* fmt = TxFormats::getInstance().findByType(tx->getTxnType())) + span.setAttribute(txq_span::attr::txType, fmt->getName().c_str()); + // Default outcome; overridden below on the direct-apply and queued paths. + // Every other early return leaves the tx rejected from the queue. + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::rejected); + // See if the transaction is valid, properly formed, // etc. before doing potentially expensive queue // replace and multi-transaction operations. @@ -753,7 +772,10 @@ TxQ::apply( // See if the transaction paid a high enough fee that it can go straight // into the ledger. if (auto directApplied = tryDirectApply(app, view, tx, flags, j)) + { + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::appliedDirect); return *directApplied; + } if ((flags & TapDryRun) != 0u) return {telCAN_NOT_QUEUE, false}; @@ -880,6 +902,10 @@ TxQ::apply( auto const metricsSnapshot = feeMetrics_.getSnapshot(); auto const feeLevelPaid = getFeeLevelPaid(view, *tx); auto const requiredFeeLevel = getRequiredFeeLevel(view, flags, metricsSnapshot, lock); + span.setAttribute( + txq_span::attr::feeLevelPaid, static_cast(feeLevelPaid.value())); + span.setAttribute( + txq_span::attr::requiredFeeLevel, static_cast(requiredFeeLevel.value())); // Is there a blocker already in the account's queue? If so, don't // allow additional transactions in the queue. @@ -1213,6 +1239,7 @@ TxQ::apply( /* Can't erase (*replacedTxIter) here because success implies that it has already been deleted. */ + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::applied); return result; } } @@ -1250,6 +1277,8 @@ TxQ::apply( JLOG(j_.info()) << "Queue is full, and transaction " << transactionID << " would kick a transaction from the same account (" << account << ") out of the queue."; + if (auto* const metrics = app.getMetricsRegistry(); metrics != nullptr) + metrics->incrementTxqDropped("queue_full"); return {telCAN_NOT_QUEUE_FULL, false}; } auto const& endAccount = byAccount_.at(lastRIter->account); @@ -1293,6 +1322,8 @@ TxQ::apply( { JLOG(j_.info()) << "Queue is full, and transaction " << transactionID << " fee is lower than end item's account average fee"; + if (auto* const metrics = app.getMetricsRegistry(); metrics != nullptr) + metrics->incrementTxqDropped("queue_full"); return {telCAN_NOT_QUEUE_FULL, false}; } } @@ -1328,6 +1359,7 @@ TxQ::apply( << " to queue." << " Flags: " << flags; + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::queued); return {terQUEUED, false}; } @@ -1346,6 +1378,11 @@ TxQ::apply( void TxQ::processClosedLedger(Application& app, ReadView const& view, bool timeLeap) { + using namespace telemetry; + auto span = + SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::cleanup); + span.setAttribute(txq_span::attr::ledgerSeq, static_cast(view.header().seq)); + std::scoped_lock const lock(mutex_); feeMetrics_.update(app, view, timeLeap, setup_); @@ -1357,18 +1394,26 @@ TxQ::processClosedLedger(Application& app, ReadView const& view, bool timeLeap) maxSize_ = std::max(snapshot.txnsExpected * setup_.ledgersInQueue, setup_.queueSizeMin); // Remove any queued candidates whose LastLedgerSequence has gone by. + auto* const metrics = app.getMetricsRegistry(); + std::int64_t expiredCount = 0; for (auto candidateIter = byFee_.begin(); candidateIter != byFee_.end();) { if (candidateIter->lastValid && *candidateIter->lastValid <= ledgerSeq) { byAccount_.at(candidateIter->account).dropPenalty = true; candidateIter = erase(candidateIter); + // Count each expired transaction: submitters who under-bid the + // escalating fee and were never included before expiry. + if (metrics != nullptr) + metrics->incrementTxqExpired(); + ++expiredCount; } else { ++candidateIter; } } + span.setAttribute(txq_span::attr::expiredCount, expiredCount); // Remove any TxQAccounts that don't have candidates // under them @@ -1417,6 +1462,8 @@ TxQ::processClosedLedger(Application& app, ReadView const& view, bool timeLeap) bool TxQ::accept(Application& app, OpenView& view) { + using namespace telemetry; + /* Move transactions from the queue from largest fee level to smallest. As we add more transactions, the required fee level will increase. Stop when the transaction fee level gets lower than the required fee @@ -1427,6 +1474,12 @@ TxQ::accept(Application& app, OpenView& view) std::scoped_lock const lock(mutex_); + // Create the span and read byFee_.size() only after taking the lock, since + // byFee_ is guarded by mutex_. + auto span = + SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::accept); + span.setAttribute(txq_span::attr::queueSize, static_cast(byFee_.size())); + auto const metricsSnapshot = feeMetrics_.getSnapshot(); for (auto candidateIter = byFee_.begin(); candidateIter != byFee_.end();) @@ -1454,10 +1507,19 @@ TxQ::accept(Application& app, OpenView& view) JLOG(j_.trace()) << "Applying queued transaction " << candidateIter->txID << " to open ledger."; + auto txSpan = SpanGuard::span( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::acceptTx); + txSpan.setAttribute(txq_span::attr::txHash, to_string(candidateIter->txID).c_str()); + txSpan.setAttribute( + txq_span::attr::retriesRemaining, + static_cast(candidateIter->retriesRemaining)); + auto const [txnResult, didApply, _metadata] = candidateIter->apply(app, view, j_); + txSpan.setAttribute(txq_span::attr::terCode, transToken(txnResult).c_str()); if (didApply) { + txSpan.setAttribute(txq_span::attr::txqStatus, txq_span::val::applied); // Remove the candidate from the queue JLOG(j_.debug()) << "Queued transaction " << candidateIter->txID << " applied successfully with " << transToken(txnResult) @@ -1478,12 +1540,14 @@ TxQ::accept(Application& app, OpenView& view) { account.dropPenalty = true; } + txSpan.setAttribute(txq_span::attr::txqStatus, txq_span::val::failed); JLOG(j_.debug()) << "Queued transaction " << candidateIter->txID << " failed with " << transToken(txnResult) << ". Remove from queue."; candidateIter = eraseAndAdvance(candidateIter); } else { + txSpan.setAttribute(txq_span::attr::txqStatus, txq_span::val::retried); JLOG(j_.debug()) << "Queued transaction " << candidateIter->txID << " failed with " << transToken(txnResult) << ". Leave in queue." << " Applied: " << didApply << ". Flags: " << candidateIter->flags; @@ -1579,6 +1643,7 @@ TxQ::accept(Application& app, OpenView& view) } } XRPL_ASSERT(byFee_.size() == startingSize, "xrpl::TxQ::accept : byFee size match"); + span.setAttribute(txq_span::attr::ledgerChanged, ledgerChanged); return ledgerChanged; } @@ -1662,6 +1727,10 @@ TxQ::tryDirectApply( ApplyFlags flags, beast::Journal j) { + using namespace telemetry; + [[maybe_unused]] auto span = SpanGuard::span( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::applyDirect); + auto const account = (*tx)[sfAccount]; auto const sleAccount = view.read(keylet::account(account)); diff --git a/src/xrpld/app/misc/detail/TxQSpanNames.h b/src/xrpld/app/misc/detail/TxQSpanNames.h new file mode 100644 index 00000000000..e062e361d14 --- /dev/null +++ b/src/xrpld/app/misc/detail/TxQSpanNames.h @@ -0,0 +1,138 @@ +#pragma once + +/** + * Compile-time span name constants for Transaction Queue tracing. + * + * Covers the TxQ lifecycle: enqueue decisions, direct apply, batch + * clear, ledger-close accept loop, per-tx apply, and cleanup. + * + * Span hierarchy: + * + * Transaction submission: + * + * +-------------------------------------------------------+ + * | tx.process (existing, from TxSpanNames.h) | + * | | + * | +--------------------------------------------------+ | + * | | txq.enqueue | | + * | | TxQ::apply() | | + * | | attrs: tx_hash, tx_type, txq_status, | | + * | | fee_level_paid, required_fee_level | | + * | | | | + * | | +-------------------+ +----------------------+ | | + * | | | txq.apply_direct | | txq.batch_clear | | | + * | | | tryDirectApply() | | tryClearAccount...() | | | + * | | +-------------------+ | attrs: num_cleared | | | + * | | +----------------------+ | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * Ledger close (consensus thread): + * + * +-------------------------------------------------------+ + * | txq.accept | + * | TxQ::accept() | + * | attrs: queue_size, ledger_changed | + * | | + * | +--------------------------------------------------+ | + * | | txq.accept_tx (per queued transaction) | | + * | | attrs: tx_hash, ter_code, retries_remaining | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * Post-close cleanup: + * + * +-------------------------------------------------------+ + * | txq.cleanup | + * | TxQ::processClosedLedger() | + * | attrs: ledger_seq, expired_count | + * +-------------------------------------------------------+ + */ + +#include + +namespace xrpl::telemetry::txq_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "txq" — root prefix for transaction queue spans. + */ +inline constexpr auto txq = makeStr("txq"); +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto enqueue = makeStr("enqueue"); +inline constexpr auto applyDirect = makeStr("apply_direct"); +inline constexpr auto batchClear = makeStr("batch_clear"); +inline constexpr auto accept = makeStr("accept"); +inline constexpr auto acceptTx = makeStr("accept_tx"); +inline constexpr auto cleanup = makeStr("cleanup"); +} // namespace op + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). + */ +using ::xrpl::telemetry::attr::ledgerSeq; +using ::xrpl::telemetry::attr::txHash; + +/** + * "txq_status" — domain-qualified (collides with tx_status, rpc_status). + */ +inline constexpr auto txqStatus = makeStr("txq_status"); +/** + * "fee_level_paid" — fee level paid by queued tx. + */ +inline constexpr auto feeLevelPaid = makeStr("fee_level_paid"); +/** + * "required_fee_level" — minimum fee level for inclusion. + */ +inline constexpr auto requiredFeeLevel = makeStr("required_fee_level"); +/** + * "queue_size" — current TxQ depth. + */ +inline constexpr auto queueSize = makeStr("queue_size"); +/** + * "ledger_changed" — whether ledger changed since last attempt. + */ +inline constexpr auto ledgerChanged = makeStr("ledger_changed"); +/** + * "expired_count" — number of expired entries cleared. + */ +inline constexpr auto expiredCount = makeStr("expired_count"); +/** + * "ter_code" — transaction engine result code. + */ +inline constexpr auto terCode = makeStr("ter_code"); +/** + * "retries_remaining" — retries left before discard. + */ +inline constexpr auto retriesRemaining = makeStr("retries_remaining"); +/** + * "num_cleared" — entries cleared in batch. + */ +inline constexpr auto numCleared = makeStr("num_cleared"); +/** + * "tx_type" — transaction type name (e.g., "Payment", "OfferCreate"). + */ +inline constexpr auto txType = makeStr("tx_type"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +inline constexpr auto queued = makeStr("queued"); +inline constexpr auto appliedDirect = makeStr("applied_direct"); +inline constexpr auto rejected = makeStr("rejected"); +inline constexpr auto applied = makeStr("applied"); +inline constexpr auto failed = makeStr("failed"); +inline constexpr auto retried = makeStr("retried"); +} // namespace val + +} // namespace xrpl::telemetry::txq_span diff --git a/src/xrpld/consensus/Consensus.h b/src/xrpld/consensus/Consensus.h index 440191939b8..917bf844631 100644 --- a/src/xrpld/consensus/Consensus.h +++ b/src/xrpld/consensus/Consensus.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -24,6 +27,7 @@ #include #include #include +#include #include namespace xrpl { @@ -447,13 +451,29 @@ class Consensus getJson(bool full) const; private: + /** + * Why startRoundInternal is being entered. + * + * Distinguishes the normal Initial entry (from public startRound) + * from a Recovered re-entry (from handleWrongLedger after the + * correct prior ledger was acquired mid-round). The Recovered path + * resets phase to Open within the SAME round, which is a state + * transition that would otherwise be invisible in traces — the + * recovery flag drives a `phase.recovery` event on the round span. + */ + enum class StartRoundReason : std::uint8_t { + Initial, + Recovered, + }; + void startRoundInternal( NetClock::time_point const& now, Ledger_t::ID const& prevLedgerID, Ledger_t const& prevLedger, ConsensusMode mode, - std::unique_ptr const& clog); + std::unique_ptr const& clog, + StartRoundReason reason = StartRoundReason::Initial); // Change our view of the previous ledger void @@ -627,6 +647,50 @@ class Consensus // nodes that have bowed out of this consensus process hash_set deadNodes_; + /** + * Span for the establish phase of consensus. + * Created when the ledger closes and we enter phaseEstablish; + * cleared (ended) when consensus is reached. Stored detached (its Scope + * is stripped right after its context is captured) because it is emplaced + * and reset on different job workers. + */ + std::optional establishSpan_; + + /** + * Captured context of establishSpan_, snapshotted while it was still + * scoped. Same-thread children (update_positions, check) build from this + * explicit context instead of the (now detached) ambient establish span. + */ + xrpl::telemetry::SpanContext establishSpanContext_; + + /** + * Span for the open phase of consensus. + * Created in startRoundInternal(); cleared (ended) in closeLedger(). + * Stored detached: emplaced and reset on different job workers; + * detached() prevents wrong-thread Scope pop. + */ + std::optional openSpan_; + + /** + * Create the establish-phase span if not yet active. + * Called on each phaseEstablish() invocation; no-op while span is live. + */ + void + startEstablishTracing(); + + /** + * Overwrite convergence metrics on the establish span each iteration. + * Final span attributes always reflect the last state before consensus. + */ + void + updateEstablishTracing(); + + /** + * End the establish span when transitioning to the accepted phase. + */ + void + endEstablishTracing(); + // Journal for debugging beast::Journal const j_; }; @@ -690,13 +754,53 @@ Consensus::startRoundInternal( Ledger_t::ID const& prevLedgerID, Ledger_t const& prevLedger, ConsensusMode mode, - std::unique_ptr const& clog) + std::unique_ptr const& clog, + StartRoundReason const reason) { + // Recovery path: handleWrongLedger acquired the correct prior ledger + // and re-entered startRoundInternal mid-round. The roundSpan_ owned by + // the adaptor is still the SAME span as before — startRoundTracing is + // not called on the recovery path, so we record the recovery as an + // event on the surviving round span. Pass empty phaseLabel to leave + // consensus_phase unchanged (the actual phase reset to open is marked + // separately by the new openSpan_ being emplaced below). + if (reason == StartRoundReason::Recovered) + { + adaptor_.onPhaseEvent(telemetry::consensus::span::event::phaseRecovery, ""); + } + phase_ = ConsensusPhase::Open; JLOG(j_.debug()) << "transitioned to ConsensusPhase::Open "; CLOG(clog) << "startRoundInternal transitioned to ConsensusPhase::Open, " "previous ledgerID: " << prevLedgerID << ", seq: " << prevLedger.seq() << ". "; + // Reset establishSpan_ so a wrongLedger recovery mid-establish doesn't + // leak the prior round's span into the new one (startEstablishTracing + // early-returns when establishSpan_ is populated). + establishSpan_.reset(); + establishSpanContext_ = telemetry::SpanContext{}; + // Child of the round span via its captured context: the round span is + // detached (no longer the thread's ambient parent), so parent phase.open + // explicitly under roundSpanContext_ rather than the ambient stack. An + // invalid round context (round span not yet created) yields a null guard. + // Detached in turn: emplaced here on one job worker and reset() on + // another, so strip the thread-local Scope to avoid a wrong-thread pop. + // No same-thread child span nests under openSpan_. + openSpan_.emplace( + telemetry::SpanGuard::childSpan( + telemetry::consensus::span::phaseOpen, adaptor_.roundSpanContext()) + .detached()); + // On the Recovered path, fire phase.open here because startRoundTracing + // (which fires it for the Initial path) is not called on re-entry. On + // the Initial path this is a no-op because the round span hasn't been + // created yet — the phase.open event is fired later by startRoundTracing + // after the new round span is in place. + if (reason == StartRoundReason::Recovered) + { + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseOpen, + telemetry::consensus::span::val::phaseOpen); + } mode_.set(mode, adaptor_); now_ = now; prevLedgerID_ = prevLedgerID; @@ -947,6 +1051,9 @@ Consensus::simulate( result_->proposers = prevProposers_ = currPeerPositions_.size(); prevRoundTime_ = result_->roundTime.read(); phase_ = ConsensusPhase::Accepted; + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseAccepted, + telemetry::consensus::span::val::phaseAccepted); adaptor_.onForceAccept( *result_, previousLedger_, closeResolution_, rawCloseTimes_, mode_.get(), getJson(true)); // NOLINTEND(bugprone-unchecked-optional-access) @@ -1095,7 +1202,13 @@ Consensus::handleWrongLedger( { JLOG(j_.info()) << "Have the consensus ledger " << prevLedgerID_; CLOG(clog) << "Have the consensus ledger " << prevLedgerID_ << ". "; - startRoundInternal(now_, lgrId, *newLedger, ConsensusMode::SwitchedLedger, clog); + startRoundInternal( + now_, + lgrId, + *newLedger, + ConsensusMode::SwitchedLedger, + clog, + StartRoundReason::Recovered); } else { @@ -1356,6 +1469,8 @@ Consensus::phaseEstablish(std::unique_ptr const& clo XRPL_ASSERT(result_, "xrpl::Consensus::phaseEstablish : result is set"); // NOLINTBEGIN(bugprone-unchecked-optional-access) assert above + startEstablishTracing(); + ++peerUnchangedCounter_; ++establishCounter_; @@ -1383,6 +1498,8 @@ Consensus::phaseEstablish(std::unique_ptr const& clo updateOurPositions(clog); + updateEstablishTracing(); + // Nothing to do if too many laggards or we don't have consensus. if (shouldPause(clog) || !haveConsensus(clog)) return; @@ -1400,7 +1517,26 @@ Consensus::phaseEstablish(std::unique_ptr const& clo adaptor_.updateOperatingMode(currPeerPositions_.size()); prevProposers_ = currPeerPositions_.size(); prevRoundTime_ = result_->roundTime.read(); + endEstablishTracing(); + { + namespace cs = telemetry::consensus::span; + if (result_->state == ConsensusState::Yes) + { + adaptor_.onOutcomeEvent(cs::event::outcomeYes); + } + else if (result_->state == ConsensusState::MovedOn) + { + adaptor_.onOutcomeEvent(cs::event::outcomeMovedOn); + } + else if (result_->state == ConsensusState::Expired) + { + adaptor_.onOutcomeEvent(cs::event::outcomeExpired); + } + } phase_ = ConsensusPhase::Accepted; + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseAccepted, + telemetry::consensus::span::val::phaseAccepted); JLOG(j_.debug()) << "transitioned to ConsensusPhase::Accepted"; adaptor_.onAccept( *result_, @@ -1420,7 +1556,23 @@ Consensus::closeLedger(std::unique_ptr const& clog) // We should not be closing if we already have a position XRPL_ASSERT(!result_, "xrpl::Consensus::closeLedger : result is not set"); + // Annotate the open-phase span with end-of-phase metadata before + // ending it, so a Tempo/Jaeger query for consensus.phase.open shows + // how long the phase ran and how many peer positions arrived during it. + openTime_.tick(clock_.now()); + if (openSpan_ && *openSpan_) + { + namespace cs = telemetry::consensus::span; + openSpan_->setAttribute( + cs::attr::openDurationMs, static_cast(openTime_.read().count())); + openSpan_->setAttribute( + cs::attr::peerPositionsAtClose, static_cast(currPeerPositions_.size())); + } + openSpan_.reset(); phase_ = ConsensusPhase::Establish; + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseEstablish, + telemetry::consensus::span::val::phaseEstablish); JLOG(j_.debug()) << "transitioned to ConsensusPhase::Establish"; rawCloseTimes_.self = now_; peerUnchangedCounter_ = 0; @@ -1477,6 +1629,17 @@ Consensus::updateOurPositions(std::unique_ptr const& // We must have a position if we are updating it XRPL_ASSERT(result_, "xrpl::Consensus::updateOurPositions : result is set"); // NOLINTBEGIN(bugprone-unchecked-optional-access) assert above + using namespace telemetry; + // Child of the establish span via its captured context (establishSpan_ is + // detached, so it is no longer the thread's ambient parent). Null context + // (establish not started) yields a null guard, same as before. + auto span = SpanGuard::childSpan(consensus::span::updatePositions, establishSpanContext_); + span.setAttribute( + consensus::span::attr::convergePercent, static_cast(convergePercent_)); + span.setAttribute( + consensus::span::attr::proposers, static_cast(currPeerPositions_.size())); + span.setAttribute( + consensus::span::attr::disputesCount, static_cast(result_->disputes.size())); ConsensusParms const& parms = adaptor_.parms(); // Compute a cutoff time @@ -1536,6 +1699,17 @@ Consensus::updateOurPositions(std::unique_ptr const& // now a no mutableSet->erase(txId); } + + auto const yaysStr = std::to_string(dispute.getYays()); + auto const naysStr = std::to_string(dispute.getNays()); + span.addEvent( + consensus::span::event::disputeResolve, + {{consensus::span::attr::txId, to_string(txId)}, + {consensus::span::attr::disputeOurVote, + dispute.getOurVote() ? std::string_view{consensus::span::val::yes} + : std::string_view{consensus::span::val::no}}, + {consensus::span::attr::disputeYays, yaysStr}, + {consensus::span::attr::disputeNays, naysStr}}); } } @@ -1560,6 +1734,8 @@ Consensus::updateOurPositions(std::unique_ptr const& if (newState) closeTimeAvalancheState_ = *newState; CLOG(clog) << "neededWeight " << neededWeight << ". "; + span.setAttribute( + consensus::span::attr::avalancheThreshold, static_cast(neededWeight)); int participants = currPeerPositions_.size(); if (mode_.get() == ConsensusMode::Proposing) @@ -1608,6 +1784,10 @@ Consensus::updateOurPositions(std::unique_ptr const& } } + span.setAttribute(consensus::span::attr::haveCloseTimeConsensus, haveCloseTimeConsensus_); + span.setAttribute( + consensus::span::attr::closeTimeThreshold, static_cast(parms.avCtConsensusPct)); + if (!ourNewSet && ((consensusCloseTime != asCloseTime(result_->position.closeTime())) || result_->position.isStale(ourCutoff))) @@ -1659,6 +1839,10 @@ Consensus::haveConsensus(std::unique_ptr const& clog // Must have a stance if we are checking for consensus XRPL_ASSERT(result_, "xrpl::Consensus::haveConsensus : has result"); // NOLINTBEGIN(bugprone-unchecked-optional-access) assert above + using namespace telemetry; + // Child of the establish span via its captured context (establishSpan_ is + // detached, so it is no longer the thread's ambient parent). + auto span = SpanGuard::childSpan(consensus::span::check, establishSpanContext_); // CHECKME: should possibly count unacquired TX sets as disagreeing int agree = 0, disagree = 0; @@ -1717,6 +1901,38 @@ Consensus::haveConsensus(std::unique_ptr const& clog j_, clog); + // Set span attributes before the early-return branches below so the + // consensus.check span carries diagnostic data even when consensus is + // not reached (the No / Expired paths return early). + span.setAttribute(consensus::span::attr::agreeCount, static_cast(agree)); + span.setAttribute(consensus::span::attr::disagreeCount, static_cast(disagree)); + span.setAttribute( + consensus::span::attr::convergePercent, static_cast(convergePercent_)); + span.setAttribute(consensus::span::attr::haveCloseTimeConsensus, haveCloseTimeConsensus_); + span.setAttribute( + consensus::span::attr::thresholdPercent, + static_cast(adaptor_.parms().avCtConsensusPct)); + span.setAttribute( + consensus::span::attr::proposersFinished, static_cast(currentFinished)); + span.setAttribute(consensus::span::attr::consensusStalled, stalled); + span.setAttribute( + consensus::span::attr::establishCount, static_cast(establishCounter_)); + + std::string_view stateStr = consensus::span::val::no; + if (result_->state == ConsensusState::Yes) + { + stateStr = consensus::span::val::yes; + } + else if (result_->state == ConsensusState::MovedOn) + { + stateStr = consensus::span::val::movedOn; + } + else if (result_->state == ConsensusState::Expired) + { + stateStr = consensus::span::val::expired; + } + span.setAttribute(consensus::span::attr::consensusResult, stateStr); + if (result_->state == ConsensusState::No) { CLOG(clog) << "No consensus. "; @@ -1879,4 +2095,55 @@ Consensus::asCloseTime(NetClock::time_point raw) const return roundCloseTime(raw, closeResolution_); } +template +void +Consensus::startEstablishTracing() +{ + if (establishSpan_) + return; + // Child of the round span via its captured context: the round span is + // detached (no longer the thread's ambient parent), so parent establish + // explicitly under roundSpanContext_. An invalid round context (round span + // not yet created) yields a null guard. + establishSpan_.emplace( + telemetry::SpanGuard::childSpan( + telemetry::consensus::span::establish, adaptor_.roundSpanContext())); + // Capture the establish context while the guard is still scoped, then + // detach. Same-thread children (update_positions, check) link to this + // captured context explicitly instead of relying on establishSpan_ being + // the ambient parent -- the guard is reset() on a different worker than it + // is emplaced on, so it must not keep a thread-local Scope. + if (*establishSpan_) + { + establishSpanContext_ = establishSpan_->captureContext(); + establishSpan_.emplace(std::move(*establishSpan_).detached()); + } +} + +template +void +Consensus::updateEstablishTracing() +{ + if (!establishSpan_) + return; + namespace cs = telemetry::consensus::span; + establishSpan_->setAttribute(cs::attr::convergePercent, static_cast(convergePercent_)); + establishSpan_->setAttribute(cs::attr::establishCount, static_cast(establishCounter_)); + establishSpan_->setAttribute( + cs::attr::proposers, static_cast(currPeerPositions_.size())); + if (result_) + { + establishSpan_->setAttribute( + cs::attr::disputesCount, static_cast(result_->disputes.size())); + } +} + +template +void +Consensus::endEstablishTracing() +{ + establishSpan_.reset(); + establishSpanContext_ = telemetry::SpanContext{}; +} + } // namespace xrpl diff --git a/src/xrpld/consensus/ConsensusSpanNames.h b/src/xrpld/consensus/ConsensusSpanNames.h new file mode 100644 index 00000000000..6c79cd3132a --- /dev/null +++ b/src/xrpld/consensus/ConsensusSpanNames.h @@ -0,0 +1,296 @@ +#pragma once + +/** + * Compile-time span name constants for consensus tracing. + * + * Used by RCLConsensus (app), Consensus.h (template), and PeerImp + * (overlay) for consensus lifecycle spans. + * Built on StaticStr/join() from SpanNames.h. + * + * ## Span Hierarchy + * + * Root span created in Adaptor::startRoundTracing(). In "deterministic" + * strategy the trace-id is derived from the previous ledger hash so all + * nodes tracing the same round share a trace. + * + * consensus.round [main thread, root] + * | Created: Adaptor::startRoundTracing() + * | Attrs: consensus_ledger_id, ledger_seq, consensus_mode, + * | trace_strategy, consensus_round_id + * | + * +-- consensus.phase.open [main thread, child] + * | Created: Consensus::startRoundInternal() + * | Ended: Consensus::closeLedger() + * | + * +-- consensus.proposal.send [main thread] + * | Created: Adaptor::propose() + * | Attrs: consensus_round (proposeSeq) + * | + * +-- consensus.ledger_close [main thread] + * | Created: Adaptor::onClose() + * | Attrs: ledger_seq, consensus_mode + * | + * +-- consensus.establish [main thread, child] + * | Created: Consensus::startEstablishTracing() + * | Ended: Consensus::phaseEstablish() on accept + * | Attrs: converge_percent, establish_count, proposers + * | + * +-- consensus.update_positions [main thread] + * | Created: Consensus::updateOurPositions() + * | Attrs: converge_percent, proposers, disputes_count + * | Events: per-dispute vote details (tx_id, our_vote, yays, nays) + * | + * +-- consensus.check [main thread] + * | Created: Consensus::haveConsensus() + * | Attrs: agree/disagree counts, threshold_percent, result + * | + * +-- consensus.accept [main thread, child of round] + * | Created: Adaptor::makeAcceptSpan(), shared_ptr kept alive + * | until doAccept() completes on jtACCEPT thread + * | Attrs: proposers, round_time_ms, quorum + * | | + * | +-- consensus.accept.apply [jtACCEPT thread, child of accept] + * | Created: Adaptor::doAccept() + * | Attrs: ledger_seq, close_time, close_time_correct, + * | close_resolution_ms, consensus_state, proposing, round_time_ms, + * | parent_close_time, close_time_self, close_time_vote_bins, + * | resolution_direction, tx_count + * | Events: tx.included (per tx, attrs: tx_id) + * | + * +~~~ consensus.validation.send [jtACCEPT thread, linked] + * | Created: Adaptor::createValidationSpan() (follows-from link) + * | Attrs: ledger_seq, proposing + * | + * +-- consensus.mode_change [main thread] + * Created: Adaptor::onModeChange() + * Attrs: mode_old, mode_new + * + * Standalone spans (no parent, created per-message in overlay): + * + * consensus.proposal.receive [PeerImp I/O thread] + * Created: PeerImp::onMessage(TMProposeSet) + * + * consensus.validation.receive [PeerImp I/O thread] + * Created: PeerImp::onMessage(TMValidation) + * + * Legend: + * +-- child-of relationship (same trace) + * +~~~ follows-from link (separate sub-tree, causal link) + */ + +#include + +namespace xrpl::telemetry::consensus::span { + +// ===== Span name segments ==================================================== + +namespace part { +inline constexpr auto proposal = makeStr("proposal"); +inline constexpr auto validation = makeStr("validation"); +inline constexpr auto accept = makeStr("accept"); +inline constexpr auto phase = makeStr("phase"); +} // namespace part + +namespace op { +inline constexpr auto round = makeStr("round"); +inline constexpr auto proposalSend = join(part::proposal, makeStr("send")); +inline constexpr auto ledgerClose = makeStr("ledger_close"); +inline constexpr auto establish = makeStr("establish"); +inline constexpr auto updatePositions = makeStr("update_positions"); +inline constexpr auto check = makeStr("check"); +inline constexpr auto accept = makeStr("accept"); +inline constexpr auto acceptApply = join(part::accept, makeStr("apply")); +inline constexpr auto validationSend = join(part::validation, makeStr("send")); +inline constexpr auto modeChange = makeStr("mode_change"); +inline constexpr auto proposalReceive = join(part::proposal, makeStr("receive")); +inline constexpr auto validationReceive = join(part::validation, makeStr("receive")); +inline constexpr auto phaseOpen = join(part::phase, makeStr("open")); +} // namespace op + +// ===== Full span names (prefix.op) =========================================== + +inline constexpr auto round = join(seg::consensus, op::round); +inline constexpr auto proposalSend = join(seg::consensus, op::proposalSend); +inline constexpr auto ledgerClose = join(seg::consensus, op::ledgerClose); +inline constexpr auto establish = join(seg::consensus, op::establish); +inline constexpr auto updatePositions = join(seg::consensus, op::updatePositions); +inline constexpr auto check = join(seg::consensus, op::check); +inline constexpr auto accept = join(seg::consensus, op::accept); +inline constexpr auto acceptApply = join(seg::consensus, op::acceptApply); +inline constexpr auto validationSend = join(seg::consensus, op::validationSend); +inline constexpr auto modeChange = join(seg::consensus, op::modeChange); +inline constexpr auto proposalReceive = join(seg::consensus, op::proposalReceive); +inline constexpr auto validationReceive = join(seg::consensus, op::validationReceive); +inline constexpr auto phaseOpen = join(seg::consensus, op::phaseOpen); + +// ===== Attribute keys ======================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). `ledgerHash` and + * `fullValidation` are shared with the peer.validation.receive span — same + * concept, same key, distinguished by span name (not an emitter prefix). + */ +using ::xrpl::telemetry::attr::closeResolutionMs; +using ::xrpl::telemetry::attr::closeTime; +using ::xrpl::telemetry::attr::closeTimeCorrect; +using ::xrpl::telemetry::attr::fullValidation; +using ::xrpl::telemetry::attr::ledgerHash; +using ::xrpl::telemetry::attr::ledgerSeq; + +/** + * Domain-qualified attrs (rule 5 — bare name ambiguous across domains). + * Use `_` underscore form for TraceQL ergonomics. + */ +inline constexpr auto ledgerId = makeStr("consensus_ledger_id"); +inline constexpr auto mode = makeStr("consensus_mode"); +inline constexpr auto round = makeStr("consensus_round"); +inline constexpr auto roundId = makeStr("consensus_round_id"); +/** + * Current phase name attached to consensus.round; updated on each + * phase transition event (open/establish/accepted). + */ +inline constexpr auto consensusPhase = makeStr("consensus_phase"); +/** + * Boolean flag set on consensus.check when checkConsensus reports stalled. + */ +inline constexpr auto consensusStalled = makeStr("consensus_stalled"); + +/** + * Domain-owned bare attrs. + */ +inline constexpr auto proposers = makeStr("proposers"); +inline constexpr auto roundTimeMs = makeStr("round_time_ms"); +inline constexpr auto proposing = makeStr("proposing"); +/** + * Round continuity / context attrs (set on consensus.round at round start). + */ +inline constexpr auto previousProposers = makeStr("previous_proposers"); +inline constexpr auto previousRoundTimeMs = makeStr("previous_round_time_ms"); +inline constexpr auto previousLedgerSeq = makeStr("previous_ledger_seq"); +inline constexpr auto closeTimeResolutionMs = makeStr("close_time_resolution_ms"); +/** + * Open-phase end metadata (set on consensus.phase.open before reset). + */ +inline constexpr auto openDurationMs = makeStr("open_duration_ms"); +inline constexpr auto peerPositionsAtClose = makeStr("peer_positions_at_close"); +/** + * Ledger-close inputs. + */ +inline constexpr auto txCountOpen = makeStr("tx_count_open"); +/** + * Establish/check additional state. + */ +inline constexpr auto proposersFinished = makeStr("proposers_finished"); +/** + * Accept/apply enrichment. + */ +inline constexpr auto disputesResolvedCount = makeStr("disputes_resolved_count"); +/** + * Validation send/receive enrichment. (`full_validation` is shared — see the + * `using` re-export above.) + */ +inline constexpr auto validationSignTime = makeStr("validation_sign_time"); +/** + * Receive-side hash prefixes for cross-peer correlation. + */ +inline constexpr auto prevLedgerPrefix = makeStr("prev_ledger_prefix"); +inline constexpr auto positionHashPrefix = makeStr("position_hash_prefix"); +/** + * "consensus_state" — domain-qualified (collides with other domains' state). + */ +inline constexpr auto consensusState = makeStr("consensus_state"); +inline constexpr auto parentCloseTime = makeStr("parent_close_time"); +inline constexpr auto closeTimeSelf = makeStr("close_time_self"); +inline constexpr auto closeTimeVoteBins = makeStr("close_time_vote_bins"); +inline constexpr auto resolutionDirection = makeStr("resolution_direction"); +inline constexpr auto convergePercent = makeStr("converge_percent"); +inline constexpr auto establishCount = makeStr("establish_count"); +inline constexpr auto avalancheThreshold = makeStr("avalanche_threshold"); +inline constexpr auto closeTimeThreshold = makeStr("close_time_threshold"); +inline constexpr auto haveCloseTimeConsensus = makeStr("have_close_time_consensus"); +inline constexpr auto agreeCount = makeStr("agree_count"); +inline constexpr auto disagreeCount = makeStr("disagree_count"); +inline constexpr auto thresholdPercent = makeStr("threshold_percent"); +/** + * "consensus_result" — domain-qualified (collides with generic result). + */ +inline constexpr auto consensusResult = makeStr("consensus_result"); +inline constexpr auto quorum = makeStr("quorum"); +inline constexpr auto traceStrategy = makeStr("trace_strategy"); +inline constexpr auto modeOld = makeStr("mode_old"); +inline constexpr auto modeNew = makeStr("mode_new"); + +/** + * "is_bow_out" — whether this proposal is a bow-out (resigning from round). + */ +inline constexpr auto isBowOut = makeStr("is_bow_out"); + +/** + * Transaction/dispute attrs used in consensus accept spans. + */ +inline constexpr auto txId = makeStr("tx_id"); +inline constexpr auto disputeOurVote = makeStr("dispute_our_vote"); +inline constexpr auto disputeYays = makeStr("dispute_yays"); +inline constexpr auto disputeNays = makeStr("dispute_nays"); +inline constexpr auto txCount = makeStr("tx_count"); +inline constexpr auto disputesCount = makeStr("disputes_count"); +/** + * Trust flag (is the message origin a trusted UNL validator). Qualified by + * message type, shared with the peer.{proposal,validation}.receive spans: + * consensus.proposal.receive uses `proposal_trusted`, consensus.validation. + * receive uses `validation_trusted`. Same concept on both emitters → same key. + */ +inline constexpr auto proposalTrusted = makeStr("proposal_trusted"); +inline constexpr auto validationTrusted = makeStr("validation_trusted"); +} // namespace attr + +// ===== Event names =========================================================== + +namespace event { +/** + * "dispute.resolve" + */ +inline constexpr auto disputeResolve = join(makeStr("dispute"), makeStr("resolve")); +/** + * "tx.included" + */ +inline constexpr auto txIncluded = join(makeStr("tx"), makeStr("included")); + +/** + * Phase transition events — fired on consensus.round at each transition + * so the round-level span carries a complete timeline of phase changes, + * including the handleWrongLedger recovery edge that re-enters Open. + */ +inline constexpr auto phaseOpen = join(makeStr("phase"), makeStr("open")); +inline constexpr auto phaseEstablish = join(makeStr("phase"), makeStr("establish")); +inline constexpr auto phaseAccepted = join(makeStr("phase"), makeStr("accepted")); +inline constexpr auto phaseRecovery = join(makeStr("phase"), makeStr("recovery")); + +/** + * Outcome events — fired on consensus.round at the establish→accepted + * transition so the path that drove acceptance is queryable. + */ +inline constexpr auto outcomeYes = join(makeStr("outcome"), makeStr("yes")); +inline constexpr auto outcomeMovedOn = join(makeStr("outcome"), makeStr("moved_on")); +inline constexpr auto outcomeExpired = join(makeStr("outcome"), makeStr("expired")); +} // namespace event + +// ===== Attribute values ====================================================== + +namespace val { +inline constexpr auto finished = makeStr("finished"); +inline constexpr auto movedOn = makeStr("moved_on"); +inline constexpr auto yes = makeStr("yes"); +inline constexpr auto no = makeStr("no"); +inline constexpr auto expired = makeStr("expired"); +inline constexpr auto increased = makeStr("increased"); +inline constexpr auto decreased = makeStr("decreased"); +inline constexpr auto unchanged = makeStr("unchanged"); +// consensus_phase attribute values (the phase the round is entering). +inline constexpr auto phaseOpen = makeStr("open"); +inline constexpr auto phaseEstablish = makeStr("establish"); +inline constexpr auto phaseAccepted = makeStr("accepted"); +} // namespace val + +} // namespace xrpl::telemetry::consensus::span diff --git a/src/xrpld/consensus/ConsensusTypes.h b/src/xrpld/consensus/ConsensusTypes.h index 4553e46f482..69c1a55aac2 100644 --- a/src/xrpld/consensus/ConsensusTypes.h +++ b/src/xrpld/consensus/ConsensusTypes.h @@ -79,10 +79,32 @@ to_string(ConsensusMode m) } } +/** + * Title Case display name for telemetry attributes and dashboards. + * Separate from to_string() which is used in logs and must remain stable. + */ +inline std::string +toDisplayString(ConsensusMode m) +{ + switch (m) + { + case ConsensusMode::Proposing: + return "Proposing"; + case ConsensusMode::Observing: + return "Observing"; + case ConsensusMode::WrongLedger: + return "Wrong Ledger"; + case ConsensusMode::SwitchedLedger: + return "Switched Ledger"; + default: + return "Unknown"; + } +} + /** * Phases of consensus for a single ledger round. * - * @code + * @code * "close" "accept" * open ------- > establish ---------> accepted * ^ | | diff --git a/src/xrpld/consensus/DisputedTx.h b/src/xrpld/consensus/DisputedTx.h index 12ed00d4607..6fc2912c831 100644 --- a/src/xrpld/consensus/DisputedTx.h +++ b/src/xrpld/consensus/DisputedTx.h @@ -198,6 +198,24 @@ class DisputedTx [[nodiscard]] json::Value getJson() const; + /** + * Number of peers voting yes. + */ + [[nodiscard]] int + getYays() const + { + return yays_; + } + + /** + * Number of peers voting no. + */ + [[nodiscard]] int + getNays() const + { + return nays_; + } + private: int yays_{0}; //< Number of yes votes int nays_{0}; //< Number of no votes diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 8838970b5ff..7f4758c4ef1 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,12 +16,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include +#include #include #include @@ -51,6 +56,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +68,8 @@ #include #include #include +#include +#include #include #include @@ -1306,6 +1314,20 @@ PeerImp::handleTransaction( auto stx = std::make_shared(sit); uint256 const txID = stx->getTransactionID(); + using namespace telemetry; + // Detached: this span is handed to a job-queue worker and must not + // leave its Scope bound to this peer thread's context stack (that + // leak would adopt later peer messages into this transaction's trace). + auto span = std::make_shared(txReceiveSpan(txID, *m).detached()); + span->setAttribute(tx_span::attr::txHash, to_string(txID).c_str()); + span->setAttribute(tx_span::attr::peerId, static_cast(id_)); + if (auto const* fmt = TxFormats::getInstance().findByType(stx->getTxnType())) + span->setAttribute(tx_span::attr::txType, fmt->getName().c_str()); + if (auto const version = getVersion(); !version.empty()) + span->setAttribute(tx_span::attr::peerVersion, version.c_str()); + // Note: suppressed and txStatus are set once at each exit path + // (not as defaults here) to avoid OTel SDK attribute duplication. + // Charge strongly for attempting to relay a txn with tfInnerBatchTxn // LCOV_EXCL_START /* @@ -1326,6 +1348,7 @@ PeerImp::handleTransaction( */ if (stx->isFlag(tfInnerBatchTxn)) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::rejectedInnerBatch); JLOG(pJournal_.warn()) << "Ignoring Network relayed Tx containing " "tfInnerBatchTxn (handleTransaction)."; fee_.update(Resource::kFeeModerateBurdenPeer, "inner batch txn"); @@ -1338,18 +1361,28 @@ PeerImp::handleTransaction( if (!app_.getHashRouter().shouldProcess(txID, id_, flags, kTxInterval)) { + span->setAttribute(tx_span::attr::suppressed, true); // we have seen this transaction recently if (any(flags & HashRouterFlags::BAD)) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::knownBad); fee_.update(Resource::kFeeUselessData, "known bad"); JLOG(pJournal_.debug()) << "Ignoring known bad tx " << txID; } - - // Erase only if the server has seen this tx. If the server has not - // seen this tx then the tx could not has been queued for this peer. - else if (eraseTxQueue && txReduceRelayEnabled()) + else { - removeTxQueue(txID); + // Recently-seen but not flagged bad — this is the plain + // duplicate-suppression path. Mark it explicitly so the + // span never exits as "new". + span->setAttribute(tx_span::attr::txStatus, tx_span::val::suppressed); + + // Erase only if the server has seen this tx. If the server + // has not seen this tx then the tx could not have been + // queued for this peer. + if (eraseTxQueue && txReduceRelayEnabled()) + { + removeTxQueue(txID); + } } overlay_.reportInboundTraffic( @@ -1358,6 +1391,7 @@ PeerImp::handleTransaction( return; } + span->setAttribute(tx_span::attr::suppressed, false); JLOG(pJournal_.debug()) << "Got tx " << txID; bool checkSignature = true; @@ -1382,10 +1416,12 @@ PeerImp::handleTransaction( if (app_.getLedgerMaster().getValidatedLedgerAge() > 4min) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::droppedNoSync); JLOG(pJournal_.trace()) << "No new transactions until synchronized"; } else if (app_.getJobQueue().getJobCount(JtTransaction) > app_.config().maxTransactions) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::droppedQueueFull); overlay_.incJqTransOverflow(); JLOG(pJournal_.info()) << "Transaction queue is full"; } @@ -1398,7 +1434,8 @@ PeerImp::handleTransaction( flags, checkSignature, batch, - stx]() { + stx, + sp = std::move(span)]() { if (auto peer = weak.lock()) peer->checkTransaction(flags, checkSignature, stx, batch); }); @@ -1719,6 +1756,12 @@ PeerImp::onMessage(std::shared_ptr const& m) void PeerImp::onMessage(std::shared_ptr const& m) { + using namespace telemetry; + // root: inbound peer message entry point (kConsumer); must not inherit + // any span left active on this peer thread. + auto span = SpanGuard::rootSpan(TraceCategory::Peer, seg::peer, peer_span::op::proposalReceive); + span.setAttribute(peer_span::attr::peerId, static_cast(id_)); + protocol::TMProposeSet const& set = *m; auto const sig = makeSlice(set.signature()); @@ -1745,6 +1788,7 @@ PeerImp::onMessage(std::shared_ptr const& m) // every time a spam packet is received PublicKey const publicKey{makeSlice(set.nodepubkey())}; auto const isTrusted = app_.getValidators().trusted(publicKey); + span.setAttribute(peer_span::attr::proposalTrusted, isTrusted); // If the operator has specified that untrusted proposals be dropped then // this happens here I.e. before further wasting CPU verifying the signature @@ -1813,9 +1857,30 @@ PeerImp::onMessage(std::shared_ptr const& m) app_.getTimeKeeper().closeTime(), calcNodeID(app_.getValidatorManifests().getMasterKey(publicKey))}); + // Create a receive span that links to the sender's trace context + // (if propagated). shared_ptr keeps it alive across the job boundary. + // Detach the guard's Scope on this peer thread so it is not popped on the + // job worker thread (which would leak this thread's context stack). + auto consSpan = + std::make_shared(telemetry::proposalReceiveSpan(set).detached()); + consSpan->setAttribute(telemetry::consensus::span::attr::proposalTrusted, isTrusted); + consSpan->setAttribute( + telemetry::consensus::span::attr::round, static_cast(set.proposeseq())); + // First 16 hex chars (8 bytes) of each hash — enough to disambiguate + // peer positions and prior ledgers without exporting full 32-byte + // hashes on every receive event. + consSpan->setAttribute( + telemetry::consensus::span::attr::prevLedgerPrefix, + to_string(prevLedger).substr(0, 16).c_str()); + consSpan->setAttribute( + telemetry::consensus::span::attr::positionHashPrefix, + to_string(proposeHash).substr(0, 16).c_str()); + std::weak_ptr const weak = shared_from_this(); app_.getJobQueue().addJob( - isTrusted ? JtProposalT : JtProposalUt, "checkPropose", [weak, isTrusted, m, proposal]() { + isTrusted ? JtProposalT : JtProposalUt, + "checkPropose", + [weak, isTrusted, m, proposal, sp = std::move(consSpan)]() { if (auto peer = weak.lock()) peer->checkPropose(isTrusted, m, proposal); }); @@ -2315,6 +2380,13 @@ PeerImp::onMessage(std::shared_ptr const& m void PeerImp::onMessage(std::shared_ptr const& m) { + using namespace telemetry; + // root: inbound peer message entry point (kConsumer); must not inherit + // any span left active on this peer thread. + auto valSpan = + SpanGuard::rootSpan(TraceCategory::Peer, seg::peer, peer_span::op::validationReceive); + valSpan.setAttribute(peer_span::attr::peerId, static_cast(id_)); + if (m->validation().size() < 50) { JLOG(pJournal_.warn()) << "Validation: Too small"; @@ -2337,6 +2409,8 @@ PeerImp::onMessage(std::shared_ptr const& m) false); val->setSeen(closeTime); } + valSpan.setAttribute(peer_span::attr::ledgerHash, to_string(val->getLedgerHash()).c_str()); + valSpan.setAttribute(peer_span::attr::fullValidation, val->isFull()); if (!isCurrent( app_.getValidations().parms(), @@ -2353,6 +2427,7 @@ PeerImp::onMessage(std::shared_ptr const& m) // suppression for 30 seconds to avoid doing a relatively expensive // lookup every time a spam packet is received auto const isTrusted = app_.getValidators().trusted(val->getSignerPublic()); + valSpan.setAttribute(peer_span::attr::validationTrusted, isTrusted); // If the operator has specified that untrusted validations be // dropped then this happens here I.e. before further wasting CPU @@ -2390,6 +2465,24 @@ PeerImp::onMessage(std::shared_ptr const& m) return; } + // Create a receive span that links to the sender's trace context + // (if propagated). shared_ptr keeps it alive across the job boundary. + // Detach the guard's Scope on this peer thread so it is not popped on + // the job worker thread (which would leak this thread's context stack). + auto consSpan = + std::make_shared(telemetry::validationReceiveSpan(*m).detached()); + consSpan->setAttribute(telemetry::consensus::span::attr::validationTrusted, isTrusted); + if (val->isFieldPresent(sfLedgerSequence)) + { + consSpan->setAttribute( + telemetry::consensus::span::attr::ledgerSeq, + static_cast(val->getFieldU32(sfLedgerSequence))); + } + consSpan->setAttribute(telemetry::consensus::span::attr::fullValidation, val->isFull()); + consSpan->setAttribute( + telemetry::consensus::span::attr::validationSignTime, + static_cast(val->getSignTime().time_since_epoch().count())); + if (!isTrusted && (tracking_.load() == Tracking::Diverged)) { JLOG(pJournal_.debug()) << "Dropping untrusted validation from diverged peer"; @@ -2400,7 +2493,9 @@ PeerImp::onMessage(std::shared_ptr const& m) std::weak_ptr const weak = shared_from_this(); app_.getJobQueue().addJob( - isTrusted ? JtValidationT : JtValidationUt, name, [weak, val, m, key]() { + isTrusted ? JtValidationT : JtValidationUt, + name, + [weak, val, m, key, sp = std::move(consSpan)]() { if (auto peer = weak.lock()) peer->checkValidation(val, key, m); }); diff --git a/src/xrpld/overlay/detail/PeerSpanNames.h b/src/xrpld/overlay/detail/PeerSpanNames.h new file mode 100644 index 00000000000..6212d8fca25 --- /dev/null +++ b/src/xrpld/overlay/detail/PeerSpanNames.h @@ -0,0 +1,45 @@ +#pragma once + +/** + * Compile-time span name constants for peer overlay tracing. + * + * Used by PeerImp for peer message handling spans (proposals, + * validations). Built on StaticStr/join() from SpanNames.h. + * + * Span hierarchy: + * + * peer.proposal.receive (PeerImp — incoming proposal) + * peer.validation.receive (PeerImp — incoming validation) + */ + +#include + +namespace xrpl::telemetry::peer_span { + +// ===== Span operation suffixes =============================================== + +namespace op { +inline constexpr auto proposalReceive = makeStr("proposal.receive"); +inline constexpr auto validationReceive = makeStr("validation.receive"); +} // namespace op + +// ===== Attribute keys ======================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). `ledgerHash` and + * `fullValidation` are shared with the consensus validation spans — same + * concept, same key, told apart by span name. + */ +using ::xrpl::telemetry::attr::fullValidation; +using ::xrpl::telemetry::attr::ledgerHash; +using ::xrpl::telemetry::attr::peerId; + +/** + * Trust flag qualified by message type, shared with consensus.*.receive. + */ +inline constexpr auto proposalTrusted = makeStr("proposal_trusted"); +inline constexpr auto validationTrusted = makeStr("validation_trusted"); +} // namespace attr + +} // namespace xrpl::telemetry::peer_span diff --git a/src/xrpld/overlay/detail/TrafficCount.h b/src/xrpld/overlay/detail/TrafficCount.h index 8dc8ddb08ee..055f9c270c3 100644 --- a/src/xrpld/overlay/detail/TrafficCount.h +++ b/src/xrpld/overlay/detail/TrafficCount.h @@ -282,7 +282,7 @@ class TrafficCount {Category::ShareCasObject, "getobject_CAS_share"}, {Category::GetCasObject, "getobject_CAS_get"}, {Category::ShareFetchPack, "getobject_Fetch_Pack_share"}, - {Category::GetFetchPack, "getobject_Fetch Pack_get"}, + {Category::GetFetchPack, "getobject_Fetch_Pack_get"}, {Category::GetTransactions, "getobject_Transactions_get"}, {Category::ShareHash, "getobject_share"}, {Category::GetHash, "getobject_get"}, diff --git a/src/xrpld/perflog/detail/PerfLogImp.cpp b/src/xrpld/perflog/detail/PerfLogImp.cpp index 3aa7e38ea25..f06df3f3ffc 100644 --- a/src/xrpld/perflog/detail/PerfLogImp.cpp +++ b/src/xrpld/perflog/detail/PerfLogImp.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -330,6 +332,10 @@ PerfLogImp::rpcStart(std::string const& method, std::uint64_t const requestId) } std::scoped_lock const lock(counters_.methodsMutex); counters_.methods[requestId] = {counter->first.c_str(), steady_clock::now()}; + + // Task 9.4: Record RPC start in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordRpcStarted(method); } void @@ -356,20 +362,42 @@ PerfLogImp::rpcEnd(std::string const& method, std::uint64_t const requestId, boo { // LCOV_EXCL_START UNREACHABLE("xrpl::perf::PerfLogImp::rpcEnd : valid requestId input"); + // Without an entry, startTime stays default-initialized; returning + // avoids recording a bogus (now - epoch) duration to the counters + // and OTel histogram below. UNREACHABLE does not halt release builds. + return; // LCOV_EXCL_STOP } } - std::scoped_lock const lock(counter->second.mutex); - if (finish) + auto const durationUs = + std::chrono::duration_cast(steady_clock::now() - startTime); { - ++counter->second.value.finished; + std::scoped_lock const lock(counter->second.mutex); + if (finish) + { + ++counter->second.value.finished; + } + else + { + ++counter->second.value.errored; + } + counter->second.value.duration += durationUs; } - else + + // Task 9.4: Record RPC completion in OTel metrics pipeline. + // Mirrors the rpcStart() instrumentation so the finished/errored + // counters and duration histogram advance with every call. + if (auto* mr = app_.getMetricsRegistry()) { - ++counter->second.value.errored; + if (finish) + { + mr->recordRpcFinished(method, durationUs.count()); + } + else + { + mr->recordRpcErrored(method, durationUs.count()); + } } - counter->second.value.duration += - std::chrono::duration_cast(steady_clock::now() - startTime); } void @@ -385,6 +413,10 @@ PerfLogImp::jobQueue(JobType const type) } std::scoped_lock const lock(counter->second.mutex); ++counter->second.value.queued; + + // Task 9.5: Record job enqueue in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordJobQueued(JobTypes::name(type)); } void @@ -411,6 +443,10 @@ PerfLogImp::jobStart( std::scoped_lock const lock(counters_.jobsMutex); if (instance >= 0 && instance < counters_.jobs.size()) counters_.jobs[instance] = {type, startTime}; + + // Task 9.5: Record job start in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordJobStarted(JobTypes::name(type), dur.count()); } void @@ -433,6 +469,10 @@ PerfLogImp::jobFinish(JobType const type, microseconds dur, int instance) std::scoped_lock const lock(counters_.jobsMutex); if (instance >= 0 && instance < counters_.jobs.size()) counters_.jobs[instance] = {JtInvalid, steady_time_point()}; + + // Task 9.5: Record job finish in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordJobFinished(JobTypes::name(type), dur.count()); } void diff --git a/src/xrpld/rpc/ServerHandler.h b/src/xrpld/rpc/ServerHandler.h index 054bec9b5b2..fbe487654ee 100644 --- a/src/xrpld/rpc/ServerHandler.h +++ b/src/xrpld/rpc/ServerHandler.h @@ -192,7 +192,12 @@ class ServerHandler void processSession(std::shared_ptr const&, std::shared_ptr coro); - void + /** + * Process an RPC request and write the reply to `output`. + * @return false if the request resulted in an error response, true + * otherwise. Lets the caller's enclosing span reflect the outcome. + */ + bool processRequest( Port const& port, std::string const& request, diff --git a/src/xrpld/rpc/detail/PathFindSpanNames.h b/src/xrpld/rpc/detail/PathFindSpanNames.h new file mode 100644 index 00000000000..65ce81bb964 --- /dev/null +++ b/src/xrpld/rpc/detail/PathFindSpanNames.h @@ -0,0 +1,114 @@ +#pragma once + +/** + * Compile-time span name constants for PathFind tracing. + * + * Covers the path_find and ripple_path_find RPC handlers, the + * PathRequest computation engine, and the Pathfinder graph exploration. + * + * Span hierarchy: + * + * RPC entry (one-shot or subscription): + * + * +----------------------------------------------------------------+ + * | pathfind.request | + * | doPathFind() / doRipplePathFind() | + * | attrs: pathfind_source_account, pathfind_dest_account | + * | (set when present in request params) | + * | | + * | +-----------------------------------------------------------+ | + * | | pathfind.compute | | + * | | PathRequest::doUpdate() | | + * | | attrs: pathfind_fast | | + * | | | | + * | | +-----------------------------------------------------+ | | + * | | | pathfind.discover (one per RPC call, hoisted above | | + * | | | the per-source-asset loop in PathRequest::findPaths)| | + * | | | attrs: pathfind_search_level, pathfind_num_paths | | + * | | +-----------------------------------------------------+ | | + * | +-----------------------------------------------------------+ | + * +----------------------------------------------------------------+ + * + * Async recomputation (ledger close): + * + * +----------------------------------------------------------------+ + * | pathfind.update_all | + * | PathRequestManager::updateAll() | + * | attrs: pathfind_ledger_index, pathfind_num_requests | + * | | + * | +-----------------------------------------------------------+ | + * | | pathfind.compute (per active request) | | + * | +-----------------------------------------------------------+ | + * +----------------------------------------------------------------+ + */ + +#include + +namespace xrpl::telemetry::pathfind_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "pathfind" — root prefix for path finding spans. + */ +inline constexpr auto pathfind = makeStr("pathfind"); +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto request = makeStr("request"); +inline constexpr auto compute = makeStr("compute"); +inline constexpr auto updateAll = makeStr("update_all"); +inline constexpr auto discover = makeStr("discover"); +} // namespace op + +// ===== Attribute keys ====================================================== +// +// All pathfind attributes are namespaced under `pathfind_*` (underscore form, +// per Phase 1c naming spec rule 5). Avoids collisions with bare keys like +// `fast` or `num_paths` that other subsystems may introduce. + +namespace attr { +/** + * "pathfind_source_account" — originating account for path search. + */ +inline constexpr auto sourceAccount = makeStr("pathfind_source_account"); +/** + * "pathfind_dest_account" — destination account. + */ +inline constexpr auto destAccount = makeStr("pathfind_dest_account"); +/** + * "pathfind_fast" — whether fast pathfinding mode enabled. + */ +inline constexpr auto fast = makeStr("pathfind_fast"); +/** + * "pathfind_search_level" — depth of graph exploration. + */ +inline constexpr auto searchLevel = makeStr("pathfind_search_level"); +/** + * "pathfind_num_paths" — total paths produced across the per-source-asset + * loop in PathRequest::findPaths (sum of getBestPaths().size() per asset). + */ +inline constexpr auto numPaths = makeStr("pathfind_num_paths"); +/** + * "pathfind_num_requests" — snapshot size of requests_ at update_all start + * (may include weak_ptrs that subsequently expire during processing). + */ +inline constexpr auto numRequests = makeStr("pathfind_num_requests"); +/** + * "pathfind_ledger_index" — pathfind target ledger index. + */ +inline constexpr auto ledgerIndex = makeStr("pathfind_ledger_index"); +/** + * "pathfind_dest_currency" — destination currency code. + */ +inline constexpr auto destCurrency = makeStr("pathfind_dest_currency"); +/** + * "pathfind_num_source_assets" — candidate source assets count. + */ +inline constexpr auto numSourceAssets = makeStr("pathfind_num_source_assets"); +} // namespace attr + +} // namespace xrpl::telemetry::pathfind_span diff --git a/src/xrpld/rpc/detail/PathRequest.cpp b/src/xrpld/rpc/detail/PathRequest.cpp index 3c09917dad0..4fb11ab10c9 100644 --- a/src/xrpld/rpc/detail/PathRequest.cpp +++ b/src/xrpld/rpc/detail/PathRequest.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -34,10 +35,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -579,6 +582,22 @@ PathRequest::findPaths( auto const dstAmount = convertAmount(saDstAmount_, convertAll_); hash_map> currencyMap; + + // One `pathfind.discover` span wraps the entire per-source-asset loop so + // that a single RPC call produces one discover span instead of N (one per + // candidate source asset). Trade-off: per-asset discovery/ranking timing + // is no longer split into individual spans — span count and Tempo storage + // are bounded per RPC at the cost of per-asset visibility. If per-asset + // breakdown is needed in the future, add child spans inside the loop body + // (`Pathfinder::findPaths`/`computePathRanks`) parented off this span. + using namespace telemetry; + auto span = SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::discover); + span.setAttribute(pathfind_span::attr::searchLevel, static_cast(level)); + span.setAttribute( + pathfind_span::attr::numSourceAssets, static_cast(sourceAssets.size())); + + std::int64_t totalPaths = 0; for (auto const& asset : sourceAssets) { if (continueCallback && !continueCallback()) @@ -598,6 +617,7 @@ PathRequest::findPaths( auto ps = pathfinder->getBestPaths( kMaxPaths, fullLiquidityPath, context_[asset], asset.getIssuer(), continueCallback); context_[asset] = ps; + totalPaths += static_cast(ps.size()); auto const& sourceAccount = [&] { if (!isXRP(asset.getIssuer())) @@ -700,6 +720,8 @@ PathRequest::findPaths( } } + span.setAttribute(pathfind_span::attr::numPaths, totalPaths); + /* The resource fee is based on the number of source currencies used. The minimum cost is 50 and the maximum is 400. The cost increases after four source currencies, 50 - (4 * 4) = 34. @@ -716,6 +738,12 @@ PathRequest::doUpdate( std::function const& continueCallback) { using namespace std::chrono; + using namespace telemetry; + auto span = SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::compute); + span.setAttribute(pathfind_span::attr::fast, fast); + span.setAttribute(pathfind_span::attr::destCurrency, to_string(saDstAmount_.asset()).c_str()); + JLOG(journal_.debug()) << iIdentifier_ << " update " << (fast ? "fast" : "normal"); { diff --git a/src/xrpld/rpc/detail/PathRequestManager.cpp b/src/xrpld/rpc/detail/PathRequestManager.cpp index 4953634181e..9141aaded48 100644 --- a/src/xrpld/rpc/detail/PathRequestManager.cpp +++ b/src/xrpld/rpc/detail/PathRequestManager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include @@ -72,6 +74,19 @@ PathRequestManager::updateAll(std::shared_ptr const& inLedger) cache = getAssetCache(inLedger, true); } + using namespace telemetry; + // updateAll runs on every ledger close. Skip span emission when there are + // no active path subscriptions, to avoid a steady stream of empty spans at + // mainnet close cadence. A null guard is used in that case; all other work + // still runs unchanged (notably the isNewPathRequest() flag reset below), + // so behaviour matches the pre-span code path. + auto span = requests.empty() + ? SpanGuard{} + : SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); + span.setAttribute(pathfind_span::attr::ledgerIndex, static_cast(inLedger->seq())); + span.setAttribute(pathfind_span::attr::numRequests, static_cast(requests.size())); + bool newRequests = app_.getLedgerMaster().isNewPathRequest(); bool mustBreak = false; diff --git a/src/xrpld/rpc/detail/RPCHandler.cpp b/src/xrpld/rpc/detail/RPCHandler.cpp index 6f46aed62dd..4990cfd01c6 100644 --- a/src/xrpld/rpc/detail/RPCHandler.cpp +++ b/src/xrpld/rpc/detail/RPCHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -17,14 +18,18 @@ #include #include #include +#include #include #include #include #include #include +#include -namespace xrpl::RPC { +namespace xrpl { +using namespace telemetry; +namespace RPC { namespace { @@ -157,6 +162,14 @@ template Status callMethod(JsonContext& context, Method method, std::string const& name, Object& result) { + auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::command, name); + span.setAttribute(rpc_span::attr::command, name.c_str()); + span.setAttribute(rpc_span::attr::version, static_cast(context.apiVersion)); + span.setAttribute( + rpc_span::attr::rpcRole, + context.role == Role::ADMIN ? std::string_view(rpc_span::val::admin) + : std::string_view(rpc_span::val::user)); + static std::atomic kRequestId{0}; auto& perfLog = context.app.getPerfLog(); std::uint64_t const curId = ++kRequestId; @@ -172,12 +185,32 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& JLOG(context.j.debug()) << "RPC call " << name << " completed in " << ((end - start).count() / 1000000000.0) << "seconds"; perfLog.rpcFinish(name, curId); + span.setAttribute(rpc_span::attr::loadType, context.loadType.label().c_str()); + // Status::operator bool() returns true when there IS an error + // (code_ != OK), so the ternary correctly maps error->error, ok->success. + span.setAttribute( + rpc_span::attr::rpcStatus, + ret ? std::string_view{rpc_span::val::error} + : std::string_view{rpc_span::val::success}); + // Reflect the result in the OTel span status, not just the attribute, + // so non-exception RPC errors (rpcTOO_BUSY, rpcNO_PERMISSION, ...) are + // visible to {status.code=error} queries. + if (ret) + { + span.setError(rpc_span::val::error); + } + else + { + span.setOk(); + } return ret; } catch (std::exception& e) { perfLog.rpcError(name, curId); JLOG(context.j.info()) << "Caught throw: " << e.what(); + span.recordException(e); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); if (context.loadType == Resource::kFeeReferenceRpc) context.loadType = Resource::kFeeExceptionRpc; @@ -187,6 +220,31 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& } } +// Resolve the span suffix / command attribute for a request that failed in +// fillHandler. Returns the canonical handler name for a recognized command +// (a finite, bounded set) or the literal "unknown" for a request that omits +// both fields or names an unregistered command. The raw request value is +// deliberately NOT used: the command attribute is promoted to a Prometheus +// label by the spanmetrics connector, so an attacker-controlled string would +// let arbitrary request input drive unbounded span-name / label cardinality. +// Resolving against the registry keeps per-command error attribution for real +// commands (e.g. a submit rejected with rpcTOO_BUSY stays rpc.command.submit) +// while collapsing garbage input to a single series. +std::string_view +resolveCommandSpanName(JsonContext const& context) +{ + if (!context.params.isMember(jss::command) && !context.params.isMember(jss::method)) + return rpc_span::val::unknownCommand; + + std::string const cmd = context.params.isMember(jss::command) + ? context.params[jss::command].asString() + : context.params[jss::method].asString(); + + auto const* handler = getHandler(context.apiVersion, context.app.config().betaRpcApi, cmd); + return (handler != nullptr) ? std::string_view{handler->name} + : std::string_view{rpc_span::val::unknownCommand}; +} + } // namespace Status @@ -195,6 +253,15 @@ doCommand(RPC::JsonContext& context, json::Value& result) Handler const* handler = nullptr; if (auto error = fillHandler(context, handler)) { + // Bound the span name and command attribute to the finite set of + // registered handler names (plus "unknown") — see the helper for why + // raw request input must not reach the telemetry pipeline. + auto const cmdName = resolveCommandSpanName(context); + auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::command, cmdName); + span.setAttribute(rpc_span::attr::command, cmdName); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(getErrorInfo(error).token.cStr()); + injectError(error, result); return error; } @@ -234,4 +301,5 @@ roleRequired(unsigned int version, bool betaEnabled, std::string const& method) return handler->role; } -} // namespace xrpl::RPC +} // namespace RPC +} // namespace xrpl diff --git a/src/xrpld/rpc/detail/RpcSpanNames.h b/src/xrpld/rpc/detail/RpcSpanNames.h new file mode 100644 index 00000000000..07202a7592e --- /dev/null +++ b/src/xrpld/rpc/detail/RpcSpanNames.h @@ -0,0 +1,190 @@ +#pragma once + +/** + * Compile-time span name constants for the RPC subsystem. + * + * All span prefixes, operation names, and attribute keys used by RPC + * tracing call sites are defined here. Built on the StaticStr/join() + * primitives from . + * + * Usage: + * @code + * #include + * using namespace telemetry; + * + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, "submit"); + * span.setAttribute(rpc_span::attr::command, "submit"); + * span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success); + * @endcode + * + * Span hierarchy (automatic nesting via OTel thread-local context): + * + * HTTP JSON-RPC path (single request): + * + * +-------------------------------------------------------+ + * | rpc.http_request | + * | ServerHandler::processSession(Session) | + * | | + * | +--------------------------------------------------+ | + * | | rpc.process | | + * | | ServerHandler::processRequest() | | + * | | | | + * | | +---------------------------------------------+ | | + * | | | rpc.command.{name} | | | + * | | | RPC::callMethod() | | | + * | | | attrs: command, version, rpc_role, rpc_status | | | + * | | +---------------------------------------------+ | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * HTTP batch path (multiple commands per request): + * + * +-------------------------------------------------------+ + * | rpc.http_request | + * | | + * | +--------------------------------------------------+ | + * | | rpc.process | | + * | | | | + * | | +------------------+ +------------------+ | | + * | | | rpc.command.{a} | | rpc.command.{b} | ... | | + * | | +------------------+ +------------------+ | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * WebSocket path: + * + * +-------------------------------------------------------+ + * | rpc.ws_message | + * | ServerHandler::processSession(WSSession) | + * | | + * | +--------------------------------------------------+ | + * | | rpc.command.{name} | | + * | | RPC::callMethod() | | + * | | attrs: command, version, rpc_role, rpc_status | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * WebSocket error paths: + * + * +-------------------------------------------------------+ + * | rpc.ws_message (error: invalid_json) | + * | ServerHandler::onWSMessage() — parse failure | + * +-------------------------------------------------------+ + * + * +-------------------------------------------------------+ + * | rpc.ws_upgrade | + * | ServerHandler::onHandoff() — upgrade try/catch | + * +-------------------------------------------------------+ + * + * Command dispatch error path: + * + * +-------------------------------------------------------+ + * | rpc.command.{name} (error: too_busy/unknown/etc) | + * | RPC::doCommand() — fillHandler() rejection | + * +-------------------------------------------------------+ + * + * gRPC path (see GrpcSpanNames.h for constants): + * + * +-------------------------------------------------------+ + * | grpc. (e.g. grpc.GetLedger) | + * | CallData::process(coro) | + * | attrs: method, grpc_status | + * +-------------------------------------------------------+ + * + * Covered paths: + * - HTTP JSON-RPC (single and batch requests) + * - WebSocket RPC commands + * - WebSocket message parse errors (invalid JSON, oversized) + * - WebSocket upgrade failures (protocol handshake errors) + * - Admin CLI (connects via HTTP internally) + * - Command dispatch rejections (unknown cmd, too busy, no perm) + * - gRPC endpoints (GetLedger, GetLedgerData, GetLedgerDiff, + * GetLedgerEntry) + * - Command execution: timing, success/failure, exceptions + * - Per-command attributes: name, API version, rpc_role, rpc_status + * + * Known gaps (not yet instrumented): + * - Early validation errors in processRequest() before rpc.process + * span (malformed JSON, auth failures, oversized requests) + * - Subscription push notifications (server-initiated, not RPC) + */ + +#include + +namespace xrpl::telemetry::rpc_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "rpc" — root prefix for transport-level spans. + */ +inline constexpr auto rpc = seg::rpc; +/** + * "rpc.command" — prefix for individual RPC command spans. + */ +inline constexpr auto command = join(seg::rpc, makeStr("command")); +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto wsMessage = makeStr("ws_message"); +inline constexpr auto wsUpgrade = makeStr("ws_upgrade"); +inline constexpr auto httpRequest = makeStr("http_request"); +inline constexpr auto process = makeStr("process"); +} // namespace op + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * "command" — RPC method name. + */ +inline constexpr auto command = makeStr("command"); +/** + * "version" — api_version per request. + */ +inline constexpr auto version = makeStr("version"); +/** + * "rpc_role" — admin|user. Domain-qualified: collides with grpc_role. + */ +inline constexpr auto rpcRole = makeStr("rpc_role"); +/** + * "rpc_status" — success|error. Domain-qualified: avoids OTel reserved span status. + */ +inline constexpr auto rpcStatus = makeStr("rpc_status"); +/** + * "request_payload_size" — bytes of inbound request payload. + */ +inline constexpr auto requestPayloadSize = makeStr("request_payload_size"); +/** + * "is_batch" — whether request is a JSON-RPC batch. + */ +inline constexpr auto isBatch = makeStr("is_batch"); +/** + * "batch_size" — number of sub-requests in a batch. + */ +inline constexpr auto batchSize = makeStr("batch_size"); +/** + * "load_type" — resource cost category after execution. + */ +inline constexpr auto loadType = makeStr("load_type"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +using telemetry::attr_val::error; +using telemetry::attr_val::success; +inline constexpr auto admin = makeStr("admin"); +inline constexpr auto user = makeStr("user"); +inline constexpr auto unknownCommand = makeStr("unknown"); +/** + * "invalid_json" — WS message parse failure or oversize. + */ +inline constexpr auto invalidJson = makeStr("invalid_json"); +} // namespace val + +} // namespace xrpl::telemetry::rpc_span diff --git a/src/xrpld/rpc/detail/ServerHandler.cpp b/src/xrpld/rpc/detail/ServerHandler.cpp index 768cbf0dc07..48d5760724e 100644 --- a/src/xrpld/rpc/detail/ServerHandler.cpp +++ b/src/xrpld/rpc/detail/ServerHandler.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include // IWYU pragma: keep #include #include @@ -43,6 +45,7 @@ #include #include #include +#include #include #include @@ -60,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +75,7 @@ #include namespace xrpl { +using namespace telemetry; class Peer; class LedgerMaster; @@ -224,13 +229,17 @@ ServerHandler::onHandoff( if (!isWs) return statusRequestResponse(request, http::status::unauthorized); + auto span = + SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::wsUpgrade); std::shared_ptr ws; try { ws = session.websocketUpgrade(); + span.setOk(); } catch (std::exception const& e) { + span.recordException(e); // LCOV_EXCL_LINE JLOG(journal_.error()) << "Exception upgrading websocket: " << e.what() << "\n"; return statusRequestResponse(request, http::status::internal_server_error); } @@ -339,10 +348,14 @@ ServerHandler::onWSMessage( auto const size = boost::asio::buffer_size(buffers); if (size > RPC::Tuning::kMaxRequestSize || !json::Reader{}.parse(jv, buffers) || !jv.isObject()) { + auto span = + SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::wsMessage); + span.setError(rpc_span::val::invalidJson); + json::Value jvResult(json::ValueType::Object); jvResult[jss::type] = jss::error; jvResult[jss::error] = "jsonInvalid"; - jvResult[jss::value] = buffersToString(buffers); + jvResult[jss::value] = ::xrpl::buffersToString(buffers); boost::beast::multi_buffer sb; json::stream(jvResult, [&sb](auto const p, auto const n) { sb.commit(boost::asio::buffer_copy(sb.prepare(n), boost::asio::buffer(p, n))); @@ -415,12 +428,24 @@ ServerHandler::processSession( std::shared_ptr const& coro, json::Value const& jv) { + auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::wsMessage); + if (jv.isMember(jss::command) && jv[jss::command].isString()) + { + span.setAttribute(rpc_span::attr::command, jv[jss::command].asString().c_str()); + } + else if (jv.isMember(jss::method) && jv[jss::method].isString()) + { + span.setAttribute(rpc_span::attr::command, jv[jss::method].asString().c_str()); + } + auto is = std::static_pointer_cast(session->appDefined); if (is->getConsumer().disconnect(journal_)) { session->close({boost::beast::websocket::policy_error, "threshold exceeded"}); // FIX: This rpcError is not delivered since the session // was just closed. + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError("resource threshold exceeded"); return rpcError(RpcSlowDown); } @@ -452,6 +477,8 @@ ServerHandler::processSession( jr[jss::api_version] = jv[jss::api_version]; is->getConsumer().charge(Resource::kFeeMalformedRpc); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(jr[jss::error].asString()); return jr; } @@ -498,6 +525,8 @@ ServerHandler::processSession( jr[jss::result] = RPC::makeError(RpcInternal); JLOG(journal_.error()) << "Exception while processing WS: " << ex.what() << "\n" << "Input JSON: " << json::Compact{json::Value{jv}}; + span.recordException(ex); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); // LCOV_EXCL_STOP } @@ -530,12 +559,19 @@ ServerHandler::processSession( } jr[jss::request] = rq; + // Mark the span according to the final result. Doing it here (rather + // than an unconditional setOk later) ensures error responses — from + // doCommand, a FORBID role, or the catch block above — are not + // overwritten as OK. + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(jr[jss::error].asString()); } else { if (jr[jss::result].isMember("forwarded") && jr[jss::result]["forwarded"]) jr = jr[jss::result]; jr[jss::status] = jss::success; + span.setOk(); } if (jv.isMember(jss::id)) @@ -557,9 +593,15 @@ ServerHandler::processSession( std::shared_ptr const& session, std::shared_ptr coro) { - processRequest( + auto span = + SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::httpRequest); + + auto const requestBody = ::xrpl::buffersToString(session->request().body().data()); + span.setAttribute(rpc_span::attr::requestPayloadSize, static_cast(requestBody.size())); + + bool const ok = processRequest( session->port(), - buffersToString(session->request().body().data()), + requestBody, session->remoteAddress().atPort(0), makeOutput(*session), coro, @@ -579,6 +621,15 @@ ServerHandler::processSession( { session->close(true); } + // Reflect the request outcome on the wrapper span instead of always OK. + if (ok) + { + span.setOk(); + } + else + { + span.setError(rpc_span::val::error); + } } static json::Value @@ -597,7 +648,7 @@ constexpr json::Int kServerOverloaded = -32604; constexpr json::Int kForbidden = -32605; constexpr json::Int kWrongVersion = -32606; -void +bool ServerHandler::processRequest( Port const& port, std::string const& request, @@ -607,20 +658,33 @@ ServerHandler::processRequest( std::string_view forwardedFor, std::string_view user) { + auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::process); auto rpcJ = app_.getJournal("RPC"); + // Tracks whether any failure occurred. Set on every error path (early + // returns, the catch block, and per-request error replies) and used at the + // end to mark the span status. The HTTP status code alone is insufficient: + // it stays 200 for batch responses and for ripplerpc < 3.0, so relying on + // it would let payload-level errors end the span as successful. + bool spanHadError = false; + + // Marks the span as failed before sending an error reply, so the + // early-return validation paths below are not later seen as successful + // (the span would otherwise end UNSET, invisible to {status.code=error}). + auto httpReplyError = [&](int status, std::string const& message) { + spanHadError = true; + span.setError(message); + httpReply(status, message, output, rpcJ); + }; + json::Value jsonOrig; { json::Reader reader; if ((request.size() > RPC::Tuning::kMaxRequestSize) || !reader.parse(request, jsonOrig) || !jsonOrig || !jsonOrig.isObject()) { - httpReply( - 400, - "Unable to parse request: " + reader.getFormattedErrorMessages(), - output, - rpcJ); - return; + httpReplyError(400, "Unable to parse request: " + reader.getFormattedErrorMessages()); + return false; } } @@ -631,11 +695,14 @@ ServerHandler::processRequest( batch = true; if (!jsonOrig.isMember(jss::params) || !jsonOrig[jss::params].isArray()) { - httpReply(400, "Malformed batch request", output, rpcJ); - return; + httpReplyError(400, "Malformed batch request"); + return false; } size = jsonOrig[jss::params].size(); } + span.setAttribute(rpc_span::attr::isBatch, batch); + if (batch) + span.setAttribute(rpc_span::attr::batchSize, static_cast(size)); json::Value reply(batch ? json::ValueType::Array : json::ValueType::Object); auto const start(std::chrono::high_resolution_clock::now()); @@ -670,8 +737,8 @@ ServerHandler::processRequest( { if (!batch) { - httpReply(400, jss::invalid_API_version.cStr(), output, rpcJ); - return; + httpReplyError(400, jss::invalid_API_version.cStr()); + return false; } json::Value r(json::ValueType::Object); r[jss::request] = jsonRPC; @@ -713,8 +780,8 @@ ServerHandler::processRequest( { if (!batch) { - httpReply(503, "Server is overloaded", output, rpcJ); - return; + httpReplyError(503, "Server is overloaded"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kServerOverloaded, "Server is overloaded"); @@ -728,8 +795,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(403, "Forbidden", output, rpcJ); - return; + httpReplyError(403, "Forbidden"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kForbidden, "Forbidden"); @@ -742,8 +809,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "Null method", output, rpcJ); - return; + httpReplyError(400, "Null method"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kMethodNotFound, "Null method"); @@ -757,8 +824,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "method is not string", output, rpcJ); - return; + httpReplyError(400, "method is not string"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kMethodNotFound, "method is not string"); @@ -772,8 +839,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "method is empty", output, rpcJ); - return; + httpReplyError(400, "method is empty"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kMethodNotFound, "method is empty"); @@ -798,8 +865,8 @@ ServerHandler::processRequest( else if (!params.isArray() || params.size() != 1) { usage.charge(Resource::kFeeMalformedRpc); - httpReply(400, "params unparsable", output, rpcJ); - return; + httpReplyError(400, "params unparsable"); + return false; } else { @@ -807,8 +874,8 @@ ServerHandler::processRequest( if (!params.isObjectOrNull()) { usage.charge(Resource::kFeeMalformedRpc); - httpReply(400, "params unparsable", output, rpcJ); - return; + httpReplyError(400, "params unparsable"); + return false; } } } @@ -825,8 +892,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "ripplerpc is not a string", output, rpcJ); - return; + httpReplyError(400, "ripplerpc is not a string"); + return false; } json::Value r = jsonRPC; @@ -883,6 +950,9 @@ ServerHandler::processRequest( JLOG(journal_.error()) << "Internal error : " << ex.what() << " when processing request: " << json::Compact{json::Value{params}}; + span.recordException(ex); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + spanHadError = true; // LCOV_EXCL_STOP } @@ -899,6 +969,7 @@ ServerHandler::processRequest( { if (result.isMember(jss::error)) { + spanHadError = true; result[jss::status] = jss::error; result["code"] = result[jss::error_code]; result["message"] = result[jss::error_message]; @@ -919,6 +990,7 @@ ServerHandler::processRequest( // received. if (result.isMember(jss::error)) { + spanHadError = true; auto rq = params; if (rq.isObject()) @@ -1014,7 +1086,20 @@ ServerHandler::processRequest( } } + // Mark the span error if any request failed or the HTTP status is an error. + // spanHadError catches payload-level errors that httpStatus misses (batch + // responses and ripplerpc < 3.0 always return HTTP 200). + if (spanHadError || httpStatus >= 400) + { + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(rpc_span::val::error); + } + else + { + span.setOk(); + } httpReply(httpStatus, response, output, rpcJ); + return !spanHadError; } //------------------------------------------------------------------------------ diff --git a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp index 3a6b52ee985..f04756be150 100644 --- a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -9,12 +10,23 @@ #include #include #include +#include +#include namespace xrpl { json::Value doPathFind(RPC::JsonContext& context) { + using namespace telemetry; + auto span = SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. + if (auto const& src = context.params[jss::source_account]; src.isString()) + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + if (auto const& dst = context.params[jss::destination_account]; dst.isString()) + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); + if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp index b7edfb6dbe3..24373a677fd 100644 --- a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -23,6 +26,26 @@ namespace xrpl { json::Value doRipplePathFind(RPC::JsonContext& context) { + using namespace telemetry; + // This RPC span is held live across context.coro->yield() below, so it is + // both rooted and detached: + // - rootSpan: on resume the coroutine may run on a different JobQueue + // worker whose thread-local context stack holds unrelated spans; a fresh + // root avoids inheriting a stale ambient parent at creation. + // - detached(): strips the thread-local Scope so the guard does not sit on + // the worker's context stack across the yield (which would adopt other + // spans run on that thread) and is not popped on the wrong thread when + // the coroutine resumes elsewhere. The span still ends when this frame + // unwinds after resume. + auto span = SpanGuard::rootSpan( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request) + .detached(); + // Addresses are hashed before emission for privacy. + if (auto const& src = context.params[jss::source_account]; src.isString()) + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + if (auto const& dst = context.params[jss::destination_account]; dst.isString()) + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); + if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/shamap/NodeFamily.cpp b/src/xrpld/shamap/NodeFamily.cpp index 2e48117d6a9..77e23d54c92 100644 --- a/src/xrpld/shamap/NodeFamily.cpp +++ b/src/xrpld/shamap/NodeFamily.cpp @@ -27,7 +27,7 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm) , j_(app.getJournal("NodeFamily")) , fbCache_( std::make_shared( - "Node family full below cache", + "Node_family_full_below_cache", stopwatch(), app.getJournal("NodeFamilyFulLBelowCache"), cm.collector(), @@ -35,7 +35,7 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm) kFullBelowExpiration)) , tnCache_( std::make_shared( - "Node family tree node cache", + "Node_family_tree_node_cache", app.config().getValueFor(SizedItem::TreeCacheSize), std::chrono::seconds(app.config().getValueFor(SizedItem::TreeCacheAge)), stopwatch(), diff --git a/src/xrpld/telemetry/ConsensusReceiveTracing.h b/src/xrpld/telemetry/ConsensusReceiveTracing.h new file mode 100644 index 00000000000..83e6db0774a --- /dev/null +++ b/src/xrpld/telemetry/ConsensusReceiveTracing.h @@ -0,0 +1,130 @@ +#pragma once + +/** + * Helper functions for creating consensus receive trace spans. + * + * Encapsulates the logic for creating SpanGuard instances for incoming + * proposal and validation messages with optional protobuf parent + * extraction. When the incoming message carries a TraceContext with a + * valid span_id, the receive span is created as a child of the + * sender's span, enabling cross-node trace correlation. + * + * Dependency diagram: + * + * protocol::TMProposeSet / TMValidation + * | + * v + * proposalReceiveSpan() / validationReceiveSpan() + * | + * +--- has trace_context? ----+ + * | yes | no + * v v + * SpanGuard::hashSpan() with SpanGuard::rootSpan() + * extracted parent context (fresh trace root) + * + * When XRPL_ENABLE_TELEMETRY is not defined, the functions return + * no-op SpanGuard instances (zero overhead, zero dependencies). + * + * Usage: + * @code + * // In PeerImp::onMessage(TMProposeSet): + * auto span = telemetry::proposalReceiveSpan(*m); + * span.setAttribute(...); + * @endcode + * + * @note Span names come from the canonical constants in + * ConsensusSpanNames.h (consensus::span::proposalReceive / + * validationReceive) so they stay in sync with the rest of Phase 4. + */ + +#include + +#include +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +/** + * Create a "consensus.proposal.receive" span for an incoming proposal. + * + * If the message carries a TraceContext with a valid span_id, the + * receive span is created with the sender's context as parent. + * Otherwise a standalone span is created. + * + * @param msg The incoming TMProposeSet protobuf message. + * @return An active SpanGuard, or a null guard if tracing is disabled. + */ +inline SpanGuard +proposalReceiveSpan([[maybe_unused]] protocol::TMProposeSet const& msg) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (msg.has_trace_context()) + { + auto const& tc = msg.trace_context(); + // Reject malformed or all-zero ids from the peer before trusting + // them as a parent. See TraceContextValidation.h. + if (isValidTraceContext(tc)) + { + // Create a child span using the sender's trace_id and + // span_id as parent. Use hashSpan with the sender's + // trace_id so the receiving span shares the same trace. + return SpanGuard::hashSpan( + TraceCategory::Consensus, + consensus::span::proposalReceive, + reinterpret_cast(tc.trace_id().data()), + tc.trace_id().size(), + reinterpret_cast(tc.span_id().data()), + tc.span_id().size(), + tc.has_trace_flags() ? static_cast(tc.trace_flags()) + : std::uint8_t{0}); + } + } +#endif + // No propagated context — start a fresh trace root (not an ambient child). + return SpanGuard::rootSpan( + TraceCategory::Consensus, seg::consensus, consensus::span::op::proposalReceive); +} + +/** + * Create a "consensus.validation.receive" span for an incoming validation. + * + * If the message carries a TraceContext with a valid span_id, the + * receive span is created with the sender's context as parent. + * Otherwise a standalone span is created. + * + * @param msg The incoming TMValidation protobuf message. + * @return An active SpanGuard, or a null guard if tracing is disabled. + */ +inline SpanGuard +validationReceiveSpan([[maybe_unused]] protocol::TMValidation const& msg) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (msg.has_trace_context()) + { + auto const& tc = msg.trace_context(); + // Reject malformed or all-zero ids from the peer before trusting + // them as a parent. See TraceContextValidation.h. + if (isValidTraceContext(tc)) + { + return SpanGuard::hashSpan( + TraceCategory::Consensus, + consensus::span::validationReceive, + reinterpret_cast(tc.trace_id().data()), + tc.trace_id().size(), + reinterpret_cast(tc.span_id().data()), + tc.span_id().size(), + tc.has_trace_flags() ? static_cast(tc.trace_flags()) + : std::uint8_t{0}); + } + } +#endif + // No propagated context — start a fresh trace root (not an ambient child). + return SpanGuard::rootSpan( + TraceCategory::Consensus, seg::consensus, consensus::span::op::validationReceive); +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp new file mode 100644 index 00000000000..4f31e890435 --- /dev/null +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -0,0 +1,1514 @@ +/** + * MetricsRegistry implementation — OpenTelemetry metric instruments for xrpld. + * + * This file contains: + * - Construction / destruction logic for the OTel MeterProvider pipeline. + * - Synchronous instrument creation (counters, histograms) for RPC, job + * queue, and NodeStore I/O metrics. + * - Observable gauge callback registration for cache hit rates, TxQ state, + * CountedObject instances, load factors, and NodeStore queue depth. + * - No-op stubs when XRPL_ENABLE_TELEMETRY is not defined. + */ + +// On Windows, OTel's spin_lock_mutex.h (transitively included from +// MetricsRegistry.h) defines _WINSOCKAPI_ and includes . +// This poisons the include state for boost/asio/detail/socket_types.hpp, +// which requires winsock2.h to be included first. Pre-including the +// boost/asio socket types header gets winsock2.h in before the OTel +// headers can interfere. +#ifdef _MSC_VER +#include +#endif + +#include + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace metric_sdk = opentelemetry::sdk::metrics; +namespace otlp_http = opentelemetry::exporter::otlp; +namespace resource = opentelemetry::sdk::resource; + +namespace { + +// Microsecond-valued duration histogram instrument names. Each is +// referenced twice — once to register the explicit-bucket view and once +// to create the instrument — so they are named constants to keep the two +// sites in sync (a mismatch would silently drop the bucket override). +constexpr char kJobQueuedDurationUs[] = "job_queued_us"; +constexpr char kJobRunningDurationUs[] = "job_running_us"; +constexpr char kRpcMethodDurationUs[] = "rpc_method_us"; + +/** + * Register an explicit-bucket histogram view for a microsecond-valued + * instrument. + * + * The SDK's default histogram buckets top out at 10,000 (10 ms when the + * values are microseconds), so any duration above 10 ms saturates and + * every quantile reads as 10 ms. Job wait/run times and RPC latencies + * routinely exceed that, so we install boundaries spanning 100 µs to + * 60 s to capture the real distribution. + * + * @param views The registry to add the view to. + * @param name Instrument name to match (e.g. "job_running_us"). + */ +void +addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) +{ + // Boundaries in microseconds: 100µs, 500µs, 1ms, 5ms, 10ms, 25ms, 50ms, + // 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s, 30s, 60s. Covers sub-millisecond + // jobs through multi-second stalls without saturating. + auto config = std::make_shared(); + config->boundaries_ = { + 100.0, + 500.0, + 1'000.0, + 5'000.0, + 10'000.0, + 25'000.0, + 50'000.0, + 100'000.0, + 250'000.0, + 500'000.0, + 1'000'000.0, + 2'500'000.0, + 5'000'000.0, + 10'000'000.0, + 30'000'000.0, + 60'000'000.0}; + + auto selector = metric_sdk::InstrumentSelectorFactory::Create( + metric_sdk::InstrumentType::kHistogram, name, ""); + auto meterSelector = metric_sdk::MeterSelectorFactory::Create("xrpld", "1.0.0", ""); + auto view = + metric_sdk::ViewFactory::Create(name, "", metric_sdk::AggregationType::kHistogram, config); + + views.AddView(std::move(selector), std::move(meterSelector), std::move(view)); +} + +} // namespace + +#endif // XRPL_ENABLE_TELEMETRY + +namespace xrpl::telemetry { + +MetricsRegistry::MetricsRegistry( + [[maybe_unused]] bool enabled, + [[maybe_unused]] ServiceRegistry& app, + [[maybe_unused]] beast::Journal journal) + : enabled_(enabled) +#ifdef XRPL_ENABLE_TELEMETRY + , app_(app) + , journal_(journal) +#endif +{ +} + +MetricsRegistry::~MetricsRegistry() +{ + stop(); +} + +void +MetricsRegistry::start(std::string const& endpoint, std::string const& instanceId) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_) + return; + + JLOG(journal_.info()) << "MetricsRegistry: starting, endpoint=" << endpoint + << ", instanceId=" << instanceId; + + initExporterAndProvider(endpoint, instanceId); + initSyncInstruments(); + registerAsyncGauges(); + + JLOG(journal_.info()) << "MetricsRegistry: started successfully"; +#else + (void)endpoint; + (void)instanceId; + (void)enabled_; +#endif // XRPL_ENABLE_TELEMETRY +} + +#ifdef XRPL_ENABLE_TELEMETRY +void +MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::string const& instanceId) +{ + // Configure OTLP/HTTP metric exporter. + otlp_http::OtlpHttpMetricExporterOptions exporterOpts; + exporterOpts.url = endpoint; + auto exporter = otlp_http::OtlpHttpMetricExporterFactory::Create(exporterOpts); + + // Configure periodic reader with 10-second export interval. + metric_sdk::PeriodicExportingMetricReaderOptions readerOpts; + readerOpts.export_interval_millis = std::chrono::milliseconds(10000); + readerOpts.export_timeout_millis = std::chrono::milliseconds(5000); + auto reader = + metric_sdk::PeriodicExportingMetricReaderFactory::Create(std::move(exporter), readerOpts); + + // Configure resource attributes so Prometheus service_instance_id labels + // distinguish metrics from different nodes (matches OTelCollector setup). + resource::ResourceAttributes attrs; + // Use std::string, not a string literal: ResourceAttributes stores an + // OTel AttributeValue variant whose char-const* overload binds to bool, + // so "xrpld" would be recorded as the boolean true. std::string selects + // the string alternative and the value round-trips as service.name=xrpld. + attrs[opentelemetry::semconv::service::kServiceName] = std::string("xrpld"); + if (!instanceId.empty()) + attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId; + auto resourceAttrs = resource::Resource::Create(attrs); + + // Build a view registry with explicit microsecond buckets for the + // duration histograms. Without this they use the SDK default buckets + // (max 10,000 = 10 ms), saturating every quantile at 10 ms. + auto views = std::make_unique(); + addMicrosecondHistogramView(*views, kJobQueuedDurationUs); + addMicrosecondHistogramView(*views, kJobRunningDurationUs); + addMicrosecondHistogramView(*views, kRpcMethodDurationUs); + + // Create MeterProvider with resource, then attach the metric reader. + provider_ = metric_sdk::MeterProviderFactory::Create(std::move(views), resourceAttrs); + provider_->AddMetricReader(std::move(reader)); + + // Get a meter for all xrpld instruments. + meter_ = provider_->GetMeter("xrpld", "1.0.0"); +} + +void +MetricsRegistry::initSyncInstruments() +{ + // RPC per-method counters and histogram. + rpcStartedCounter_ = + meter_->CreateUInt64Counter("rpc_method_started_total", "Total RPC method calls started"); + rpcFinishedCounter_ = meter_->CreateUInt64Counter( + "rpc_method_finished_total", "Total RPC method calls completed successfully"); + rpcErroredCounter_ = meter_->CreateUInt64Counter( + "rpc_method_errored_total", "Total RPC method calls that errored"); + rpcDurationHistogram_ = meter_->CreateDoubleHistogram( + kRpcMethodDurationUs, "RPC method execution time in microseconds"); + + // Job queue per-type counters and histograms. + jobQueuedCounter_ = meter_->CreateUInt64Counter("job_queued_total", "Total jobs enqueued"); + jobStartedCounter_ = meter_->CreateUInt64Counter("job_started_total", "Total jobs started"); + jobFinishedCounter_ = meter_->CreateUInt64Counter("job_finished_total", "Total jobs completed"); + jobQueuedDurationHistogram_ = meter_->CreateDoubleHistogram( + kJobQueuedDurationUs, "Time jobs spent waiting in the queue (microseconds)"); + jobRunningDurationHistogram_ = + meter_->CreateDoubleHistogram(kJobRunningDurationUs, "Job execution time in microseconds"); + + // --- External dashboard parity counters (Task 7.14) --- + ledgersClosedCounter_ = + meter_->CreateUInt64Counter("ledgers_closed_total", "Total ledgers closed by consensus"); + validationsSentCounter_ = meter_->CreateUInt64Counter( + "validations_sent_total", "Total validations sent by this node"); + validationsCheckedCounter_ = meter_->CreateUInt64Counter( + "validations_checked_total", "Total network validations received and checked"); + stateChangesCounter_ = + meter_->CreateUInt64Counter("state_changes_total", "Total operating mode changes"); + // jq_trans_overflow_total is observed from Overlay's existing cumulative + // atomic (Overlay::getJqTransOverflow()) rather than pushed. The overlay + // owns the only increment site (PeerImp), so an ObservableCounter reads the + // live total each collection cycle without threading a push path through + // develop-owned overlay code. + jqTransOverflowObservable_ = meter_->CreateInt64ObservableCounter( + "jq_trans_overflow_total", "Total job queue transaction overflows"); + jqTransOverflowObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->app_.getOverlay().getJqTransOverflow())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); + ledgerHistoryMismatchCounter_ = meter_->CreateUInt64Counter( + "ledger_history_mismatch_total", "Total built-vs-validated ledger mismatches by reason"); + txqExpiredCounter_ = meter_->CreateUInt64Counter( + "txq_expired_total", "Total transactions expired out of the transaction queue"); + txqDroppedCounter_ = meter_->CreateUInt64Counter( + "txq_dropped_total", "Total transactions refused admission to the queue by reason"); + // Note: validation_agreements_total / validation_missed_total are monotonic + // ObservableCounters created in registerValidationTotalsCounters() (below). +} +#endif // XRPL_ENABLE_TELEMETRY + +void +MetricsRegistry::detachCallbacks() noexcept +{ +#ifdef XRPL_ENABLE_TELEMETRY + // Release so every subsequent callback acquire-load sees true. + callbacksDetached_.store(true, std::memory_order_release); +#endif // XRPL_ENABLE_TELEMETRY +} + +void +MetricsRegistry::stop() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!provider_) + return; + + JLOG(journal_.info()) << "MetricsRegistry: stopping"; + + // Belt-and-suspenders: detachCallbacks() should have already been + // called by Application shutdown before any service the callbacks + // observe was stopped. Setting the flag here is redundant for a + // correct caller but protects against a future caller that forgets + // to detach first. + callbacksDetached_.store(true, std::memory_order_release); + + // SDK teardown order: Shutdown() stops the PeriodicExportingMetricReader + // thread (so no further gauge callbacks fire) and performs the final + // collect-and-export drain itself. The trailing ForceFlush() is a + // redundant safety net (a no-op once the reader is shut down), then + // reset() destroys the provider. + provider_->Shutdown(); + provider_->ForceFlush(); + provider_.reset(); + + JLOG(journal_.info()) << "MetricsRegistry: stopped"; +#endif // XRPL_ENABLE_TELEMETRY +} + +// ----------------------------------------------------------------- +// Synchronous instrument recording — RPC metrics (Task 9.4) +// ----------------------------------------------------------------- + +void +MetricsRegistry::recordRpcStarted(std::string_view method) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !rpcStartedCounter_) + return; + rpcStartedCounter_->Add(1, {{"method", std::string(method)}}); +#else + (void)method; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordRpcFinished(std::string_view method, std::int64_t durationUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !rpcFinishedCounter_) + return; + rpcFinishedCounter_->Add(1, {{"method", std::string(method)}}); + if (rpcDurationHistogram_) + { + rpcDurationHistogram_->Record( + static_cast(durationUs), + {{"method", std::string(method)}}, + opentelemetry::context::Context{}); + } +#else + (void)method; + (void)durationUs; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordRpcErrored(std::string_view method, std::int64_t durationUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !rpcErroredCounter_) + return; + rpcErroredCounter_->Add(1, {{"method", std::string(method)}}); + if (rpcDurationHistogram_) + { + rpcDurationHistogram_->Record( + static_cast(durationUs), + {{"method", std::string(method)}}, + opentelemetry::context::Context{}); + } +#else + (void)method; + (void)durationUs; + (void)enabled_; +#endif +} + +// ----------------------------------------------------------------- +// Synchronous instrument recording — Job Queue metrics (Task 9.5) +// ----------------------------------------------------------------- + +void +MetricsRegistry::recordJobQueued(std::string_view jobType) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !jobQueuedCounter_) + return; + jobQueuedCounter_->Add(1, {{"job_type", std::string(jobType)}}); +#else + (void)jobType; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordJobStarted(std::string_view jobType, std::int64_t queuedDurUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !jobStartedCounter_) + return; + jobStartedCounter_->Add(1, {{"job_type", std::string(jobType)}}); + if (jobQueuedDurationHistogram_ && queuedDurUs >= 0) + { + // Guard against negative queued durations: the caller derives this + // from a steady-clock delta that can go slightly negative under clock + // skew or reordering. The OTel SDK rejects negative histogram values + // (logging a warning per call), so skip them rather than spam. + jobQueuedDurationHistogram_->Record( + static_cast(queuedDurUs), + {{"job_type", std::string(jobType)}}, + opentelemetry::context::Context{}); + } +#else + (void)jobType; + (void)queuedDurUs; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordJobFinished(std::string_view jobType, std::int64_t runningDurUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !jobFinishedCounter_) + return; + jobFinishedCounter_->Add(1, {{"job_type", std::string(jobType)}}); + if (jobRunningDurationHistogram_) + { + jobRunningDurationHistogram_->Record( + static_cast(runningDurUs), + {{"job_type", std::string(jobType)}}, + opentelemetry::context::Context{}); + } +#else + (void)jobType; + (void)runningDurUs; + (void)enabled_; +#endif +} + +// ----------------------------------------------------------------- +// Observable gauge callbacks (Tasks 9.1, 9.2, 9.3, 9.6, 9.7) +// ----------------------------------------------------------------- + +#ifdef XRPL_ENABLE_TELEMETRY + +void +MetricsRegistry::registerAsyncGauges() +{ + // Each helper creates one observable instrument and attaches one + // callback. Keeping the registration bodies in separate methods + // preserves the 80-line-per-function limit enforced by CLAUDE.md. + registerCacheHitRateGauge(); + registerTxqGauge(); + registerObjectCountGauge(); + registerLoadFactorGauge(); + registerNodeStoreGauge(); + registerServerInfoGauge(); + registerBuildInfoGauge(); + registerCompleteLedgersGauge(); + registerDbMetricsGauge(); + registerValidatorHealthGauge(); + registerPeerQualityGauge(); + registerReduceRelayGauge(); + registerLedgerEconomyGauge(); + registerStateTrackingGauge(); + registerStorageDetailGauge(); + registerValidationAgreementGauge(); + registerValidationTotalsCounters(); +} + +void +MetricsRegistry::registerCacheHitRateGauge() +{ + // --- Task 9.2: Cache hit rate and size gauges --- + cacheHitRateGauge_ = + meter_->CreateDoubleObservableGauge("cache_metrics", "Cache hit rates and sizes"); + cacheHitRateGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + // SLE cache hit rate (0.0 - 1.0). + auto sleRate = app.getCachedSLEs().rate(); + opentelemetry::nostd::get>>(result) + ->Observe(sleRate, {{"metric", "SLE_hit_rate"}}); + + // Ledger cache hit rate. + // TaggedCache::getHitRate() returns 0-100; normalize to + // 0.0-1.0 so the Grafana panel using "percentunit" renders + // correctly. + auto ledgerRate = app.getLedgerMaster().getCacheHitRate() / 100.0; + opentelemetry::nostd::get>>(result) + ->Observe(ledgerRate, {{"metric", "ledger_hit_rate"}}); + + // AcceptedLedger cache hit rate (also 0-100 from + // TaggedCache; normalize to 0.0-1.0). + auto alRate = app.getAcceptedLedgerCache().getHitRate() / 100.0; + opentelemetry::nostd::get>>(result) + ->Observe(alRate, {{"metric", "AL_hit_rate"}}); + + // TreeNode cache size. + auto tnCacheSize = app.getNodeFamily().getTreeNodeCache()->getCacheSize(); + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(tnCacheSize), {{"metric", "treenode_cache_size"}}); + + // TreeNode track size. + auto tnTrackSize = app.getNodeFamily().getTreeNodeCache()->getTrackSize(); + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(tnTrackSize), {{"metric", "treenode_track_size"}}); + + // FullBelow cache size. + auto fbSize = app.getNodeFamily().getFullBelowCache()->size(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(fbSize), {{"metric", "fullbelow_size"}}); + + // AcceptedLedger cache size (entry count). + auto alSize = app.getAcceptedLedgerCache().size(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(alSize), {{"metric", "AL_size"}}); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerTxqGauge() +{ + // --- Task 9.3: TxQ metrics gauges --- + txqGauge_ = meter_->CreateDoubleObservableGauge("txq_metrics", "Transaction queue metrics"); + txqGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto const metrics = app.getTxQ().getMetrics(*app.getOpenLedger().current()); + + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + observe("txq_count", static_cast(metrics.txCount)); + observe( + "txq_max_size", + metrics.txQMaxSize ? static_cast(*metrics.txQMaxSize) : 0.0); + observe("txq_in_ledger", static_cast(metrics.txInLedger)); + observe("txq_per_ledger", static_cast(metrics.txPerLedger)); + observe( + "txq_reference_fee_level", + static_cast(metrics.referenceFeeLevel.fee())); + observe( + "txq_min_processing_fee_level", + static_cast(metrics.minProcessingFeeLevel.fee())); + observe("txq_med_fee_level", static_cast(metrics.medFeeLevel.fee())); + observe( + "txq_open_ledger_fee_level", + static_cast(metrics.openLedgerFeeLevel.fee())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if TxQ or OpenLedger are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerObjectCountGauge() +{ + // --- Task 9.6: Counted object instance gauges --- + objectCountGauge_ = meter_->CreateInt64ObservableGauge( + "object_count", "Live instance counts for key internal object types"); + objectCountGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + // Iterate through all CountedObject types via the linked + // list in CountedObjects. We report all types with count + // > 0, filtering to the key types of interest. + auto counts = CountedObjects::getInstance().getCounts(0); + for (auto const& [name, count] : counts) + { + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(count), {{"type", name}}); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerLoadFactorGauge() +{ + // --- Task 9.7: Load factor breakdown gauges --- + loadFactorGauge_ = + meter_->CreateDoubleObservableGauge("load_factor_metrics", "Fee load factor breakdown"); + loadFactorGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto& feeTrack = app.getFeeTrack(); + auto const loadBase = static_cast(feeTrack.getLoadBase()); + + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Combined load factor (server component). + observe( + "load_factor_server", static_cast(feeTrack.getLoadFactor()) / loadBase); + + // Individual factor components. + observe( + "load_factor_local", static_cast(feeTrack.getLocalFee()) / loadBase); + observe("load_factor_net", static_cast(feeTrack.getRemoteFee()) / loadBase); + observe( + "load_factor_cluster", + static_cast(feeTrack.getClusterFee()) / loadBase); + + // Fee escalation factors from TxQ. + auto const metrics = app.getTxQ().getMetrics(*app.getOpenLedger().current()); + auto refLevel = static_cast(metrics.referenceFeeLevel.fee()); + if (refLevel > 0) + { + observe( + "load_factor_fee_escalation", + static_cast(metrics.openLedgerFeeLevel.fee()) / refLevel); + observe( + "load_factor_fee_queue", + static_cast(metrics.minProcessingFeeLevel.fee()) / refLevel); + } + + // Combined load factor (max of server and fee escalation). + auto const loadFactorServer = feeTrack.getLoadFactor(); + auto const loadBaseServer = feeTrack.getLoadBase(); + double combined = static_cast(loadFactorServer) / loadBase; + if (refLevel > 0) + { + double const feeEscalation = + static_cast(metrics.openLedgerFeeLevel.fee()) * loadBaseServer / + refLevel; + if (feeEscalation > static_cast(loadFactorServer)) + { + combined = feeEscalation / loadBase; + } + } + observe("load_factor", combined); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerNodeStoreGauge() +{ + // --- Task 9.1: NodeStore I/O gauges --- + // The cumulative counters (reads, writes, bytes) are also exposed here + // as observable gauges. This avoids adding an xrpld dependency into the + // libxrpl nodestore code — the MetricsRegistry reads the existing atomic + // counters from Database via its public accessors. + nodeStoreGauge_ = meter_->CreateInt64ObservableGauge( + "nodestore_state", "NodeStore I/O counters, queue depth, and write load"); + nodeStoreGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto& db = app.getNodeStore(); + + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Cumulative counters (monotonically increasing). + observe("node_reads_total", static_cast(db.getFetchTotalCount())); + observe("node_reads_hit", static_cast(db.getFetchHitCount())); + observe("node_writes", static_cast(db.getStoreCount())); + observe("node_written_bytes", static_cast(db.getStoreSize())); + observe("node_read_bytes", static_cast(db.getFetchSize())); + + // Write load score (instantaneous). + observe("write_load", static_cast(db.getWriteLoad())); + + // Read queue depth (instantaneous). + json::Value obj(json::ValueType::Object); + db.getCountsJson(obj); + if (obj.isMember("read_queue")) + { + observe("read_queue", static_cast(obj["read_queue"].asUInt())); + } + + // Cumulative read duration (stored as JSON string, not int). + if (obj.isMember(jss::node_reads_duration_us)) + { + auto durStr = obj[jss::node_reads_duration_us].asString(); + if (!durStr.empty()) + { + observe("node_reads_duration_us", static_cast(std::stoll(durStr))); + } + } + + // Read thread pool stats (native JSON ints, no jss:: constants). + if (obj.isMember("read_request_bundle")) + { + observe( + "read_request_bundle", + static_cast(obj["read_request_bundle"].asInt())); + } + if (obj.isMember("read_threads_running")) + { + observe( + "read_threads_running", + static_cast(obj["read_threads_running"].asInt())); + } + if (obj.isMember("read_threads_total")) + { + observe( + "read_threads_total", + static_cast(obj["read_threads_total"].asInt())); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerServerInfoGauge() +{ + // --- Task 9.7a: Server info gauges --- + serverInfoGauge_ = + meter_->CreateInt64ObservableGauge("server_info", "Server-level health metrics"); + serverInfoGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Server operating mode (DISCONNECTED=0 .. FULL=4). + observe("server_state", static_cast(app.getOPs().getOperatingMode())); + + // Uptime in seconds since server start. + observe( + "uptime", static_cast(UptimeClock::now().time_since_epoch().count())); + + // Total peer count (inbound + outbound). + observe("peers", static_cast(app.getOverlay().size())); + + // Validated ledger sequence (0 if none yet). + observe( + "validated_ledger_seq", + static_cast(app.getLedgerMaster().getValidLedgerIndex())); + + // Current open ledger sequence. + observe( + "ledger_current_index", + static_cast(app.getLedgerMaster().getCurrentLedgerIndex())); + + // Cumulative resource-related peer disconnects. + observe( + "peer_disconnects_resources", + static_cast(app.getOverlay().getPeerDisconnectCharges())); + + // Last consensus round data (from JSON — only public API). + auto const consensusInfo = app.getOPs().getConsensusInfo(); + if (consensusInfo.isMember("previous_proposers")) + { + observe( + "last_close_proposers", + static_cast(consensusInfo["previous_proposers"].asUInt())); + } + if (consensusInfo.isMember("previous_mseconds")) + { + observe( + "last_close_converge_time_ms", + static_cast(consensusInfo["previous_mseconds"].asUInt())); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerBuildInfoGauge() +{ + // --- Task 9.7b: Build info gauge --- + buildInfoGauge_ = meter_->CreateInt64ObservableGauge("build_info", "Build version information"); + buildInfoGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* /* state */) { + try + { + opentelemetry::nostd::get>>(result) + ->Observe(1, {{"version", std::string(BuildInfo::getVersionString())}}); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + }, + nullptr); +} + +void +MetricsRegistry::registerCompleteLedgersGauge() +{ + // --- Task 9.7c: Complete ledgers range gauge --- + completeLedgersGauge_ = meter_->CreateInt64ObservableGauge( + "complete_ledgers", "Complete ledger range start/end pairs"); + completeLedgersGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto const rangeStr = app.getLedgerMaster().getCompleteLedgers(); + if (rangeStr.empty() || rangeStr == "empty") + return; + + // Parse comma-separated ranges like + // "32570-50000,50005-75891421". + std::size_t rangeIndex = 0; + std::istringstream stream(rangeStr); + std::string segment; + while (std::getline(stream, segment, ',')) + { + auto const dashPos = segment.find('-'); + if (dashPos == std::string::npos || dashPos == 0 || + dashPos == segment.size() - 1) + continue; + + auto const startStr = segment.substr(0, dashPos); + auto const endStr = segment.substr(dashPos + 1); + + auto const idxStr = std::to_string(rangeIndex); + + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(std::stoll(startStr)), + {{"bound", "start"}, {"index", idxStr}}); + + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(std::stoll(endStr)), + {{"bound", "end"}, {"index", idxStr}}); + + ++rangeIndex; + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on parse error or if services not ready. + } + }, + this); +} + +void +MetricsRegistry::registerDbMetricsGauge() +{ + // --- Task 9.7d: Database size and fetch rate gauges --- + dbMetricsGauge_ = + meter_->CreateInt64ObservableGauge("db_metrics", "Database storage sizes and fetch rates"); + dbMetricsGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + auto& rdb = app.getRelationalDatabase(); + observe("db_kb_total", static_cast(rdb.getKBUsedAll())); + observe("db_kb_ledger", static_cast(rdb.getKBUsedLedger())); + observe("db_kb_transaction", static_cast(rdb.getKBUsedTransaction())); + + // Historical ledger fetches per minute. + observe( + "historical_perminute", + static_cast(app.getInboundLedgers().fetchRate())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerValidatorHealthGauge() +{ + // --- Task 7.9: Validator health gauges --- + validatorHealthGauge_ = + meter_->CreateDoubleObservableGauge("validator_health", "Validator health indicators"); + validatorHealthGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + observe("amendment_blocked", app.getOPs().isAmendmentBlocked() ? 1.0 : 0.0); + observe("unl_blocked", app.getOPs().isUNLBlocked() ? 1.0 : 0.0); + observe("validation_quorum", static_cast(app.getValidators().quorum())); + + // Days until UNL list expiry (-1 if no expiry known). + auto const expiry = app.getValidators().expires(); + if (expiry) + { + auto const now = app.getTimeKeeper().closeTime(); + auto const diffHours = + std::chrono::duration_cast(*expiry - now).count(); + observe("unl_expiry_days", static_cast(diffHours) / 24.0); + } + else + { + observe("unl_expiry_days", -1.0); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerPeerQualityGauge() +{ + // --- Task 7.10: Peer quality gauges --- + // Uses Peer::json() to read latency and version since those accessors + // are not on the abstract Peer interface (they live on PeerImp). + peerQualityGauge_ = + meter_->CreateDoubleObservableGauge("peer_quality", "Peer network quality metrics"); + peerQualityGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Collect latencies, version info, and tracking state from + // each peer's JSON. + std::vector latencies; + int higherVersionCount = 0; + int totalPeers = 0; + int divergedCount = 0; + + // Encode a version string into BuildInfo's comparable numeric + // form. Peers report the full "xrpld-3.3.0-b0" string while + // our baseline is the bare "3.3.0-b0", and SemanticVersion + // requires a leading digit, so strip any non-digit prefix + // first. Numeric encoding avoids the lexicographic bug where + // "2.3.0" > "2.10.0" and "xrpld-..." > "3...". + auto const encodeVersion = [](std::string_view v) -> std::uint64_t { + auto const firstDigit = v.find_first_of("0123456789"); + if (firstDigit == std::string_view::npos) + return 0; + return BuildInfo::encodeSoftwareVersion(v.substr(firstDigit)); + }; + auto const ownEncoded = encodeVersion(BuildInfo::getVersionString()); + + app.getOverlay().foreach([&](std::shared_ptr const& peer) { + ++totalPeers; + auto const pj = peer->json(); + if (pj.isMember(jss::latency)) + { + latencies.push_back(pj[jss::latency].asInt()); + } + if (pj.isMember(jss::version)) + { + // Unparseable peer versions encode below ownEncoded, so + // the comparison correctly leaves them uncounted. + auto const pv = pj[jss::version].asString(); + if (encodeVersion(pv) > ownEncoded) + ++higherVersionCount; + } + // PeerImp::json() sets "track" to "diverged" when the peer's + // tracking state is Tracking::Diverged (i.e. it is following + // a different ledger chain than us). + if (pj.isMember(jss::track) && pj[jss::track].asString() == "diverged") + ++divergedCount; + }); + + // P90 latency across connected peers. + if (!latencies.empty()) + { + std::ranges::sort(latencies); + auto p90idx = static_cast(latencies.size() * 0.9); + if (p90idx >= latencies.size()) + p90idx = latencies.size() - 1; + observe("peer_latency_p90_ms", static_cast(latencies[p90idx])); + } + else + { + observe("peer_latency_p90_ms", 0.0); + } + + // Percentage of peers running a higher version. + double const higherPct = totalPeers > 0 + ? (static_cast(higherVersionCount) / totalPeers * 100.0) + : 0.0; + observe("peers_higher_version_pct", higherPct); + + // Count peers diverged from our ledger chain, read from the + // peer's "track" JSON field (set by PeerImp::json()). Diverged + // peers are following a different chain and are a leading + // indicator of local sync trouble. + observe("peers_insane_count", static_cast(divergedCount)); + + // Binary flag: recommend upgrade if >60% run a newer version. + observe("upgrade_recommended", higherPct > 60.0 ? 1.0 : 0.0); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerReduceRelayGauge() +{ + // Transaction reduce-relay efficiency. Overlay::txMetrics() exposes the + // rolling averages as a JSON object with string values (std::to_string), + // so parse each field. A high suppressed:selected ratio proves the + // feature is saving bandwidth; a high not_enabled count means stale peers + // force full relay. + reduceRelayGauge_ = meter_->CreateInt64ObservableGauge( + "reduce_relay_metrics", "Transaction reduce-relay efficiency metrics"); + reduceRelayGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto const tm = app.getOverlay().txMetrics(); + + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Each field is a decimal string; emit when present and parseable. + auto observeField = [&](auto const& field, char const* name) { + if (tm.isMember(field)) + { + auto const s = tm[field].asString(); + if (!s.empty()) + observe(name, static_cast(std::stoll(s))); + } + }; + + observeField(jss::txr_selected_cnt, "selected_peers"); + observeField(jss::txr_suppressed_cnt, "suppressed_peers"); + observeField(jss::txr_not_enabled_cnt, "not_enabled_peers"); + observeField(jss::txr_missing_tx_freq, "missing_tx_freq"); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready or a value is + // not parseable. + } + }, + this); +} + +void +MetricsRegistry::registerLedgerEconomyGauge() +{ + // --- Task 7.11: Ledger economy gauges --- + ledgerEconomyGauge_ = + meter_->CreateDoubleObservableGauge("ledger_economy", "Ledger fee and economy metrics"); + ledgerEconomyGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Local fee (drops). + observe("base_fee_xrp", static_cast(app.getFeeTrack().getLocalFee())); + + // Reserve values from the validated ledger. + auto const ledger = app.getLedgerMaster().getValidatedLedger(); + if (ledger) + { + auto const& fees = ledger->fees(); + // Base reserve = one account, zero owned objects: + // accountReserve(ownerCount=0, accountCount=1) == reserve. + observe( + "reserve_base_xrp", static_cast(fees.accountReserve(0, 1).drops())); + observe("reserve_inc_xrp", static_cast(fees.increment.drops())); + } + + // Seconds since the last validated ledger closed. + auto const age = app.getLedgerMaster().getValidatedLedgerAge(); + observe("ledger_age_seconds", static_cast(age.count())); + + // Transaction rate from the open ledger's tx count. + // OpenView::txCount() tracks transactions in the current + // open ledger; dividing by the ledger age gives an + // approximate rate. + auto const& openLedger = app.getOpenLedger(); + auto const txInLedger = openLedger.current()->txCount(); + auto const ageVal = age.count(); + if (ageVal > 0) + { + observe( + "transaction_rate", + static_cast(txInLedger) / static_cast(ageVal)); + } + else + { + observe("transaction_rate", 0.0); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerStateTrackingGauge() +{ + // --- Task 7.12: State tracking gauges --- + stateTrackingGauge_ = + meter_->CreateDoubleObservableGauge("state_tracking", "Node state and mode tracking"); + stateTrackingGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // State value: 0-4 from OperatingMode, 5=validating, 6=proposing. + auto const mode = app.getOPs().getOperatingMode(); + auto stateValue = static_cast(mode); + + // If FULL, refine using consensus info for validating/proposing. + if (mode == OperatingMode::FULL) + { + auto const info = app.getOPs().getConsensusInfo(); + if (info.isMember("proposing") && info["proposing"].asBool()) + { + stateValue = 6.0; + } + else if (info.isMember("validating") && info["validating"].asBool()) + { + stateValue = 5.0; + } + } + observe("state_value", stateValue); + + // Time spent in the current operating mode, sourced from + // NetworkOPs' StateAccounting via a lightweight accessor. + auto const stateDurUs = app.getOPs().getServerStateDurationUs(); + observe( + "time_in_current_state_seconds", static_cast(stateDurUs.count()) / 1e6); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerStorageDetailGauge() +{ + // --- Task 7.13: Storage detail gauges --- + // Reports NuDB on-disk size via the NodeStore JSON counters interface. + storageDetailGauge_ = + meter_->CreateInt64ObservableGauge("storage_detail", "Storage detail metrics"); + storageDetailGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // NuDB on-disk size reported by the NodeStore backend. + // getStoreSize() returns the total bytes stored. + observe("nudb_bytes", static_cast(app.getNodeStore().getStoreSize())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerValidationAgreementGauge() +{ + // --- Task 7.15: Validation agreement gauges --- + // Reports rolling-window agreement percentages and counts from + // ValidationTracker. reconcile() is called at the start of the + // callback so that pending ledger events are resolved before the + // window data is read (the callback fires every ~10 s from the + // PeriodicExportingMetricReader thread). + validationAgreementGauge_ = meter_->CreateDoubleObservableGauge( + "validation_agreement", "Validation agreement percentages and counts (1h/24h windows)"); + validationAgreementGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + + try + { + // Reconcile pending events before reading window data. + self->validationTracker_.reconcile(); + + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + observe("agreement_pct_1h", self->validationTracker_.agreementPct1h()); + observe("agreement_pct_24h", self->validationTracker_.agreementPct24h()); + observe( + "agreements_1h", static_cast(self->validationTracker_.agreements1h())); + observe("missed_1h", static_cast(self->validationTracker_.missed1h())); + observe( + "agreements_24h", + static_cast(self->validationTracker_.agreements24h())); + observe("missed_24h", static_cast(self->validationTracker_.missed24h())); + + // 7-day window (matches external xrpl-validator-dashboard). + observe("agreement_pct_7d", self->validationTracker_.agreementPct7d()); + observe( + "agreements_7d", static_cast(self->validationTracker_.agreements7d())); + observe("missed_7d", static_cast(self->validationTracker_.missed7d())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerValidationTotalsCounters() +{ + // Lifetime validation agreement/miss counters. + // + // These are monotonic ObservableCounters (not the sync Counters they used + // to be): a Prometheus _total must never decrease, but ValidationTracker's + // NET totals are non-monotonic (a late repair decrements the net miss + // count). We therefore observe the tracker's GROSS lifetime tallies, which + // count each ledger once at first classification and are never adjusted on + // repair (initial-classification semantics — see ValidationTracker). The + // repaired/agreement view remains available from validation_agreement. + // + // reconcile() is called first so pending events are resolved before the + // tallies are read; the callback fires every ~10 s from the + // PeriodicExportingMetricReader thread. + validationAgreementsObservable_ = meter_->CreateInt64ObservableCounter( + "validation_agreements_total", + "Lifetime validations that initially agreed with network consensus"); + validationAgreementsObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + self->validationTracker_.reconcile(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->validationTracker_.totalAgreementsEver())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); + + validationMissedObservable_ = meter_->CreateInt64ObservableCounter( + "validation_missed_total", "Lifetime validations that initially missed network consensus"); + validationMissedObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + self->validationTracker_.reconcile(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->validationTracker_.totalMissedEver())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +#endif // XRPL_ENABLE_TELEMETRY + +// ----------------------------------------------------------------- +// External dashboard parity counter increments (Task 7.14) +// ----------------------------------------------------------------- + +void +MetricsRegistry::incrementLedgersClosed() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && ledgersClosedCounter_) + ledgersClosedCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementValidationsSent() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && validationsSentCounter_) + validationsSentCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementValidationsChecked() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && validationsCheckedCounter_) + validationsCheckedCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementStateChanges() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && stateChangesCounter_) + stateChangesCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementLedgerHistoryMismatch(std::string_view reason) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && ledgerHistoryMismatchCounter_) + ledgerHistoryMismatchCounter_->Add(1, {{"reason", std::string(reason)}}); +#endif +} + +void +MetricsRegistry::incrementTxqExpired() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && txqExpiredCounter_) + txqExpiredCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementTxqDropped(std::string_view reason) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && txqDroppedCounter_) + txqDroppedCounter_->Add(1, {{"reason", std::string(reason)}}); +#endif +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h new file mode 100644 index 00000000000..85aed97e2cd --- /dev/null +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -0,0 +1,715 @@ +#pragma once + +/** + * Central OTel Metrics Registry for xrpld. + * + * Owns all OpenTelemetry metric instruments (counters, histograms, + * observable gauges) that are NOT already covered by the beast::insight + * StatsD pipeline. The instruments are created once at startup and polled + * by the OTel PeriodicExportingMetricReader at a configurable interval + * (default 10 s). + * + * When XRPL_ENABLE_TELEMETRY is **not** defined, this class compiles to a + * lightweight no-op: every public method is an empty inline. + * + * Dependency / ownership diagram (ASCII): + * + * Application + * | + * +-- MetricsRegistry (unique_ptr, created in setup(), started/stopped with telemetry) + * | + * +-- OTel MeterProvider (owns reader + exporter) + * | | + * | +-- PeriodicExportingMetricReader + * | +-- OtlpHttpMetricExporter + * | + * +-- Counters / Histograms (synchronous instruments) + * | +-- rpc_method_started_total + * | +-- rpc_method_finished_total + * | +-- rpc_method_errored_total + * | +-- rpc_method_us (Histogram) + * | +-- job_queued_total + * | +-- job_started_total + * | +-- job_finished_total + * | +-- job_queued_us (Histogram) + * | +-- job_running_us (Histogram) + * | +-- ledgers_closed_total + * | +-- validations_sent_total + * | +-- validations_checked_total + * | +-- state_changes_total + * | +-- ledger_history_mismatch_total{reason} + * | +-- txq_expired_total + * | +-- txq_dropped_total{reason} + * | + * +-- ValidationTracker (validation agreement tracker) + * | + * +-- Observable Gauges (async callbacks, polled by reader) + * +-- Cache hit rates (SLE, ledger, AL) + * +-- TreeNode / FullBelow sizes + * +-- TxQ metrics + * +-- CountedObject counts + * +-- Load factor breakdown + * +-- NodeStore I/O gauges + * +-- Server info (state, uptime, peers, consensus) + * +-- Build info (version label) + * +-- Complete ledger ranges (start/end pairs) + * +-- DB metrics (storage KB, fetch rate) + * +-- Validator health (amend blocked, UNL, quorum) + * +-- Peer quality (P90 latency, version spread) + * +-- Reduce-relay efficiency (selected/suppressed peers) + * +-- Ledger economy (fees, reserves, age) + * +-- State tracking (mode value, time in state) + * +-- Storage detail (NuDB sizes) + * +-- Validation agreement (1h/24h pct, counts) + * +-- jq_trans_overflow_total (observed from Overlay) + * + * Control-flow for async gauges: + * + * PeriodicExportingMetricReader (background thread, 10 s tick) + * | + * v + * OTel SDK invokes registered ObservableGauge callbacks + * | + * v + * Each callback reads current value from Application services + * (e.g. app.getTxQ().getMetrics(), app.getFeeTrack().getLoadFactor()) + * | + * v + * Result set is exported via OTLP/HTTP to the collector + * + * Control-flow for synchronous instruments: + * + * PerfLogImp::rpcStart/rpcEnd/jobQueue/jobStart/jobFinish + * | + * v + * MetricsRegistry::recordRpc*(method, ...) / recordJob*(type, ...) + * | + * v + * OTel Counter::Add() or Histogram::Record() + * | + * v + * Periodically flushed by the MetricReader + * + * Example usage: + * + * @code + * // In Application::setup(), after telemetry_ is created: + * metricsRegistry_ = std::make_unique( + * telemetry_->isEnabled(), app, journal); + * metricsRegistry_->start(setup.exporterEndpoint); + * + * // In PerfLogImp::rpcStart(): + * if (auto* mr = app_.getMetricsRegistry()) + * mr->recordRpcStarted("server_info"); + * + * // In PerfLogImp::rpcEnd(): + * if (auto* mr = app_.getMetricsRegistry()) + * { + * mr->recordRpcFinished("server_info", durationUs); + * // or: mr->recordRpcErrored("server_info", durationUs); + * } + * + * // In PerfLogImp::jobQueue(): + * if (auto* mr = app_.getMetricsRegistry()) + * mr->recordJobQueued("ledgerData"); + * + * // Shutdown: + * metricsRegistry_->stop(); + * @endcode + * + * Caveats: + * - The MetricsRegistry must be created AFTER the Telemetry object because + * it reads isEnabled() to decide whether to initialize the OTel SDK. + * - Observable gauge callbacks capture a reference to the Application; the + * Application must outlive the MetricsRegistry (guaranteed because + * MetricsRegistry is stopped before Application teardown). + * - If a new CountedObject type is added, it will NOT appear automatically + * in the object_count gauge; the callback iterates a fixed list. + * - Adding a new synchronous instrument requires updating both the header + * and the .cpp, then calling the new record*() method from the + * instrumentation site. + */ + +#include + +#include + +#include +#include +#include +#include +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#endif + +namespace xrpl { + +class ServiceRegistry; + +namespace telemetry { + +/** + * Central OpenTelemetry metric registry. + * + * Owns all OTel instruments (counters, histograms, observable gauges) + * that are not covered by the beast::insight StatsD pipeline. See the + * file-level header comment above for the full dependency diagram, + * gauge domain list, and usage examples. + * + * Class / collaborator diagram (ASCII): + * + * +-----------------+ +-------------------+ + * | Application |------->| MetricsRegistry | + * +-----------------+ +-------------------+ + * | | | + * creates/owns v v v + * +-----------+ +---------+ +-------------------+ + * | Meter | | Counter | | ValidationTracker | + * | Provider | | /Hist. | | (rolling windows) | + * +-----------+ +---------+ +-------------------+ + * | + * v + * Periodic reader thread (~10 s) + * -> ObservableGauge callbacks + * -> OTLP/HTTP export + * + * @note Thread safety: + * - The recordRpc, recordJob, and increment methods are invoked + * from xrpld hot paths. OTel Counter::Add() and + * Histogram::Record() are documented thread-safe, and + * null-guard checks protect uninitialized instruments. + * - ObservableGauge callbacks run on the OTel SDK background + * reader thread (~10 s tick), concurrently with writers. + * Each callback reads only lock-protected or atomic state + * from Application services and wraps the body in a + * catch-all try block so a transient failure never crashes + * the reader thread. + * - ValidationTracker protects its rolling windows internally. + * - start() and stop() are NOT thread-safe with each other and + * must be called from the single Application lifecycle + * thread. + * + * @note Lifetime: + * - Must be constructed AFTER telemetry_ (reads isEnabled()). + * - Must be stopped BEFORE Application services it observes are + * destroyed; the Application owns it via unique_ptr so normal + * teardown guarantees this. + * + * @note Extending: + * - Adding a new CountedObject type is auto-picked up by the + * object_count gauge via iteration. + * - Adding a new synchronous instrument requires a header field, + * an initializer in start(), and a record method. + */ +class MetricsRegistry +{ +public: + /** + * Construct a MetricsRegistry. + * + * @param enabled Whether OTel metric export is active. When false, + * all methods become no-ops. + * @param app Reference to the ServiceRegistry (Application) for + * reading current metric values in gauge callbacks. + * @param journal Journal for log output. + */ + MetricsRegistry(bool enabled, ServiceRegistry& app, beast::Journal journal); + + ~MetricsRegistry(); + + /** + * Non-copyable, non-movable. + */ + MetricsRegistry(MetricsRegistry const&) = delete; + MetricsRegistry& + operator=(MetricsRegistry const&) = delete; + + /** + * Initialize the OTel metrics pipeline and register all instruments. + * + * @param endpoint OTLP/HTTP endpoint URL for metric export + * (e.g. "http://localhost:4318/v1/metrics"). + * @param instanceId Value for the service.instance.id resource + * attribute. When non-empty, Prometheus metrics + * carry a service_instance_id label for per-node + * filtering. + */ + void + start(std::string const& endpoint, std::string const& instanceId = {}); + + /** + * Detach all ObservableGauge callbacks so they no-op on the next + * reader-thread tick. + * + * Must be called BEFORE any Application service that the callbacks + * read (nodeStore, overlay, networkOPs, ledgerMaster, etc.) is + * stopped. The flag is checked with acquire ordering at the top of + * every callback; together with the release store here it + * guarantees that once `detachCallbacks()` returns, no subsequent + * callback invocation will dereference an already-stopped service. + * + * Idempotent. Safe to call multiple times. Safe to call before + * `start()` (has no effect). The actual SDK-level provider + * shutdown still happens in `stop()`. + */ + void + detachCallbacks() noexcept; + + /** + * Flush pending metrics and shut down the pipeline. + * + * @pre `detachCallbacks()` should have been called earlier in the + * shutdown sequence; otherwise there is a narrow race between + * the final reader-thread tick and the destruction of + * Application services that the gauge callbacks read from. + */ + void + stop(); + + /** + * @return true if the registry is actively exporting metrics. + */ + [[nodiscard]] bool + isEnabled() const noexcept + { + return enabled_; + } + + // ----------------------------------------------------------------- + // Synchronous instrument recording (called from PerfLog hot paths) + // ----------------------------------------------------------------- + + /** + * Record an RPC method call start. + * @param method The RPC method name (e.g. "server_info"). + */ + void + recordRpcStarted(std::string_view method); + + /** + * Record an RPC method call completion. + * @param method The RPC method name. + * @param durationUs Execution time in microseconds. + */ + void + recordRpcFinished(std::string_view method, std::int64_t durationUs); + + /** + * Record an RPC method call error. + * @param method The RPC method name. + * @param durationUs Execution time in microseconds. + */ + void + recordRpcErrored(std::string_view method, std::int64_t durationUs); + + /** + * Record a job enqueued event. + * @param jobType The job type name (e.g. "ledgerData"). + */ + void + recordJobQueued(std::string_view jobType); + + /** + * Record a job start event. + * @param jobType The job type name. + * @param queuedDurUs Time the job spent waiting in the queue (us). + */ + void + recordJobStarted(std::string_view jobType, std::int64_t queuedDurUs); + + /** + * Record a job finish event. + * @param jobType The job type name. + * @param runningDurUs Execution time in microseconds. + */ + void + recordJobFinished(std::string_view jobType, std::int64_t runningDurUs); + + // ----------------------------------------------------------------- + // External dashboard parity counters (Tasks 7.9-7.14) + // ----------------------------------------------------------------- + + /** + * Increment the ledgers_closed_total counter. + * Called from RCLConsensus::Adaptor::doAccept() after a ledger is + * accepted by consensus. + */ + void + incrementLedgersClosed(); + + /** + * Increment the validations_sent_total counter. + * Called from RCLConsensus::Adaptor::validate() when a validation + * is produced and broadcast. + */ + void + incrementValidationsSent(); + + /** + * Increment the validations_checked_total counter. + * Called from NetworkOPs::recvValidation() when a network validation + * is received and checked. + */ + void + incrementValidationsChecked(); + + /** + * Increment the state_changes_total counter. + * Called from NetworkOPsImp::setMode() when the server operating mode + * changes (e.g. CONNECTED -> SYNCING -> TRACKING -> FULL). + */ + void + incrementStateChanges(); + + /** + * Increment the ledger_history_mismatch_total counter for a reason. + * Called from LedgerHistory::handleMismatch() once the mismatch has + * been classified. The reason label turns fork diagnosis from a + * log-grep into a queryable time series. + * @param reason Classified mismatch cause (e.g. "prior_ledger", + * "close_time", "consensus_txset", "same_txset_diff_result", + * "unknown"). + */ + void + incrementLedgerHistoryMismatch(std::string_view reason); + + /** + * Increment the txq_expired_total counter. + * Called from TxQ::processClosedLedger() for each queued transaction + * removed because its LastLedgerSequence has passed — submitters who + * under-bid the escalating fee and were never included. + */ + void + incrementTxqExpired(); + + /** + * Increment the txq_dropped_total{reason} counter. + * Called from TxQ::apply() when a transaction is refused admission to + * the queue (e.g. the queue is full). Distinct from expiry (already + * queued) and from jq_trans_overflow (job queue, not TxQ). + * @param reason Admission-control rejection cause (e.g. "queue_full"). + */ + void + incrementTxqDropped(std::string_view reason); + + /** + * Access the validation agreement tracker. + * Used by consensus and ledger hooks to record our validations and + * network validations so the tracker can compute agreement percentages. + * @return Reference to the internal ValidationTracker instance. + */ + ValidationTracker& + getValidationTracker() + { + return validationTracker_; + } + +private: + /** + * Master enable flag; when false all methods are no-ops. + */ + bool const enabled_; + + /** + * Tracks validation agreement between this node and the network. + * Lives outside the XRPL_ENABLE_TELEMETRY guard because it is + * always safe to record events; the gauge callback simply won't + * fire when telemetry is disabled. + */ + ValidationTracker validationTracker_; + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Reference to Application services for gauge callbacks. + * Only needed when OTel is compiled in, since observable gauge + * callbacks live entirely inside the XRPL_ENABLE_TELEMETRY guard. + */ + ServiceRegistry& app_; + + /** + * Journal for logging. + */ + beast::Journal const journal_; + + /** + * Set by detachCallbacks() during shutdown so every ObservableGauge + * callback returns early before reading Application services that + * may already be stopped. Checked with memory_order_acquire at the + * top of each callback to pair with the memory_order_release store + * in detachCallbacks(). + */ + std::atomic callbacksDetached_{false}; + + /** + * The SDK MeterProvider that owns the export pipeline. + */ + std::shared_ptr provider_; + + /** + * The Meter used to create all instruments. + */ + opentelemetry::nostd::shared_ptr meter_; + + // --- Synchronous instruments (RPC) --- + /** + * Counter: rpc_method_started_total{method=""} + */ + opentelemetry::nostd::unique_ptr> rpcStartedCounter_; + /** + * Counter: rpc_method_finished_total{method=""} + */ + opentelemetry::nostd::unique_ptr> rpcFinishedCounter_; + /** + * Counter: rpc_method_errored_total{method=""} + */ + opentelemetry::nostd::unique_ptr> rpcErroredCounter_; + /** + * Histogram: rpc_method_duration_us{method=""} + */ + opentelemetry::nostd::unique_ptr> + rpcDurationHistogram_; + + // --- Synchronous instruments (Job Queue) --- + /** + * Counter: job_queued_total{job_type=""} + */ + opentelemetry::nostd::unique_ptr> jobQueuedCounter_; + /** + * Counter: job_started_total{job_type=""} + */ + opentelemetry::nostd::unique_ptr> jobStartedCounter_; + /** + * Counter: job_finished_total{job_type=""} + */ + opentelemetry::nostd::unique_ptr> jobFinishedCounter_; + /** + * Histogram: job_queued_duration_us{job_type=""} + */ + opentelemetry::nostd::unique_ptr> + jobQueuedDurationHistogram_; + /** + * Histogram: job_running_duration_us{job_type=""} + */ + opentelemetry::nostd::unique_ptr> + jobRunningDurationHistogram_; + + // --- Observable gauges (registered via callbacks) --- + // Handles are stored so we can remove callbacks on shutdown. + /** + * Observable gauges for cache hit rates and sizes. + */ + opentelemetry::nostd::shared_ptr + cacheHitRateGauge_; + /** + * Observable gauges for TxQ metrics. + */ + opentelemetry::nostd::shared_ptr txqGauge_; + /** + * Observable gauges for counted object instances. + */ + opentelemetry::nostd::shared_ptr + objectCountGauge_; + /** + * Observable gauges for load factor breakdown. + */ + opentelemetry::nostd::shared_ptr loadFactorGauge_; + /** + * Observable gauges for NodeStore write_load and read_queue. + */ + opentelemetry::nostd::shared_ptr nodeStoreGauge_; + /** + * Observable gauge for server-level health metrics (state, uptime, peers, etc.). + */ + opentelemetry::nostd::shared_ptr serverInfoGauge_; + /** + * Observable gauge for build version info (label-based, value=1). + */ + opentelemetry::nostd::shared_ptr buildInfoGauge_; + /** + * Observable gauge for complete ledger range start/end pairs. + */ + opentelemetry::nostd::shared_ptr + completeLedgersGauge_; + /** + * Observable gauge for database sizes and historical fetch rate. + */ + opentelemetry::nostd::shared_ptr dbMetricsGauge_; + + // --- External dashboard parity gauges (Tasks 7.9-7.13) --- + /** + * Observable gauge for validator health indicators (amendment blocked, + * UNL blocked, quorum, UNL expiry). + */ + opentelemetry::nostd::shared_ptr + validatorHealthGauge_; + /** + * Observable gauge for peer network quality metrics (P90 latency, + * insane peer count, version spread, upgrade recommendation). + */ + opentelemetry::nostd::shared_ptr + peerQualityGauge_; + /** + * Observable gauge for transaction reduce-relay efficiency (selected vs + * suppressed peers, feature-disabled peers, missing-tx frequency). + */ + opentelemetry::nostd::shared_ptr + reduceRelayGauge_; + /** + * Observable gauge for ledger economy metrics (base fee, reserve, + * reserve increment, ledger age). + */ + opentelemetry::nostd::shared_ptr + ledgerEconomyGauge_; + /** + * Observable gauge for node state tracking (operating mode value, + * time in current state). + */ + opentelemetry::nostd::shared_ptr + stateTrackingGauge_; + /** + * Observable gauge for storage detail metrics (NuDB on-disk size). + */ + opentelemetry::nostd::shared_ptr + storageDetailGauge_; + /** + * Observable gauge for validation agreement metrics (1h/24h percentages + * and counts from ValidationTracker). + */ + opentelemetry::nostd::shared_ptr + validationAgreementGauge_; + + // --- External dashboard parity counters (Task 7.14) --- + /** + * Counter: ledgers_closed_total — incremented each consensus round. + */ + opentelemetry::nostd::unique_ptr> + ledgersClosedCounter_; + /** + * Counter: validations_sent_total — incremented when this node sends a validation. + */ + opentelemetry::nostd::unique_ptr> + validationsSentCounter_; + /** + * Counter: validations_checked_total — incremented for each network validation + * received. + */ + opentelemetry::nostd::unique_ptr> + validationsCheckedCounter_; + /** + * Counter: state_changes_total — incremented on operating mode transitions. + */ + opentelemetry::nostd::unique_ptr> + stateChangesCounter_; + /** + * ObservableCounter: jq_trans_overflow_total — observed from + * Overlay::getJqTransOverflow() (cumulative overflow tally owned by the overlay). + */ + opentelemetry::nostd::shared_ptr + jqTransOverflowObservable_; + /** + * Counter: ledger_history_mismatch_total{reason} — incremented per classified + * built-vs-validated ledger mismatch. + */ + opentelemetry::nostd::unique_ptr> + ledgerHistoryMismatchCounter_; + /** + * Counter: txq_expired_total — incremented per transaction expired out of the + * transaction queue. + */ + opentelemetry::nostd::unique_ptr> txqExpiredCounter_; + /** + * Counter: txq_dropped_total{reason} — incremented when a transaction is refused + * admission to the queue. + */ + opentelemetry::nostd::unique_ptr> txqDroppedCounter_; + /** + * ObservableCounter: validation_agreements_total — observed from + * ValidationTracker::totalAgreementsEver() (monotonic gross lifetime + * tally, initial-classification semantics). + */ + opentelemetry::nostd::shared_ptr + validationAgreementsObservable_; + /** + * ObservableCounter: validation_missed_total — observed from + * ValidationTracker::totalMissedEver() (monotonic gross lifetime tally, + * initial-classification semantics). + */ + opentelemetry::nostd::shared_ptr + validationMissedObservable_; + + /** + * Build the OTLP/HTTP exporter, periodic reader, resource attributes and + * histogram views, then create the MeterProvider and meter. Extracted + * from start() to keep each function under the 80-line limit. + * + * @param endpoint OTLP/HTTP metrics endpoint URL. + * @param instanceId service.instance.id resource attribute (may be empty). + */ + void + initExporterAndProvider(std::string const& endpoint, std::string const& instanceId); + + /** + * Create the synchronous instruments (RPC and job-queue counters and + * histograms, plus the external dashboard parity counters). Extracted + * from start() to keep each function under the 80-line limit. + */ + void + initSyncInstruments(); + + /** + * Register all observable gauge callbacks with the OTel SDK. + * Dispatches to one helper per metric domain so that each helper + * stays well under the 80-line-per-function limit. + */ + void + registerAsyncGauges(); + + // Per-domain gauge registration helpers. Each creates its instrument + // and attaches a single ObservableGauge callback that reads current + // values from Application services. The callbacks run on the OTel + // PeriodicExportingMetricReader background thread (~10 s tick). + void + registerCacheHitRateGauge(); // Task 9.2 + void + registerTxqGauge(); // Task 9.3 + void + registerObjectCountGauge(); // Task 9.6 + void + registerLoadFactorGauge(); // Task 9.7 + void + registerNodeStoreGauge(); // Task 9.1 + void + registerServerInfoGauge(); // Task 9.7a + void + registerBuildInfoGauge(); // Task 9.7b + void + registerCompleteLedgersGauge(); // Task 9.7c + void + registerDbMetricsGauge(); // Task 9.7d + void + registerValidatorHealthGauge(); // Task 7.9 + void + registerPeerQualityGauge(); // Task 7.10 + void + registerReduceRelayGauge(); // Reduce-relay efficiency + void + registerLedgerEconomyGauge(); // Task 7.11 + void + registerStateTrackingGauge(); // Task 7.12 + void + registerStorageDetailGauge(); // Task 7.13 + void + registerValidationAgreementGauge(); // Task 7.15 + void + registerValidationTotalsCounters(); // gap-fill: lifetime agree/miss _total +#endif // XRPL_ENABLE_TELEMETRY +}; + +} // namespace telemetry +} // namespace xrpl diff --git a/src/xrpld/telemetry/PropagationHelpers.h b/src/xrpld/telemetry/PropagationHelpers.h new file mode 100644 index 00000000000..ce64733d82f --- /dev/null +++ b/src/xrpld/telemetry/PropagationHelpers.h @@ -0,0 +1,63 @@ +#pragma once + +/** + * Helpers for injecting trace context into protobuf messages. + * + * Bridges the gap between SpanGuard (which hides OTel types) and the + * protobuf TraceContext message used for cross-node propagation. + * + * Dependency diagram: + * + * SpanGuard::getTraceBytes() protocol::TraceContext (proto) + * \ / + * +--- TraceBytes -----+ + * | | + * injectSpanContext(span, proto) + * + * @note When XRPL_ENABLE_TELEMETRY is disabled, getTraceBytes() returns + * {.valid=false}, so injectSpanContext becomes a no-op with zero overhead. + * + * Usage: + * @code + * // Send side — inject from a SpanGuard reference: + * protocol::TMTransaction tx; + * // ... populate tx fields ... + * injectSpanContext(mySpanGuard, *tx.mutable_trace_context()); + * overlay.relay(txID, tx, toSkip); + * @endcode + * + * @see ConsensusReceiveTracing.h and TxTracing.h for receive-side + * extraction helpers. + * @see TraceContextPropagator.h for low-level OTel context serialization. + */ + +#include +#include + +namespace xrpl::telemetry { + +/** + * Inject trace context from an active SpanGuard into a protobuf + * TraceContext message for cross-node propagation. + * + * Reads the span's trace_id, span_id, and trace_flags via + * getTraceBytes() and writes them into the protobuf fields. + * Safe to call from any thread that holds a reference to the span. + * No-op if the span is null or inactive. + * + * @param span The active SpanGuard whose context to propagate. + * @param proto The protobuf TraceContext to populate. + */ +inline void +injectSpanContext(SpanGuard const& span, protocol::TraceContext& proto) +{ + auto const bytes = span.getTraceBytes(); + if (!bytes.valid) + return; + + proto.set_trace_id(bytes.traceId.data(), bytes.traceId.size()); + proto.set_span_id(bytes.spanId.data(), bytes.spanId.size()); + proto.set_trace_flags(bytes.traceFlags); +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/TxSpanNames.h b/src/xrpld/telemetry/TxSpanNames.h new file mode 100644 index 00000000000..e8d75f17afc --- /dev/null +++ b/src/xrpld/telemetry/TxSpanNames.h @@ -0,0 +1,122 @@ +#pragma once + +/** + * Compile-time span name constants for transaction tracing. + * + * Used by PeerImp (overlay) and NetworkOPs (app) for transaction + * lifecycle spans. Built on StaticStr/join() from SpanNames.h. + * + * Span hierarchy (cross-node propagation): + * + * Node A (sender) Node B (receiver) + * +---------------------+ +---------------------+ + * | tx.process | protobuf | tx.receive | + * | injectSpanContext | ---------> | txReceiveSpan() | + * | (PropagationHelp.) | trace_ctx | extracts parent | + * +---------------------+ +---------------------+ + */ + +#include + +namespace xrpl::telemetry::tx_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "tx" — root prefix for transaction lifecycle spans. + */ +inline constexpr auto tx = seg::tx; +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto receive = makeStr("receive"); +inline constexpr auto process = makeStr("process"); +} // namespace op + +// ===== Full span names (prefix.op) ========================================= + +inline constexpr auto receive = join(prefix::tx, op::receive); +inline constexpr auto process = join(prefix::tx, op::process); + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). + */ +using ::xrpl::telemetry::attr::peerId; +using ::xrpl::telemetry::attr::txHash; + +/** + * "local" — whether tx originated locally. + */ +inline constexpr auto local = makeStr("local"); +/** + * "path" — sync or async processing path. + */ +inline constexpr auto path = makeStr("path"); +/** + * "suppressed" — whether tx was suppressed as duplicate. + */ +inline constexpr auto suppressed = makeStr("suppressed"); +/** + * "tx_status" — domain-qualified (collides with rpc_status, txq_status). + */ +inline constexpr auto txStatus = makeStr("tx_status"); +/** + * "peer_version" — version of peer that sent the tx. + */ +inline constexpr auto peerVersion = makeStr("peer_version"); +/** + * "tx_type" — transaction type name (e.g., "Payment", "OfferCreate"). + */ +inline constexpr auto txType = makeStr("tx_type"); +/** + * "fee" — transaction fee in drops. + */ +inline constexpr auto fee = makeStr("fee"); +/** + * "sequence" — transaction sequence number. + */ +inline constexpr auto sequence = makeStr("sequence"); +/** + * "ter_result" — engine result code after application. + */ +inline constexpr auto terResult = makeStr("ter_result"); +/** + * "applied" — whether the transaction was applied to the ledger. + */ +inline constexpr auto applied = makeStr("applied"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +inline constexpr auto sync = makeStr("sync"); +inline constexpr auto async = makeStr("async"); +inline constexpr auto knownBad = makeStr("known_bad"); +/** + * Transaction was suppressed via HashRouter (duplicate, not flagged bad). + */ +inline constexpr auto suppressed = makeStr("suppressed"); +/** + * Transaction was rejected because it carried tfInnerBatchTxn, which + * must never appear in network-relayed traffic. + */ +inline constexpr auto rejectedInnerBatch = makeStr("rejected_inner_batch"); +/** + * Transaction was dropped because the validated ledger is too old to + * confidently apply new transactions (server is out of sync). + */ +inline constexpr auto droppedNoSync = makeStr("dropped_no_sync"); +/** + * Transaction was dropped because the local job queue for jtTRANSACTION + * is at MAX_TRANSACTIONS — backpressure on the receive side. + */ +inline constexpr auto droppedQueueFull = makeStr("dropped_queue_full"); +} // namespace val + +} // namespace xrpl::telemetry::tx_span diff --git a/src/xrpld/telemetry/TxTracing.h b/src/xrpld/telemetry/TxTracing.h new file mode 100644 index 00000000000..02900fa9b42 --- /dev/null +++ b/src/xrpld/telemetry/TxTracing.h @@ -0,0 +1,69 @@ +#pragma once + +/** + * Helper functions for creating transaction trace spans. + * + * Encapsulates the logic for creating SpanGuard instances with + * hash-derived trace IDs and optional protobuf parent extraction. + * Call sites in PeerImp and NetworkOPs stay simple one-liners. + * + * When XRPL_ENABLE_TELEMETRY is not defined, the functions return + * no-op SpanGuard instances (zero overhead, zero dependencies). + */ + +#include + +#include +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +/** + * Create a "tx.receive" span for a transaction received from a peer. + * trace_id is derived from txID[0:16]. If the incoming message carries + * a protobuf TraceContext with a valid span_id, it is used as the + * parent to preserve relay ordering. + */ +inline SpanGuard +txReceiveSpan(uint256 const& txID, [[maybe_unused]] protocol::TMTransaction const& msg) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (msg.has_trace_context()) + { + auto const& tc = msg.trace_context(); + // Only the span_id is taken from the peer here; the trace_id is + // derived locally from txID, so validate the span_id alone. + if (tc.has_span_id() && isValidSpanId(tc.span_id())) + { + return SpanGuard::hashSpan( + TraceCategory::Transactions, + tx_span::receive, + txID.data(), + txID.kBytes, + reinterpret_cast(tc.span_id().data()), + tc.span_id().size(), + tc.has_trace_flags() ? static_cast(tc.trace_flags()) + : std::uint8_t{0}); + } + } +#endif + return SpanGuard::hashSpan( + TraceCategory::Transactions, tx_span::receive, txID.data(), txID.kBytes); +} + +/** + * Create a "tx.process" span for transaction processing in NetworkOPs. + * trace_id is derived from txID[0:16]. + */ +inline SpanGuard +txProcessSpan(uint256 const& txID) +{ + return SpanGuard::hashSpan( + TraceCategory::Transactions, tx_span::process, txID.data(), txID.kBytes); +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/ValidationTracker.h b/src/xrpld/telemetry/ValidationTracker.h new file mode 100644 index 00000000000..ac80f5cdf50 --- /dev/null +++ b/src/xrpld/telemetry/ValidationTracker.h @@ -0,0 +1,410 @@ +#pragma once + +/** + * @file ValidationTracker.h + * Standalone validation agreement tracker for telemetry. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +/** + * Tracks whether this validator's validations agree with network consensus, + * maintaining rolling 1-hour and 24-hour windows plus lifetime totals. + * + * The tracker operates by recording two independent events per ledger: + * 1. "We validated" -- our node published a validation for a ledger hash. + * 2. "Network validated" -- the network reached consensus on a ledger hash. + * + * After a configurable grace period (kGracePeriod), the reconcile() method + * compares the two flags. If both are set the ledger is counted as an + * "agreement"; otherwise it is a "miss". A late-repair mechanism allows a + * miss to be upgraded to an agreement if matching evidence arrives within + * kLateRepairWindow. + * + * Architecture / dependency diagram: + * @code + * +--------------------------+ + * | ConsensusAdapter / | + * | ValidatorSite | + * | (callers) | + * +---+-------------+-------+ + * | | + * | recordOur | recordNetwork + * | Validation | Validation + * v v + * +---------------------------+ + * | ValidationTracker | + * |---------------------------| + * | pending_ (hash_map) |----> LedgerEvent per hash + * | window1h_ (deque) |----> WindowEvent sliding window + * | window24h_ (deque) |----> WindowEvent sliding window + * | atomic totals | + * +---------------------------+ + * | + * | reconcile() called periodically + * v + * agreement / miss counters updated + * @endcode + * + * Usage -- basic recording and querying: + * @code + * xrpl::telemetry::ValidationTracker tracker; + * + * // On local validation: + * tracker.recordOurValidation(ledgerHash, seq); + * + * // On network consensus: + * tracker.recordNetworkValidation(ledgerHash, seq); + * + * // Periodically (e.g. every few seconds): + * tracker.reconcile(); + * + * // Query agreement percentage: + * double pct = tracker.agreementPct1h(); + * @endcode + * + * Usage -- edge case with late arrival: + * @code + * xrpl::telemetry::ValidationTracker tracker; + * + * // Network validates first, our validation arrives late: + * tracker.recordNetworkValidation(hash, seq); + * tracker.reconcile(); // initially counted as a miss + * + * // Late local validation arrives within repair window: + * tracker.recordOurValidation(hash, seq); + * tracker.reconcile(); // repaired to agreement + * @endcode + * + * @note Thread-safety: all public methods are thread-safe. The pending_ + * map and sliding-window deques are protected by mutex_. Lifetime totals + * use std::atomic for lock-free reads. + */ +class ValidationTracker +{ +public: + /** + * Monotonic clock used for all internal timestamps. + */ + using Clock = std::chrono::steady_clock; + + /** + * Time point type from the monotonic clock. + */ + using TimePoint = Clock::time_point; + + /** + * Record that this node sent a validation for the given ledger. + * @param ledgerHash Hash of the ledger we validated. + * @param seq Ledger sequence number. + */ + void + recordOurValidation(uint256 const& ledgerHash, LedgerIndex seq); + + /** + * Record that the network reached consensus on the given ledger. + * @param ledgerHash Hash of the network-validated ledger. + * @param seq Ledger sequence number. + */ + void + recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq); + + /** + * Reconcile pending ledger events whose grace period has elapsed. + * Should be called periodically (e.g. every few seconds). Moves + * reconciled events into the sliding windows and updates totals. + * Also performs late-repair and eviction of stale data. + */ + void + reconcile(); + + /** + * @name Rolling-window percentage getters + */ + /** @{ */ + + /** + * Agreement percentage over the last 1 hour. + * @return Percentage [0.0, 100.0], or 0.0 if no data. + */ + double + agreementPct1h() const; + + /** + * Agreement percentage over the last 24 hours. + * @return Percentage [0.0, 100.0], or 0.0 if no data. + */ + double + agreementPct24h() const; + + /** + * Agreement percentage over the last 7 days. + * @return Percentage [0.0, 100.0], or 0.0 if no data. + */ + double + agreementPct7d() const; + + /** @} */ + + /** + * @name Rolling-window count getters + */ + /** @{ */ + + /** + * Number of agreements in the 1-hour window. + */ + uint64_t + agreements1h() const; + + /** + * Number of misses in the 1-hour window. + */ + uint64_t + missed1h() const; + + /** + * Number of agreements in the 24-hour window. + */ + uint64_t + agreements24h() const; + + /** + * Number of misses in the 24-hour window. + */ + uint64_t + missed24h() const; + + /** + * Number of agreements in the 7-day window. + */ + uint64_t + agreements7d() const; + + /** + * Number of misses in the 7-day window. + */ + uint64_t + missed7d() const; + + /** @} */ + + /** + * @name Lifetime totals (atomic, lock-free reads) + */ + /** @{ */ + + /** + * Total agreements since process start. + */ + uint64_t + totalAgreements() const; + + /** + * Total misses since process start. + */ + uint64_t + totalMissed() const; + + /** + * Lifetime agreements counted at first classification only. + * + * @note Unlike totalAgreements(), this is strictly monotonic: it is + * incremented only when a ledger is first reconciled as an agreement and + * is never adjusted by a late repair. It backs the monotonic Prometheus + * counter validation_agreements_total. See the counting-semantics + * note in detail/ValidationTracker.cpp. + */ + uint64_t + totalAgreementsEver() const; + + /** + * Lifetime misses counted at first classification only. + * + * @note Unlike totalMissed(), this is strictly monotonic: it is + * incremented only when a ledger is first reconciled as a miss and is + * never decremented by a late repair. It backs the monotonic Prometheus + * counter validation_missed_total. See the counting-semantics note + * in detail/ValidationTracker.cpp. + */ + uint64_t + totalMissedEver() const; + + /** + * Total validations this node sent. + */ + uint64_t + totalValidationsSent() const; + + /** + * Total network validations observed for comparison. + */ + uint64_t + totalValidationsChecked() const; + + /** @} */ + +private: + /** + * Per-ledger tracking state held in the pending map. + */ + struct LedgerEvent + { + uint256 ledgerHash; ///< Ledger hash being tracked. + LedgerIndex seq{0}; ///< Ledger sequence number. + TimePoint recordTime; ///< Time the event was first recorded. + bool weValidated = false; ///< True if we sent a validation. + bool networkValidated = false; ///< True if network reached consensus. + bool reconciled = false; ///< True once grace period elapsed. + bool agreed = false; ///< True if both flags set at reconcile. + }; + + /** + * Lightweight event stored in the sliding-window deques. + */ + struct WindowEvent + { + TimePoint time; ///< When the event was reconciled. + uint256 ledgerHash; ///< Ledger hash for late-repair matching. + bool agreed{false}; ///< Whether this was an agreement. + }; + + /** + * Grace period before reconciling a ledger event. + */ + static constexpr auto kGracePeriod = std::chrono::seconds(8); + + /** + * Window during which a missed event can be repaired. + */ + static constexpr auto kLateRepairWindow = std::chrono::minutes(5); + + /** + * Maximum number of pending (unreconciled + recently reconciled) events. + */ + static constexpr std::size_t kMaxPendingEvents = 1000; + + /** + * Duration of the short rolling window. + */ + static constexpr auto kWindow1h = std::chrono::hours(1); + + /** + * Duration of the long rolling window. + */ + static constexpr auto kWindow24h = std::chrono::hours(24); + + /** + * Duration of the extended rolling window (7 days). + */ + static constexpr auto kWindow7d = std::chrono::hours(168); + + /** + * Protects pending_, window1h_, window24h_, and window7d_. + */ + mutable std::mutex mutex_; + + /** + * Pending ledger events indexed by ledger hash. + */ + hash_map pending_; + + /** + * Sliding window of reconciled events (last 1 hour). + */ + std::deque window1h_; + + /** + * Sliding window of reconciled events (last 24 hours). + */ + std::deque window24h_; + + /** + * Sliding window of reconciled events (last 7 days). + */ + std::deque window7d_; + + /** + * Lifetime count of agreements (net: incremented on agree, also on + * repair). May be read via totalAgreements(); feeds the windowed gauge. + */ + std::atomic totalAgreements_{0}; + + /** + * Lifetime count of misses (net: incremented on miss, decremented on + * repair). NON-monotonic. May be read via totalMissed(). + */ + std::atomic totalMissed_{0}; + + /** + * Monotonic "gross" lifetime tallies for the Prometheus _total counters. + * + * Counting decision (initial-classification only): each reconciled + * ledger is counted exactly once, at its first classification, into + * exactly one of the two tallies below. A later late-repair + * (miss -> agreement) does NOT move either tally. This keeps both + * strictly monotonic (a Prometheus _total must never decrease) and + * additive: totalAgreementsGross_ + totalMissedGross_ == ledgers + * reconciled. The repaired/agreement view is still available from the + * windowed gauge (validation_agreement) and the net totals above. + */ + + /** + * Monotonic lifetime initial agreements; backs + * validation_agreements_total. Never adjusted on repair. + */ + std::atomic totalAgreementsGross_{0}; + + /** + * Monotonic lifetime initial misses; backs validation_missed_total. + * Never decremented on repair. + */ + std::atomic totalMissedGross_{0}; + + /** + * Lifetime count of validations this node sent. + */ + std::atomic totalValidationsSent_{0}; + + /** + * Lifetime count of network validations observed. + */ + std::atomic totalValidationsChecked_{0}; + + /** + * Remove entries older than their respective window durations. + * @param now Current time point. + */ + void + evictStaleWindows(TimePoint now); + + /** + * Remove reconciled pending entries older than the late-repair window. + * Also trims the map if it exceeds kMaxPendingEvents. + * @param now Current time point. + */ + void + evictOldPending(TimePoint now); + + /** + * Scan a window deque and flip the first non-agreed entry matching + * the given ledger hash to agreed. + * @param window The sliding-window deque to repair. + * @param hash Ledger hash to match. + */ + static void + repairWindowEntry(std::deque& window, uint256 const& hash); +}; + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/detail/ValidationTracker.cpp b/src/xrpld/telemetry/detail/ValidationTracker.cpp new file mode 100644 index 00000000000..c7f9c599bce --- /dev/null +++ b/src/xrpld/telemetry/detail/ValidationTracker.cpp @@ -0,0 +1,297 @@ +/** + * @file ValidationTracker.cpp + * Implementation of the ValidationTracker class. + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +void +ValidationTracker::recordOurValidation(uint256 const& ledgerHash, LedgerIndex seq) +{ + std::scoped_lock const lock(mutex_); + auto& evt = pending_[ledgerHash]; + if (evt.recordTime == TimePoint{}) + { + // First time seeing this ledger hash -- initialize. + evt.ledgerHash = ledgerHash; + evt.seq = seq; + evt.recordTime = Clock::now(); + } + evt.weValidated = true; + totalValidationsSent_.fetch_add(1, std::memory_order_relaxed); +} + +void +ValidationTracker::recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq) +{ + std::scoped_lock const lock(mutex_); + auto& evt = pending_[ledgerHash]; + if (evt.recordTime == TimePoint{}) + { + evt.ledgerHash = ledgerHash; + evt.seq = seq; + evt.recordTime = Clock::now(); + } + evt.networkValidated = true; + totalValidationsChecked_.fetch_add(1, std::memory_order_relaxed); +} + +void +ValidationTracker::reconcile() +{ + std::scoped_lock const lock(mutex_); + auto const now = Clock::now(); + + for (auto& [hash, evt] : pending_) + { + if (!evt.reconciled && (now - evt.recordTime) >= kGracePeriod) + { + // Initial reconciliation after grace period. + evt.reconciled = true; + evt.agreed = evt.weValidated && evt.networkValidated; + + if (evt.agreed) + { + totalAgreements_.fetch_add(1, std::memory_order_relaxed); + // Gross tally: count the initial agreement once. See the + // counting-decision note below (repair branch). + totalAgreementsGross_.fetch_add(1, std::memory_order_relaxed); + } + else + { + totalMissed_.fetch_add(1, std::memory_order_relaxed); + // Gross tally: count the initial miss once. See the + // counting-decision note below (repair branch). + totalMissedGross_.fetch_add(1, std::memory_order_relaxed); + } + + WindowEvent const we{.time = now, .ledgerHash = evt.ledgerHash, .agreed = evt.agreed}; + window1h_.push_back(we); + window24h_.push_back(we); + window7d_.push_back(we); + } + else if ( + evt.reconciled && !evt.agreed && evt.weValidated && evt.networkValidated && + (now - evt.recordTime) <= kLateRepairWindow) + { + // Late repair: was a miss, now both flags set. Adjust the NET + // totals (used by the windowed agreement gauge) so the live view + // reflects the repair. + evt.agreed = true; + totalMissed_.fetch_sub(1, std::memory_order_relaxed); + totalAgreements_.fetch_add(1, std::memory_order_relaxed); + + // Counting decision (initial-classification only): the gross + // tallies (totalAgreementsGross_ / totalMissedGross_) that back the + // monotonic Prometheus _total counters are deliberately NOT touched + // here. Each ledger is counted once, at first classification; a + // repair must not decrement missed (a _total may never decrease) + // nor add a second agreement (which would double-count the ledger). + + // Flip the corresponding window entries from miss to agreement. + repairWindowEntry(window1h_, evt.ledgerHash); + repairWindowEntry(window24h_, evt.ledgerHash); + repairWindowEntry(window7d_, evt.ledgerHash); + } + } + + evictStaleWindows(now); + evictOldPending(now); +} + +void +ValidationTracker::evictStaleWindows(TimePoint now) +{ + auto const cutoff1h = now - kWindow1h; + while (!window1h_.empty() && window1h_.front().time < cutoff1h) + window1h_.pop_front(); + + auto const cutoff24h = now - kWindow24h; + while (!window24h_.empty() && window24h_.front().time < cutoff24h) + window24h_.pop_front(); + + auto const cutoff7d = now - kWindow7d; + while (!window7d_.empty() && window7d_.front().time < cutoff7d) + window7d_.pop_front(); +} + +void +ValidationTracker::evictOldPending(TimePoint now) +{ + auto const cutoff = now - kLateRepairWindow; + for (auto it = pending_.begin(); it != pending_.end();) + { + if (it->second.reconciled && it->second.recordTime < cutoff) + { + it = pending_.erase(it); + } + else + { + ++it; + } + } + + // Hard trim if still over limit. The loop above already removed every + // reconciled entry older than the late-repair window, so here we drop + // any remaining reconciled entry as a last resort. + if (pending_.size() > kMaxPendingEvents) + { + for (auto it = pending_.begin(); + it != pending_.end() && pending_.size() > kMaxPendingEvents;) + { + if (it->second.reconciled) + { + it = pending_.erase(it); + } + else + { + ++it; + } + } + } +} + +double +ValidationTracker::agreementPct1h() const +{ + std::scoped_lock const lock(mutex_); + if (window1h_.empty()) + return 0.0; + auto const agreed = static_cast( + std::count_if(window1h_.begin(), window1h_.end(), [](auto const& e) { return e.agreed; })); + return (agreed / static_cast(window1h_.size())) * 100.0; +} + +double +ValidationTracker::agreementPct24h() const +{ + std::scoped_lock const lock(mutex_); + if (window24h_.empty()) + return 0.0; + auto const agreed = static_cast(std::count_if( + window24h_.begin(), window24h_.end(), [](auto const& e) { return e.agreed; })); + return (agreed / static_cast(window24h_.size())) * 100.0; +} + +uint64_t +ValidationTracker::agreements1h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window1h_.begin(), window1h_.end(), [](auto const& e) { return e.agreed; })); +} + +uint64_t +ValidationTracker::missed1h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window1h_.begin(), window1h_.end(), [](auto const& e) { return !e.agreed; })); +} + +uint64_t +ValidationTracker::agreements24h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast(std::count_if( + window24h_.begin(), window24h_.end(), [](auto const& e) { return e.agreed; })); +} + +uint64_t +ValidationTracker::missed24h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast(std::count_if( + window24h_.begin(), window24h_.end(), [](auto const& e) { return !e.agreed; })); +} + +double +ValidationTracker::agreementPct7d() const +{ + std::scoped_lock const lock(mutex_); + if (window7d_.empty()) + return 0.0; + auto const agreed = static_cast( + std::count_if(window7d_.begin(), window7d_.end(), [](auto const& e) { return e.agreed; })); + return (agreed / static_cast(window7d_.size())) * 100.0; +} + +uint64_t +ValidationTracker::agreements7d() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window7d_.begin(), window7d_.end(), [](auto const& e) { return e.agreed; })); +} + +uint64_t +ValidationTracker::missed7d() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window7d_.begin(), window7d_.end(), [](auto const& e) { return !e.agreed; })); +} + +uint64_t +ValidationTracker::totalAgreements() const +{ + return totalAgreements_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalMissed() const +{ + return totalMissed_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalAgreementsEver() const +{ + return totalAgreementsGross_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalMissedEver() const +{ + return totalMissedGross_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalValidationsSent() const +{ + return totalValidationsSent_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalValidationsChecked() const +{ + return totalValidationsChecked_.load(std::memory_order_relaxed); +} + +void +ValidationTracker::repairWindowEntry(std::deque& window, uint256 const& hash) +{ + // Scan backwards since late repairs target recently added entries. + for (auto& event : std::views::reverse(window)) + { + if (!event.agreed && event.ledgerHash == hash) + { + event.agreed = true; + return; + } + } +} + +} // namespace xrpl::telemetry diff --git a/tasks/fix-validation-checks.md b/tasks/fix-validation-checks.md new file mode 100644 index 00000000000..ea33fd92235 --- /dev/null +++ b/tasks/fix-validation-checks.md @@ -0,0 +1,169 @@ +# Fix Telemetry Validation Checks + +## Context + +The CI pipeline infrastructure is fully operational (build + deploy + run). However, +the `validate_telemetry.py` validation suite fails 35 checks due to mismatches between +what the validation expects and what the telemetry stack actually produces. These fall +into 4 categories. + +CI run: https://github.com/XRPLF/rippled/actions/runs/23026466191 + +--- + +## Category 1: StatsD Metrics — 0 Series (25 failures) + +**Symptoms:** + +``` +[FAIL] metric.statsd_gauges.xrpld_LedgerMaster_Validated_Ledger_Age: 0 series +[FAIL] metric.statsd_counters.xrpld_rpc_requests: 0 series +[FAIL] metric.statsd_histograms.xrpld_rpc_time: 0 series +[FAIL] metric.overlay_traffic.xrpld_total_Bytes_In: 0 series +[FAIL] metric.phase9_nodestore.xrpld_nodestore_reads_total: 0 series +... (25 total) +``` + +**Root Cause:** Two issues compounding: + +1. **StatsD receiver is commented out** in `otel-collector-config.yaml` (lines 39-54). + The collector config was updated to expect native OTLP metrics from beast::insight + (comment: "StatsD UDP port removed — beast::insight now uses native OTLP"), but + the validation harness configures xrpld nodes with `server=statsd`. + +2. **Metric name mismatch:** The `expected_metrics.json` expects StatsD-style metric + names (e.g., `xrpld_LedgerMaster_Validated_Ledger_Age`). When using `server=otel`, + beast::insight emits OTLP metrics which may have different names/structure. + +**Fix Options (pick one):** + +- **Option A (recommended):** Change the node config in `run-full-validation.sh` from + `server=statsd` to `server=otel` (line 255), remove the `address=127.0.0.1:8125` line, + then update `expected_metrics.json` with the actual OTLP metric names. This aligns with + the collector config's OTLP-first design and avoids re-enabling the StatsD receiver. + +- **Option B:** Uncomment the StatsD receiver in `otel-collector-config.yaml`, add + `statsd` to the metrics pipeline receivers list, and keep node config as `server=statsd`. + Simpler but goes against the migration to native OTLP. + +**Investigation needed for Option A:** + +- Run xrpld locally with `server=otel`, query Prometheus, and capture the actual OTLP + metric names to update `expected_metrics.json`. + +**Files to modify:** + +- `docker/telemetry/workload/run-full-validation.sh` — change `[insight]` section +- `docker/telemetry/workload/expected_metrics.json` — update metric names for OTLP +- `docker/telemetry/workload/validate_telemetry.py` — may need metric query adjustments + +--- + +## Category 2: Missing Spans — tx.process, tx.receive (2 failures) + +**Symptoms:** + +``` +[FAIL] span.tx.process: tx.process: 0 traces (expected > 0) +[FAIL] span.tx.receive: tx.receive: 0 traces (expected > 0) +``` + +**Root Cause:** The span names exist in the code: + +- `src/xrpld/app/misc/NetworkOPs.cpp:1228` — `XRPL_TRACE_TX("tx.process")` +- `src/xrpld/overlay/detail/PeerImp.cpp:1273` — `XRPL_TRACE_TX("tx.receive")` + +Likely causes (investigate in order): + +1. **Batch delay:** The 2-second batch delay (`batch_delay_ms=2000`) plus 30s propagation + wait may not be enough if these spans are created late in the workload. +2. **Code path not triggered:** `tx.process` fires in `NetworkOPs::processTransaction()`. + The tx_submitter submits via RPC `submit` command which calls this path. But if the + transactions fail validation before reaching `processTransaction()`, no span is emitted. +3. **Span naming mismatch:** The validation queries Tempo for exact operation name + `tx.process`. Verify Tempo stores the span with this exact name. + +**Investigation:** + +- Check the tx_submitter output in CI logs — are transactions actually succeeding? +- Query Tempo API locally for all span names to see what's actually emitted. + +**Files to modify:** + +- Possibly `docker/telemetry/workload/validate_telemetry.py` — adjust timing/queries +- Possibly `docker/telemetry/workload/run-full-validation.sh` — increase propagation wait + +--- + +## Category 3: Span Hierarchy — rpc.request -> rpc.process (1 failure) + +**Symptoms:** + +``` +[FAIL] span.hierarchy.rpc.request->rpc.process: rpc.process not found in rpc.request traces +``` + +**Root Cause:** The validator fetches traces containing `rpc.request` from Tempo and +checks if any child span is named `rpc.process`. Both spans are emitted (they pass +individual checks), but the parent-child relationship isn't established. + +**Investigation:** + +- Check `src/xrpld/rpc/detail/ServerHandler.cpp` — `rpc.request` (line 271) and + `rpc.process` (line 573) are in the same file. Verify that `rpc.process` is created + as a child of `rpc.request` (i.e., its parent context is set). +- The issue may be that `rpc.process` creates a new root span instead of linking to the + `rpc.request` span context. + +**Files to modify:** + +- Possibly `src/xrpld/rpc/detail/ServerHandler.cpp` — fix span parenting +- OR `docker/telemetry/workload/validate_telemetry.py` — if hierarchy check logic is wrong + +--- + +## Category 4: Dashboard 404s (5 failures) + +**Symptoms:** + +``` +[FAIL] dashboard.xrpld-statsd-node-health: HTTP 404 +[FAIL] dashboard.xrpld-statsd-network: HTTP 404 +[FAIL] dashboard.xrpld-statsd-rpc: HTTP 404 +[FAIL] dashboard.xrpld-statsd-overlay-detail: HTTP 404 +[FAIL] dashboard.xrpld-statsd-ledger-sync: HTTP 404 +``` + +**Root Cause:** Dashboard UIDs were renamed from `xrpld-statsd-*` to `xrpld-system-*` +but `expected_metrics.json` still references the old names. + +**Actual UIDs in `docker/telemetry/grafana/dashboards/`:** + +| Expected (in expected_metrics.json) | Actual (in dashboard JSON) | +| ----------------------------------- | ----------------------------- | +| `xrpld-statsd-node-health` | `xrpld-system-node-health` | +| `xrpld-statsd-network` | `xrpld-system-network` | +| `xrpld-statsd-rpc` | `xrpld-system-rpc` | +| `xrpld-statsd-overlay-detail` | `xrpld-system-overlay-detail` | +| `xrpld-statsd-ledger-sync` | `xrpld-system-ledger-sync` | + +**Fix:** Update the 5 UIDs in `expected_metrics.json` → `grafana_dashboards.uids[]`. + +**Files to modify:** + +- `docker/telemetry/workload/expected_metrics.json` — update dashboard UIDs + +--- + +## Execution Order + +1. **Category 4 (Dashboard UIDs)** — trivial rename, no investigation needed +2. **Category 1 (StatsD/OTLP metrics)** — requires investigation to choose Option A vs B + and capture actual metric names +3. **Category 2 (Missing tx spans)** — requires investigation into transaction code paths +4. **Category 3 (Span hierarchy)** — requires investigation into span context propagation + +## Branch + +All changes go on: `pratik/otel-phase10-workload-validation` +Worktree: `/tmp/otel-phase10-iter`