diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 070515c..40e907d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,7 +26,7 @@ jobs: - platform: linux/arm64 platform_suffix: arm64 - runner: ubuntu-24.04 + runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca0d057..2514fdf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,20 +11,17 @@ on: jobs: test: - name: Test - runs-on: ${{ matrix.os }} - strategy: - matrix: - go-version: [1.24.x] - os: [ubuntu-latest] + name: test + runs-on: ubuntu-latest steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ matrix.go-version }} - - - name: Check out code - uses: actions/checkout@v4 + go-version-file: go.mod + check-latest: true - name: Cache Go modules uses: actions/cache@v4 diff --git a/.gitignore b/.gitignore index c44ac6b..41922bb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,12 @@ devel/data /firestellar /build /dist + +# Runtime data dirs produced by devel/poller.sh and devel/captive-core.sh +/data +/data-cc +/data-cc-mainnet +/captive-core + +# Tests +test/.data \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 163f9ec..bc2a4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md) for instructions to keep up to date. +## Unreleased + +* Add captive-core fetcher backend (`firestellar fetch captive-core`) that spawns a `stellar-core` subprocess and streams ledgers via the captive-core peer + history archive path. Captive-core is now the supported backend going forward; the RPC poller is kept for compatibility but no longer actively developed. +* Add cursor persistence shared between both backends: `--state-dir` writes `cursor.json` after each emitted block so restarts resume at `last_fired_block + 1`. Default `--state-dir` is now `/data/work` for both backends (was `/data/poller` / `/data/captive-core`). +* Add `--ignore-cursor` flag to start fresh from `` when running under a supervisor that tracks downstream state (e.g. `firecore reader-node`). +* Add `--stellar-core-network` plus `--stellar-core-network-passphrase` / `--stellar-core-history-archive-urls` for custom-network captive-core deployments. +* Add `test/` battlefield integration suite: in-process captive-core + poller fetchers driven against `stellar/quickstart`, with deterministic snapshot comparison and cross-backend diffing. +* Enforce `stellar-core >= 26.1.0-3210.427aa3978` (SDF May 2026 security advisory) for captive-core. +* CI now reads the Go toolchain version from `go.mod` (`go-version-file`) instead of pinning it inside the workflow. +* Fix poller hash / previous-hash encoding bug. +* Fix `json.Number` handling in XDR normalizers (preserves large-int precision in snapshot/diff round-trips). +* Restore `go-stellar-sdk` v0.5.0 and walk the current store when searching for cursor data. + ## v1.0.6 * bump SDK to match rpc V26 diff --git a/Dockerfile b/Dockerfile index 4b8f776..06deef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,13 @@ ARG FIRECORE_VERSION=v1.14.1 -FROM golang:1.25-bookworm AS build +FROM golang:1.26-bookworm AS build WORKDIR /app -# Copy go mod files first for better caching COPY go.mod go.sum ./ RUN go mod download -# Copy source code COPY . ./ -# Build the binary with version information ARG VERSION="edge" ARG BINARY_NAME=firestellar @@ -20,8 +17,43 @@ FROM ghcr.io/streamingfast/firehose-core:${FIRECORE_VERSION} ARG BINARY_NAME=firestellar -# Copy the binary to the firehose-core image +# Install stellar-core from SDF apt repo so the captive-core fetcher works +# standalone (default --stellar-core-bin is /usr/bin/stellar-core). +# SDF only publishes amd64 packages; arm64 images ship without stellar-core +# and require mounting a binary at /usr/bin/stellar-core or overriding +# --stellar-core-bin. The RPC fetcher works on arm64 without stellar-core. +# +# Security: stellar-core 26.1.0-3210.427aa3978 fixes a critical network +# vulnerability (SDF advisory, May 2026). The build pulls from SDF's +# `stable` apt channel which now ships the patched version; rebuilds +# after that publish date pick it up automatically. STELLAR_CORE_MIN_VERSION +# is asserted post-install to fail the build loudly if the apt index is +# pinned/cached to a pre-fix package somehow. +ARG TARGETARCH +ARG STELLAR_CORE_MIN_VERSION=26.1.0-3210.427aa3978 +RUN set -eux; \ + if [ "${TARGETARCH}" = "amd64" ]; then \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl gnupg dpkg; \ + install -m 0755 -d /etc/apt/keyrings; \ + curl -sSL https://apt.stellar.org/SDF.asc | gpg --dearmor -o /etc/apt/keyrings/SDF.gpg; \ + chmod a+r /etc/apt/keyrings/SDF.gpg; \ + . /etc/os-release; \ + echo "deb [signed-by=/etc/apt/keyrings/SDF.gpg] https://apt.stellar.org ${VERSION_CODENAME} stable" \ + > /etc/apt/sources.list.d/SDF.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends stellar-core; \ + rm -rf /var/lib/apt/lists/*; \ + stellar-core version; \ + INSTALLED=$(dpkg-query -W -f='${Version}' stellar-core); \ + if ! dpkg --compare-versions "${INSTALLED}" ge "${STELLAR_CORE_MIN_VERSION}"; then \ + echo "stellar-core ${INSTALLED} is older than required ${STELLAR_CORE_MIN_VERSION}; refusing to build (see SDF May 2026 advisory)." >&2; \ + exit 1; \ + fi; \ + else \ + echo "Skipping stellar-core install on ${TARGETARCH} (SDF amd64-only). Mount /usr/bin/stellar-core or use --stellar-core-bin."; \ + fi + COPY --from=build "/app/${BINARY_NAME}" "/app/${BINARY_NAME}" -# We use firecore entrypoint since it's the main application that people should run to setup Firehose stack ENTRYPOINT ["/app/firecore"] diff --git a/Dockerfile-rpc-fetcher b/Dockerfile-rpc-fetcher new file mode 100644 index 0000000..36e865b --- /dev/null +++ b/Dockerfile-rpc-fetcher @@ -0,0 +1,27 @@ +ARG FIRECORE_VERSION=v1.14.1 + +FROM golang:1.26-bookworm AS build +WORKDIR /app + +# Copy go mod files first for better caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . ./ + +# Build the binary with version information +ARG VERSION="edge" +ARG BINARY_NAME=firestellar + +RUN go build -v -ldflags "-X main.version=${VERSION}" -o "${BINARY_NAME}" "./cmd/${BINARY_NAME}" + +FROM ghcr.io/streamingfast/firehose-core:${FIRECORE_VERSION} + +ARG BINARY_NAME=firestellar + +# Copy the binary to the firehose-core image +COPY --from=build "/app/${BINARY_NAME}" "/app/${BINARY_NAME}" + +# We use firecore entrypoint since it's the main application that people should run to setup Firehose stack +ENTRYPOINT ["/app/firecore"] diff --git a/README.md b/README.md index 8b0109b..d099409 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,40 @@ Quick start with Firehose for Stellar can be found in the official Firehose docs - [Data Storage](https://firehose.streamingfast.io/concepts-and-architeceture/data-storage) - [Design Principles](https://firehose.streamingfast.io/concepts-and-architeceture/design-principles) -## Running the Firehose poller +## Running the Firehose fetcher -The below command with start streaming Firehose Stellar blocks, check `proto/sf/stellar/type/v1/block.proto` for more information. +Two fetcher backends are available. Both emit the same `pbbstream.Block` shape; check `proto/sf/stellar/type/v1/block.proto` for the payload schema. + +> **Captive-core is the supported backend going forward.** The RPC poller is kept for compatibility but is no longer actively developed — new deployments should use captive-core. + +### Captive-core backend (recommended) + +Spawns a `stellar-core` subprocess and streams ledgers from it. + +```bash +firestellar fetch captive-core {FIRST_STREAMABLE_BLOCK} \ + --stellar-core-bin /usr/bin/stellar-core \ + --stellar-core-network mainnet \ + --state-dir {STATE_DIR} +``` + +### RPC backend (legacy) + +Streams ledgers from a Stellar RPC endpoint. Maintenance-only — prefer captive-core for new work. ```bash firestellar fetch rpc {FIRST_STREAMABLE_BLOCK} --endpoints {STELLAR_RPC_ENDPOINT} --state-dir {STATE_DIR} ``` +### Resume behavior (`--state-dir` / `--ignore-cursor`) + +Both backends persist the last fired block to `{STATE_DIR}/cursor.json` after each successful emission. On restart, the fetcher resumes at `last_fired_block + 1` instead of replaying from `{FIRST_STREAMABLE_BLOCK}`. + +- `--state-dir` — directory holding `cursor.json`. Default: `/data/work` (both backends). Pass an empty string to disable persistence. +- `--ignore-cursor` — ignore any persisted `cursor.json` and start fresh from `{FIRST_STREAMABLE_BLOCK}`. Use this when running under a supervisor (e.g. `firecore reader-node`) that already tracks downstream state and passes the correct start block on restart. + +The cursor schema is shared between the two backends, so a single state directory can be reused if you switch backends. + ## Contributing For more information, please read the [CONTRIBUTING.md](CONTRIBUTING.md) file. diff --git a/captivecore/captivecore.go b/captivecore/captivecore.go new file mode 100644 index 0000000..a93db83 --- /dev/null +++ b/captivecore/captivecore.go @@ -0,0 +1,508 @@ +// Package captivecore exposes the captive-core block fetcher as a library. +// +// Two layers: +// +// 1. Fetcher — converter from xdr.LedgerCloseMeta to pbbstream.Block. +// Stateless apart from network passphrase + logger. +// +// 2. Backend — wraps *ledgerbackend.CaptiveStellarCore (the stellar-core +// subprocess) and offers PrepareRange / GetBlock / Close. +package captivecore + +import ( + "context" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "time" + + "github.com/sirupsen/logrus" + "github.com/stellar/go-stellar-sdk/ingest" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stellar/go-stellar-sdk/network" + "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" + "github.com/streamingfast/firehose-stellar/types" + "github.com/streamingfast/firehose-stellar/utils" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Config is the parameter set for a captive-core backend. Use +// ResolveNetwork to fill defaults for mainnet/testnet. +type Config struct { + // BinaryPath to the stellar-core executable. Required. + BinaryPath string + + // NetworkPassphrase the chain uses (e.g. "Public Global Stellar + // Network ; September 2015"). Required. + NetworkPassphrase string + + // HistoryArchiveURLs stellar-core pulls from to catch up. At least + // one required. + HistoryArchiveURLs []string + + // StellarCoreConfPath to a stellar-core.cfg on disk. If empty, + // DefaultTomlData must be set. + StellarCoreConfPath string + + // DefaultTomlData is the bundled config bytes for the chosen + // network (PubnetDefaultConfig / TestnetDefaultConfig from + // stellar/go SDK). Only used when StellarCoreConfPath is empty. + DefaultTomlData []byte + + // StoragePath is the working directory stellar-core uses for its + // sqlite db, buckets, and tmp files. Defaults to a fresh temp dir + // created at New() time when empty. + StoragePath string + + // LogLevel for the stellar-core subprocess log stream. Defaults to + // logrus.InfoLevel. + LogLevel logrus.Level + + // LogOutput is where stellar-core's logrus stream writes. When nil, + // the SDK default (stderr) is used. Set this to a file handle to + // redirect the spammy stellar-core output away from the terminal. + LogOutput io.Writer + + // Logger receives high-level fetcher events. Required. + Logger *zap.Logger +} + +// ResolveNetwork fills NetworkPassphrase, HistoryArchiveURLs, and +// DefaultTomlData based on the logical network name (mainnet|testnet| +// custom). Values already set on Config are preserved. +func (c *Config) ResolveNetwork(name string) error { + switch name { + case "mainnet": + if c.NetworkPassphrase == "" { + c.NetworkPassphrase = network.PublicNetworkPassphrase + } + if len(c.HistoryArchiveURLs) == 0 { + c.HistoryArchiveURLs = network.PublicNetworkhistoryArchiveURLs + } + if c.DefaultTomlData == nil { + c.DefaultTomlData = ledgerbackend.PubnetDefaultConfig + } + case "testnet": + if c.NetworkPassphrase == "" { + c.NetworkPassphrase = network.TestNetworkPassphrase + } + if len(c.HistoryArchiveURLs) == 0 { + c.HistoryArchiveURLs = network.TestNetworkhistoryArchiveURLs + } + if c.DefaultTomlData == nil { + c.DefaultTomlData = ledgerbackend.TestnetDefaultConfig + } + case "custom": + // Passphrase + archive URLs must come from caller. No defaults. + default: + return fmt.Errorf("unsupported stellar network: %s (want mainnet|testnet|custom)", name) + } + return nil +} + +// validate checks required fields. Called from New(). +func (c *Config) validate() error { + if c.BinaryPath == "" { + return errors.New("captivecore: BinaryPath is required") + } + if c.NetworkPassphrase == "" { + return errors.New("captivecore: NetworkPassphrase is required") + } + if len(c.HistoryArchiveURLs) == 0 { + return errors.New("captivecore: HistoryArchiveURLs is required (at least one)") + } + if c.StellarCoreConfPath == "" && c.DefaultTomlData == nil { + return errors.New("captivecore: either StellarCoreConfPath or DefaultTomlData must be set") + } + if c.Logger == nil { + return errors.New("captivecore: Logger is required") + } + return nil +} + +// Backend drives a stellar-core subprocess and converts each fetched +// ledger to pbbstream.Block via the embedded Fetcher. +type Backend struct { + core *ledgerbackend.CaptiveStellarCore + fetcher *Fetcher + logger *zap.Logger +} + +// New constructs a Backend. The stellar-core subprocess starts on the +// first PrepareRange call. Always defer Close. +func New(cfg Config) (*Backend, error) { + if err := cfg.validate(); err != nil { + return nil, err + } + + // Match what soroban-rpc emits so blocks are byte-equivalent across + // both fetchers. All three flags require stellar-core protocol >= 23: + // - EmitUnifiedEvents: TransactionMetaV4 (CAP-67) — populates + // classic-tx transaction-events + per-op contract-events. + // - EnforceSorobanDiagnosticEvents: forces ENABLE_SOROBAN_DIAGNOSTIC_EVENTS. + // - EnforceSorobanTransactionMetaExtV1: extra Soroban meta ext. + params := ledgerbackend.CaptiveCoreTomlParams{ + NetworkPassphrase: cfg.NetworkPassphrase, + HistoryArchiveURLs: cfg.HistoryArchiveURLs, + CoreBinaryPath: cfg.BinaryPath, + EmitUnifiedEvents: true, + EnforceSorobanDiagnosticEvents: true, + EnforceSorobanTransactionMetaExtV1: true, + } + + var toml *ledgerbackend.CaptiveCoreToml + if cfg.StellarCoreConfPath != "" { + t, err := ledgerbackend.NewCaptiveCoreTomlFromFile(cfg.StellarCoreConfPath, params) + if err != nil { + return nil, fmt.Errorf("captivecore: setting up toml from file %s: %w", cfg.StellarCoreConfPath, err) + } + toml = t + } else { + t, err := ledgerbackend.NewCaptiveCoreTomlFromData(cfg.DefaultTomlData, params) + if err != nil { + return nil, fmt.Errorf("captivecore: setting up toml from default: %w", err) + } + toml = t + } + + coreLogger := log.New() + level := cfg.LogLevel + if level == 0 { + level = logrus.InfoLevel + } + coreLogger.SetLevel(level) + if cfg.LogOutput != nil { + coreLogger.SetOutput(cfg.LogOutput) + } + + core, err := ledgerbackend.NewCaptive(ledgerbackend.CaptiveCoreConfig{ + BinaryPath: cfg.BinaryPath, + NetworkPassphrase: cfg.NetworkPassphrase, + HistoryArchiveURLs: cfg.HistoryArchiveURLs, + StoragePath: cfg.StoragePath, + Toml: toml, + Log: coreLogger, + }) + if err != nil { + return nil, fmt.Errorf("captivecore: setting up captive-core backend: %w", err) + } + + return &Backend{ + core: core, + fetcher: &Fetcher{NetworkPassphrase: cfg.NetworkPassphrase, Logger: cfg.Logger}, + logger: cfg.Logger, + }, nil +} + +// PrepareRange launches stellar-core and catches up to startLedger. +// Subsequent GetBlock calls expect ledgers in [startLedger, ∞). Stays +// open until Close. +func (b *Backend) PrepareRange(ctx context.Context, startLedger uint64) error { + if startLedger < 1 { + return fmt.Errorf("captivecore: start ledger must be >= 1 (stellar ledger sequences start at 1)") + } + if startLedger > math.MaxUint32 { + return fmt.Errorf("captivecore: start ledger %d exceeds stellar ledger sequence range (uint32)", startLedger) + } + b.logger.Info("captivecore preparing range", zap.Uint64("start_block", startLedger)) + if err := b.core.PrepareRange(ctx, ledgerbackend.UnboundedRange(uint32(startLedger))); err != nil { + return fmt.Errorf("captivecore: prepare range from %d: %w", startLedger, err) + } + b.logger.Info("captivecore range prepared") + return nil +} + +// GetBlock returns one ledger as pbbstream.Block. Blocks until the +// ledger is available or ctx fires. +func (b *Backend) GetBlock(ctx context.Context, ledgerSeq uint64) (*pbbstream.Block, error) { + if ledgerSeq > math.MaxUint32 { + return nil, fmt.Errorf("captivecore: ledger %d exceeds uint32", ledgerSeq) + } + meta, err := b.core.GetLedger(ctx, uint32(ledgerSeq)) + if err != nil { + return nil, fmt.Errorf("captivecore: get ledger %d: %w", ledgerSeq, err) + } + blk, err := b.fetcher.ConvertLedgerCloseMetaToBstreamBlock(&meta) + if err != nil { + return nil, fmt.Errorf("captivecore: convert ledger %d: %w", ledgerSeq, err) + } + return blk, nil +} + +// Close terminates the stellar-core subprocess. Idempotent. +func (b *Backend) Close() error { + if b.core == nil { + return nil + } + err := b.core.Close() + b.core = nil + return err +} + +// Fetcher converts xdr.LedgerCloseMeta to the pbbstream.Block shape the +// RPC fetcher emits. NetworkPassphrase is used to recompute tx hashes. +type Fetcher struct { + NetworkPassphrase string + Logger *zap.Logger +} + +// ConvertLedgerCloseMetaToBstreamBlock converts one ledger to a +// pbbstream.Block. +func (f *Fetcher) ConvertLedgerCloseMetaToBstreamBlock(ledgerMetadata *xdr.LedgerCloseMeta) (*pbbstream.Block, error) { + var ledgerHeader xdr.LedgerHeaderHistoryEntry + var ledgerSeq uint32 + var ledgerHash xdr.Hash + + switch { + case ledgerMetadata.V0 != nil: + ledgerHeader = ledgerMetadata.V0.LedgerHeader + ledgerSeq = uint32(ledgerHeader.Header.LedgerSeq) + ledgerHash = ledgerMetadata.V0.LedgerHeader.Hash + case ledgerMetadata.V1 != nil: + ledgerHeader = ledgerMetadata.V1.LedgerHeader + ledgerSeq = uint32(ledgerHeader.Header.LedgerSeq) + ledgerHash = ledgerMetadata.V1.LedgerHeader.Hash + case ledgerMetadata.V2 != nil: + ledgerHeader = ledgerMetadata.V2.LedgerHeader + ledgerSeq = uint32(ledgerHeader.Header.LedgerSeq) + ledgerHash = ledgerMetadata.V2.LedgerHeader.Hash + default: + return nil, fmt.Errorf("unsupported LedgerCloseMeta version") + } + + ledgerCloseTime := int64(ledgerHeader.Header.ScpValue.CloseTime) + + transactions, err := f.extractTransactionsFromLedgerMetadata(ledgerMetadata) + if err != nil { + return nil, fmt.Errorf("extracting transactions: %w", err) + } + + stellarTransactions := make([]*pbstellar.Transaction, 0) + for i, trx := range transactions { + txHashBytes, err := hex.DecodeString(trx.TxHash) + if err != nil { + return nil, fmt.Errorf("decoding tx hash %s: %w", trx.TxHash, err) + } + envelopeXdr, err := base64.StdEncoding.DecodeString(trx.EnvelopeXdr) + if err != nil { + return nil, fmt.Errorf("decoding envelope XDR: %w", err) + } + resultXdr, err := base64.StdEncoding.DecodeString(trx.ResultXdr) + if err != nil { + return nil, fmt.Errorf("decoding result XDR: %w", err) + } + + events := &pbstellar.Events{} + if trx.Events != nil { + diagnosticEvents := make([][]byte, 0) + for _, event := range trx.Events.DiagnosticEventsXdr { + decodedEvent, err := base64.StdEncoding.DecodeString(event) + if err != nil { + return nil, fmt.Errorf("decoding diagnostic event: %w", err) + } + diagnosticEvents = append(diagnosticEvents, decodedEvent) + } + + transactionsEvents := make([][]byte, 0) + for _, event := range trx.Events.TransactionEventsXdr { + decodedEvent, err := base64.StdEncoding.DecodeString(event) + if err != nil { + return nil, fmt.Errorf("decoding transaction event: %w", err) + } + transactionsEvents = append(transactionsEvents, decodedEvent) + } + + contractEvents := make([]*pbstellar.ContractEvent, 0) + for _, eventsGroup := range trx.Events.ContractEventsXdr { + innerContractEvents := make([][]byte, 0) + for _, event := range eventsGroup { + decodedEvent, err := base64.StdEncoding.DecodeString(event) + if err != nil { + return nil, fmt.Errorf("decoding contract event: %w", err) + } + innerContractEvents = append(innerContractEvents, decodedEvent) + } + contractEvents = append(contractEvents, &pbstellar.ContractEvent{ + Events: innerContractEvents, + }) + } + + events.DiagnosticEventsXdr = diagnosticEvents + events.TransactionEventsXdr = transactionsEvents + events.ContractEventsXdr = contractEvents + } + + stellarTransactions = append(stellarTransactions, &pbstellar.Transaction{ + Hash: txHashBytes, + Status: utils.ConvertTransactionStatus(trx.Status), + CreatedAt: timestamppb.New(time.Unix(ledgerCloseTime, 0)), + ApplicationOrder: uint64(i + 1), + EnvelopeXdr: envelopeXdr, + ResultXdr: resultXdr, + Events: events, + }) + } + + previousLedgerHash := ledgerHeader.Header.PreviousLedgerHash[:] + + stellarBlk := &pbstellar.Block{ + Number: uint64(ledgerSeq), + Hash: ledgerHash[:], + Header: &pbstellar.Header{ + LedgerVersion: uint32(ledgerHeader.Header.LedgerVersion), + PreviousLedgerHash: previousLedgerHash, + TotalCoins: int64(ledgerHeader.Header.TotalCoins), + BaseFee: uint32(ledgerHeader.Header.BaseFee), + BaseReserve: uint32(ledgerHeader.Header.BaseReserve), + }, + Version: 1, + Transactions: stellarTransactions, + CreatedAt: timestamppb.New(time.Unix(ledgerCloseTime, 0)), + } + + return f.convertStellarBlockToBstreamBlock(stellarBlk) +} + +func (f *Fetcher) extractTransactionsFromLedgerMetadata(ledgerMetadata *xdr.LedgerCloseMeta) ([]types.Transaction, error) { + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(f.NetworkPassphrase, *ledgerMetadata) + if err != nil { + return nil, fmt.Errorf("failed to create ledger transaction reader: %w", err) + } + defer reader.Close() + + transactions := make([]types.Transaction, 0) + for { + tx, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("failed to read transaction: %w", err) + } + + transaction, err := f.convertLedgerTransactionToTypes(tx) + if err != nil { + return nil, fmt.Errorf("failed to convert transaction: %w", err) + } + + transactions = append(transactions, *transaction) + } + + return transactions, nil +} + +func (f *Fetcher) convertLedgerTransactionToTypes(tx ingest.LedgerTransaction) (*types.Transaction, error) { + envelopeXdr, err := tx.Envelope.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("failed to marshal envelope: %w", err) + } + envelopeXdrStr := base64.StdEncoding.EncodeToString(envelopeXdr) + + resultXdr, err := tx.Result.Result.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("failed to marshal result: %w", err) + } + resultXdrStr := base64.StdEncoding.EncodeToString(resultXdr) + + txHash := tx.Result.TransactionHash.HexString() + + status := "UNKNOWN" + if tx.Successful() { + status = "SUCCESS" + } else { + status = "FAILED" + } + + events, err := f.convertLedgerTransactionEventsToRPCEvents(tx) + if err != nil { + f.Logger.Warn("failed to convert events", zap.Error(err)) + events = nil + } + + return &types.Transaction{ + TxHash: txHash, + EnvelopeXdr: envelopeXdrStr, + ResultXdr: resultXdrStr, + Status: status, + Events: events, + }, nil +} + +func (f *Fetcher) convertLedgerTransactionEventsToRPCEvents(tx ingest.LedgerTransaction) (*types.RPCEvents, error) { + rpcEvents := &types.RPCEvents{ + DiagnosticEventsXdr: make([]string, 0), + TransactionEventsXdr: make([]string, 0), + ContractEventsXdr: make([][]string, 0), + } + + diagnosticEvents, err := tx.GetDiagnosticEvents() + if err != nil { + return nil, fmt.Errorf("failed to get diagnostic events: %w", err) + } + for _, event := range diagnosticEvents { + eventXdr, err := event.MarshalBinary() + if err != nil { + continue + } + rpcEvents.DiagnosticEventsXdr = append(rpcEvents.DiagnosticEventsXdr, base64.StdEncoding.EncodeToString(eventXdr)) + } + + transactionEvents, err := tx.GetTransactionEvents() + if err != nil { + return nil, fmt.Errorf("failed to get transaction events: %w", err) + } + for _, event := range transactionEvents.TransactionEvents { + eventXdr, err := event.MarshalBinary() + if err != nil { + continue + } + rpcEvents.TransactionEventsXdr = append(rpcEvents.TransactionEventsXdr, base64.StdEncoding.EncodeToString(eventXdr)) + } + + for _, operationEvents := range transactionEvents.OperationEvents { + operationEventStrings := make([]string, 0, len(operationEvents)) + if operationEvents == nil { + continue + } + for _, event := range operationEvents { + eventXdr, err := event.MarshalBinary() + if err != nil { + continue + } + operationEventStrings = append(operationEventStrings, base64.StdEncoding.EncodeToString(eventXdr)) + } + rpcEvents.ContractEventsXdr = append(rpcEvents.ContractEventsXdr, operationEventStrings) + } + + return rpcEvents, nil +} + +func (f *Fetcher) convertStellarBlockToBstreamBlock(stellarBlk *pbstellar.Block) (*pbbstream.Block, error) { + anyBlock, err := anypb.New(stellarBlk) + if err != nil { + return nil, fmt.Errorf("unable to create anypb: %w", err) + } + + // Hex-encode Id / ParentId so they're safe for one-block-file names + // (firecore mindreader splits on '/'). pbstellar.Block.Hash stays raw. + stellarBlockHash := hex.EncodeToString(stellarBlk.Hash) + previousStellarBlockHash := hex.EncodeToString(stellarBlk.Header.PreviousLedgerHash) + + return &pbbstream.Block{ + Number: stellarBlk.Number, + Id: stellarBlockHash, + ParentId: previousStellarBlockHash, + Timestamp: stellarBlk.CreatedAt, + LibNum: stellarBlk.Number - 1, + ParentNum: stellarBlk.Number - 1, + Payload: anyBlock, + }, nil +} diff --git a/cmd/firestellar/fetcher.go b/cmd/firestellar/fetcher.go index c4ce7db..5c76d62 100644 --- a/cmd/firestellar/fetcher.go +++ b/cmd/firestellar/fetcher.go @@ -6,6 +6,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/stellar/go-stellar-sdk/network" "github.com/streamingfast/cli/sflags" firecore "github.com/streamingfast/firehose-core" "github.com/streamingfast/firehose-core/blockpoller" @@ -15,26 +16,35 @@ import ( "go.uber.org/zap" ) -func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { +func NewFetchRpcCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd := &cobra.Command{ Use: "rpc ", Short: "fetch blocks from rpc endpoint", Args: cobra.ExactArgs(1), - RunE: fetchRunE(logger, tracer), + RunE: fetchRpcRunE(logger, tracer), } cmd.Flags().StringArray("endpoints", []string{}, "List of endpoints to use to fetch different method calls") - cmd.Flags().String("state-dir", "/data/poller", "interval between fetch") - cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch") - cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch") + cmd.Flags().String("state-dir", "/data/poller", "directory used to persist poller state between runs") + cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch attempts when the chain head has not advanced") + cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval to wait before retrying after a failed latest-block fetch") cmd.Flags().Duration("max-block-fetch-duration", 3*time.Second, "maximum delay before considering a block fetch as failed") cmd.Flags().Int("block-fetch-batch-size", 1, "Number of blocks to fetch in a single batch") cmd.Flags().Int("transaction-fetch-limit", 200, "Maximum number of transactions to fetch at the same time") - cmd.Flags().Bool("is-mainnet", true, "This is for passphrase selection") + cmd.Flags().String("stellar-rpc-network", "mainnet", "stellar network the rpc endpoint serves (mainnet, testnet, or custom)") + cmd.Flags().String("stellar-rpc-network-passphrase", "", "override network passphrase (required for custom; overrides the value derived from --stellar-rpc-network when set)") + + // Deprecated: --is-mainnet was the original flag and is kept for + // backwards compatibility. Prefer --stellar-rpc-network=mainnet|testnet + // or --stellar-rpc-network-passphrase=... for explicit control. When + // both are set, the new flags win. + cmd.Flags().Bool("is-mainnet", false, "DEPRECATED: use --stellar-rpc-network=mainnet|testnet instead") + _ = cmd.Flags().MarkDeprecated("is-mainnet", "use --stellar-rpc-network=mainnet|testnet instead") + return cmd } -func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecutor { +func fetchRpcRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecutor { return func(cmd *cobra.Command, args []string) (err error) { stateDir := sflags.MustGetString(cmd, "state-dir") @@ -46,7 +56,11 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch") latestBlockRetryInterval := sflags.MustGetDuration(cmd, "latest-block-retry-interval") maxBlockFetchDuration := sflags.MustGetDuration(cmd, "max-block-fetch-duration") - isMainnet := sflags.MustGetBool(cmd, "is-mainnet") + + networkPassphrase, err := resolveRPCNetworkPassphrase(cmd) + if err != nil { + return err + } logger.Info( "launching firehose-stellar poller", @@ -54,6 +68,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut zap.Uint64("first_streamable_block", startBlock), zap.Duration("interval_between_fetch", fetchInterval), zap.Duration("latest_block_retry_interval", latestBlockRetryInterval), + zap.String("network_passphrase", networkPassphrase), ) rollingStrategy := firecoreRPC.NewStickyRollingStrategy[*rpc.Client]() @@ -68,7 +83,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut transactionFetchLimit := sflags.MustGetInt(cmd, "transaction-fetch-limit") poller := blockpoller.New( - rpc.NewFetcher(fetchInterval, latestBlockRetryInterval, transactionFetchLimit, isMainnet, logger), + rpc.NewFetcher(fetchInterval, latestBlockRetryInterval, transactionFetchLimit, networkPassphrase, logger), blockpoller.NewFireBlockHandler("type.googleapis.com/sf.stellar.type.v1.Block"), rpcClients, blockpoller.WithStoringState[*rpc.Client](stateDir), @@ -83,3 +98,42 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut return nil } } + +// resolveRPCNetworkPassphrase derives the network passphrase to use for +// the rpc fetcher. Resolution order, highest precedence first: +// +// 1. --stellar-rpc-network-passphrase= (explicit override) +// 2. --stellar-rpc-network=mainnet|testnet|custom +// 3. --is-mainnet (deprecated; only consulted if the new flags are +// untouched) +// +// `custom` requires --stellar-rpc-network-passphrase to also be set. +func resolveRPCNetworkPassphrase(cmd *cobra.Command) (string, error) { + networkName := sflags.MustGetString(cmd, "stellar-rpc-network") + override := sflags.MustGetString(cmd, "stellar-rpc-network-passphrase") + + // Explicit override always wins. + if override != "" { + return override, nil + } + + // Back-compat: if the user explicitly set --is-mainnet and did not + // override --stellar-rpc-network, honor the deprecated flag. + if cmd.Flags().Changed("is-mainnet") && !cmd.Flags().Changed("stellar-rpc-network") { + if sflags.MustGetBool(cmd, "is-mainnet") { + return network.PublicNetworkPassphrase, nil + } + return network.TestNetworkPassphrase, nil + } + + switch networkName { + case "mainnet": + return network.PublicNetworkPassphrase, nil + case "testnet": + return network.TestNetworkPassphrase, nil + case "custom": + return "", fmt.Errorf("--stellar-rpc-network-passphrase is required when --stellar-rpc-network=custom") + default: + return "", fmt.Errorf("unsupported stellar rpc network: %s (want mainnet|testnet|custom)", networkName) + } +} diff --git a/cmd/firestellar/fetcher_captive_core.go b/cmd/firestellar/fetcher_captive_core.go new file mode 100644 index 0000000..ef71e49 --- /dev/null +++ b/cmd/firestellar/fetcher_captive_core.go @@ -0,0 +1,161 @@ +// Cobra wrapper around the captivecore package. All meaningful logic +// lives in github.com/streamingfast/firehose-stellar/captivecore — this +// file just parses flags into captivecore.Config and runs the +// PrepareRange + GetBlock loop that firecore expects. +package main + +import ( + "fmt" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/streamingfast/cli/sflags" + firecore "github.com/streamingfast/firehose-core" + "github.com/streamingfast/firehose-core/blockpoller" + "github.com/streamingfast/firehose-stellar/captivecore" + "github.com/streamingfast/firehose-stellar/cursor" + "github.com/streamingfast/logging" + "go.uber.org/zap" +) + +func NewFetchCaptiveCoreCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { + cmd := &cobra.Command{ + Use: "captive-core ", + Short: "fetch blocks from stellar captive core", + Args: cobra.ExactArgs(1), + RunE: fetchCaptiveCoreRunE(logger, tracer), + } + + cmd.Flags().String("stellar-core-bin", "/usr/bin/stellar-core", "path to stellar-core binary") + cmd.Flags().String("stellar-core-conf", "", "path to stellar-core config file (empty = use bundled SDF default for the network; required for custom)") + cmd.Flags().String("stellar-core-network", "testnet", "stellar network (mainnet, testnet, or custom)") + cmd.Flags().String("stellar-core-network-passphrase", "", "override network passphrase (required for custom; overrides the value derived from --stellar-core-network when set)") + cmd.Flags().StringSlice("stellar-core-history-archive-urls", nil, "override history archive URLs (required for custom; overrides the values derived from --stellar-core-network when set)") + cmd.Flags().String("stellar-core-log-level", "info", "log level for stellar-core subprocess (debug, info, warn, error)") + cmd.Flags().String("state-dir", "/data/work", "directory used to persist the last-fired block (cursor.json) so restarts resume where they stopped") + cmd.Flags().Bool("ignore-cursor", false, "ignore any persisted cursor.json and start from ") + + return cmd +} + +func fetchCaptiveCoreRunE(logger *zap.Logger, _ logging.Tracer) firecore.CommandExecutor { + return func(cmd *cobra.Command, args []string) error { + startBlock, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + return fmt.Errorf("unable to parse first streamable block %s: %w", args[0], err) + } + + logLevel, err := parseStellarCoreLogLevel(sflags.MustGetString(cmd, "stellar-core-log-level")) + if err != nil { + return err + } + + // Build the captivecore.Config from flags. ResolveNetwork fills + // defaults for mainnet/testnet; explicit overrides still win + // because we re-apply them after the call. + cfg := captivecore.Config{ + BinaryPath: sflags.MustGetString(cmd, "stellar-core-bin"), + StellarCoreConfPath: sflags.MustGetString(cmd, "stellar-core-conf"), + LogLevel: logLevel, + Logger: logger, + } + if err := cfg.ResolveNetwork(sflags.MustGetString(cmd, "stellar-core-network")); err != nil { + return err + } + if pass := sflags.MustGetString(cmd, "stellar-core-network-passphrase"); pass != "" { + cfg.NetworkPassphrase = pass + } + if urls := sflags.MustGetStringSlice(cmd, "stellar-core-history-archive-urls"); len(urls) > 0 { + cfg.HistoryArchiveURLs = urls + } + + // For custom networks, the bundled toml data is nil. The user + // must supply --stellar-core-conf in that case (captivecore + // validation also enforces this). + if cfg.StellarCoreConfPath == "" && cfg.DefaultTomlData == nil { + return fmt.Errorf("--stellar-core-conf is required for custom network (no bundled default)") + } + + backend, err := captivecore.New(cfg) + if err != nil { + return err + } + defer backend.Close() + + handler := blockpoller.NewFireBlockHandler("type.googleapis.com/sf.stellar.type.v1.Block") + handler.Init() + + stateDir := sflags.MustGetString(cmd, "state-dir") + ignoreCursor := sflags.MustGetBool(cmd, "ignore-cursor") + + seq := startBlock + if !ignoreCursor { + persisted, err := cursor.Load(stateDir) + if err != nil { + return fmt.Errorf("loading cursor: %w", err) + } + if persisted != nil { + resumeFrom := persisted.LastFiredBlock.Num + 1 + if resumeFrom > startBlock { + seq = resumeFrom + logger.Info("resuming from persisted cursor", + zap.String("state_dir", stateDir), + zap.Uint64("last_fired_block", persisted.LastFiredBlock.Num), + zap.Uint64("resume_block", seq), + ) + } else { + logger.Info("persisted cursor is below first streamable block, ignoring", + zap.Uint64("last_fired_block", persisted.LastFiredBlock.Num), + zap.Uint64("first_streamable_block", startBlock), + ) + } + } + } + + ctx := cmd.Context() + if err := backend.PrepareRange(ctx, seq); err != nil { + return err + } + + for { + if err := ctx.Err(); err != nil { + return err + } + + blk, err := backend.GetBlock(ctx, seq) + if err != nil { + return fmt.Errorf("get block %d: %w", seq, err) + } + + logger.Info("processing block", zap.Uint64("seq", seq), zap.String("hash", blk.Id)) + if err := handler.Handle(blk); err != nil { + return fmt.Errorf("handling block %d: %w", blk.Number, err) + } + + if err := cursor.Save(stateDir, blk); err != nil { + return fmt.Errorf("saving cursor at block %d: %w", blk.Number, err) + } + + seq++ + } + } +} + +// parseStellarCoreLogLevel translates the CLI flag string into a +// logrus.Level. Kept here so the cmd shim is self-contained. +func parseStellarCoreLogLevel(s string) (logrus.Level, error) { + switch strings.ToLower(s) { + case "debug": + return logrus.DebugLevel, nil + case "info": + return logrus.InfoLevel, nil + case "warn", "warning": + return logrus.WarnLevel, nil + case "error": + return logrus.ErrorLevel, nil + default: + return 0, fmt.Errorf("invalid stellar-core log level %q (want debug|info|warn|error)", s) + } +} diff --git a/cmd/firestellar/main.go b/cmd/firestellar/main.go index c657c7a..4d8e27c 100644 --- a/cmd/firestellar/main.go +++ b/cmd/firestellar/main.go @@ -4,6 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/streamingfast/cli" . "github.com/streamingfast/cli" + "github.com/streamingfast/firehose-stellar/cmd/tools/fix" "github.com/streamingfast/logging" "go.uber.org/zap" ) @@ -37,8 +38,13 @@ func main() { ConfigureVersion(version), ConfigureViper("FIRESTELLAR"), - Group("fetch", "Reader Node fetch RPC command", - CobraCmd(NewFetchCmd(logger, tracer)), + Group("fetch", "Reader Node block fetchers (rpc, captive-core)", + CobraCmd(NewFetchRpcCmd(logger, tracer)), + CobraCmd(NewFetchCaptiveCoreCmd(logger, tracer)), + ), + + Group("fix", "One-shot maintenance commands for stored blocks", + CobraCmd(fix.NewToolsFixBlockHashesCmd(logger)), ), CobraCmd(NewToolDecodeBlockCmd()), @@ -48,6 +54,7 @@ func main() { CobraCmd(NewToolSendPaymentAssetCmd()), CobraCmd(NewToolDecodeSeedCmd()), CobraCmd(NewToolCompareFetcherBlocksCmd()), + CobraCmd(NewToolCompareMergedBlocksCmd(logger)), OnCommandErrorLogAndExit(logger), ) diff --git a/cmd/firestellar/tool_compare_fetcher_blocks.go b/cmd/firestellar/tool_compare_fetcher_blocks.go index f195b4b..22f3ff7 100644 --- a/cmd/firestellar/tool_compare_fetcher_blocks.go +++ b/cmd/firestellar/tool_compare_fetcher_blocks.go @@ -9,6 +9,7 @@ import ( "time" "github.com/spf13/cobra" + stellarnetwork "github.com/stellar/go-stellar-sdk/network" "github.com/streamingfast/bstream" pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" "github.com/streamingfast/dstore" @@ -114,12 +115,24 @@ func runCompareFetcherBlocksE(cmd *cobra.Command, args []string) error { return nil } -func fetchBlockViaFetcher(ctx context.Context, blockNum uint64, rpcEndpoint, network string, logger *zap.Logger) (*pbbstream.Block, error) { +func fetchBlockViaFetcher(ctx context.Context, blockNum uint64, rpcEndpoint, networkName string, logger *zap.Logger) (*pbbstream.Block, error) { // Create a client client := rpc.NewClient(rpcEndpoint, logger, nil) + // Resolve the network passphrase from the --network flag (was previously + // hardcoded to mainnet, regardless of the flag value). + var passphrase string + switch networkName { + case "mainnet": + passphrase = stellarnetwork.PublicNetworkPassphrase + case "testnet": + passphrase = stellarnetwork.TestNetworkPassphrase + default: + return nil, fmt.Errorf("unsupported network %q for compare-fetcher-blocks (want mainnet|testnet)", networkName) + } + // Create a Fetcher instance - fetcher := rpc.NewFetcher(0, time.Second, 200, true, logger) + fetcher := rpc.NewFetcher(0, time.Second, 200, passphrase, logger) // Fetch the block block, skipped, err := fetcher.Fetch(ctx, client, blockNum) diff --git a/cmd/firestellar/tool_compare_merged_blocks.go b/cmd/firestellar/tool_compare_merged_blocks.go new file mode 100644 index 0000000..6429049 --- /dev/null +++ b/cmd/firestellar/tool_compare_merged_blocks.go @@ -0,0 +1,702 @@ +// tool-compare-merged-blocks diffs two stellar merged-block stores +// over a block range. Modeled on firehose-core's compare-blocks but +// scoped to merged-block (100-block) bundles and aware of the legacy +// v1 RPC fetcher's broken hash encoding (see cmd/tools/fix). +// +// Either side can be flagged as "broken" via --sanitize-reference / +// --sanitize-current. A sanitized side has its pbstellar.Block.Hash +// and Header.PreviousLedgerHash recovered via fix.ConvertBrokenHash, +// with pbbstream.Block.Id / ParentId recomputed as hex of those bytes +// and the Payload re-marshalled, before the comparison runs. +package main + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "strconv" + "sync" + + "github.com/spf13/cobra" + "github.com/streamingfast/bstream" + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + "github.com/streamingfast/cli/sflags" + "github.com/streamingfast/dstore" + "github.com/streamingfast/firehose-core/cmd/tools/check" + fctypes "github.com/streamingfast/firehose-core/types" + "github.com/streamingfast/firehose-stellar/cmd/tools/fix" + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" + "github.com/streamingfast/firehose-stellar/utils" + "go.uber.org/zap" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const mergedBundleSize = uint64(100) + +func NewToolCompareMergedBlocksCmd(logger *zap.Logger) *cobra.Command { + cmd := &cobra.Command{ + Use: "tool-compare-merged-blocks ", + Short: "Compare stellar merged-block bundles across two stores over a block range", + Long: `Walks 100-block merged-block bundles in both stores over the given +range and reports any blocks that differ. Useful for validating a new +fetcher (e.g. captive-core) against the existing stored output. + +Either side can be flagged as broken via --sanitize-reference or +--sanitize-current. A sanitized side has its pbstellar.Block.Hash and +Header.PreviousLedgerHash run through fix.ConvertBrokenHash (the same +recovery used by the fix-block-hashes tool), and its bstream Id / +ParentId regenerated as hex of those recovered bytes, before the +comparison. Use this to compare legacy v1-RPC-fetcher output against +correctly-hashed blocks. + +Arguments: + reference_store dstore URL (gs://, file://, ...) — left side + current_store dstore URL — right side + block_range e.g. "100:200", "0:16000000", or single block "60132634"`, + Args: cobra.ExactArgs(3), + RunE: runCompareMergedBlocksE(logger), + Example: `firestellar tool-compare-merged-blocks \ + gs://old-v1-store/stellar-mainnet/v1 \ + gs://captive-core-store/stellar-mainnet/v2 \ + 60132600:60132700 \ + --sanitize-reference`, + } + + cmd.Flags().Bool("sanitize-reference", false, "treat reference store as legacy v1-RPC output: recover Hash / PreviousLedgerHash via fix.ConvertBrokenHash before comparing") + cmd.Flags().Bool("sanitize-current", false, "treat current store as legacy v1-RPC output: recover Hash / PreviousLedgerHash via fix.ConvertBrokenHash before comparing") + cmd.Flags().Bool("diff", false, "print JSON diff (protojson) of each differing block") + cmd.Flags().Bool("stop-on-first-diff", false, "stop walking as soon as the first differing block is found") + cmd.Flags().Bool("oneblock", false, "[test] read per-block oneblock dbin files instead of 100-block merged bundles") + cmd.Flags().Bool("ignore-nondeterministic", false, "strip diagnostic events that are wall-clock dependent (core_metrics.invoke_time_nsecs) from both sides before comparing; stored data is left intact") + + return cmd +} + +func runCompareMergedBlocksE(logger *zap.Logger) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + refStore, err := dstore.NewDBinStore(args[0]) + if err != nil { + return fmt.Errorf("creating reference store: %w", err) + } + curStore, err := dstore.NewDBinStore(args[1]) + if err != nil { + return fmt.Errorf("creating current store: %w", err) + } + + blockRange, err := fctypes.GetBlockRangeFromArg(args[2]) + if err != nil { + return fmt.Errorf("parsing block range: %w", err) + } + // Allow single-block argument: "60132634" -> [60132634, 60132635) + if blockRange.IsOpen() && blockRange.Start >= 0 { + n := uint64(blockRange.Start) + blockRange = fctypes.NewClosedRange(int64(n), n+1) + } + if !blockRange.IsResolved() { + return fmt.Errorf("block range must be closed (got %s)", blockRange.String()) + } + + sanitizeRef := sflags.MustGetBool(cmd, "sanitize-reference") + sanitizeCur := sflags.MustGetBool(cmd, "sanitize-current") + showDiff := sflags.MustGetBool(cmd, "diff") + stopOnFirstDiff := sflags.MustGetBool(cmd, "stop-on-first-diff") + useOneblock := sflags.MustGetBool(cmd, "oneblock") + ignoreNonDet := sflags.MustGetBool(cmd, "ignore-nondeterministic") + + stopBlock := blockRange.MustGetStopBlock() + startBlock := uint64(blockRange.Start) + + fmt.Printf("Comparing merged blocks [%d, %d)\n", startBlock, stopBlock) + fmt.Printf(" Reference: %s%s\n", args[0], sanitizeNote(sanitizeRef)) + fmt.Printf(" Current: %s%s\n", args[1], sanitizeNote(sanitizeCur)) + + if useOneblock { + return runOneblockCompare(ctx, refStore, curStore, startBlock, stopBlock, sanitizeRef, sanitizeCur, ignoreNonDet, showDiff, stopOnFirstDiff) + } + + var totalCompared, totalDifferent, totalMissingInCur, totalMissingInRef int + var stopErr = errors.New("stop-on-first-diff") + + // Bundles already handled by the reference walk; the follow-up + // current-store walk skips them and only reports bundles that + // exist exclusively in the current store. + visited := map[string]bool{} + + walkErr := refStore.Walk(ctx, check.WalkBlockPrefix(blockRange, mergedBundleSize), func(filename string) error { + fileStart, err := strconv.ParseUint(filename, 10, 64) + if err != nil { + // Non-bundle file (one-block file etc); skip. + return nil + } + if fileStart >= stopBlock { + return dstore.StopIteration + } + // Bundle overlap with requested range. + if fileStart+mergedBundleSize <= startBlock { + return nil + } + visited[filename] = true + logger.Debug("comparing bundle", zap.String("file", filename), zap.Uint64("file_start", fileStart)) + + var ( + wg sync.WaitGroup + refMap map[uint64]*pbbstream.Block + curMap map[uint64]*pbbstream.Block + refErr error + curErr error + ) + wg.Add(2) + go func() { + defer wg.Done() + refMap, refErr = readMergedBundle(ctx, refStore, filename, startBlock, stopBlock, sanitizeRef, ignoreNonDet) + }() + go func() { + defer wg.Done() + exists, existsErr := curStore.FileExists(ctx, filename) + if existsErr != nil { + curErr = fmt.Errorf("checking current bundle %s: %w", filename, existsErr) + return + } + if !exists { + curMap = map[uint64]*pbbstream.Block{} + return + } + curMap, curErr = readMergedBundle(ctx, curStore, filename, startBlock, stopBlock, sanitizeCur, ignoreNonDet) + }() + wg.Wait() + if refErr != nil { + return fmt.Errorf("reading reference bundle %s: %w", filename, refErr) + } + if curErr != nil { + return fmt.Errorf("reading current bundle %s: %w", filename, curErr) + } + + // Compare every reference block to its current counterpart. + for blockNum := max(startBlock, fileStart); blockNum < min(stopBlock, fileStart+mergedBundleSize); blockNum++ { + refBlk, refOK := refMap[blockNum] + curBlk, curOK := curMap[blockNum] + + switch { + case !refOK && !curOK: + continue + case !refOK: + totalMissingInRef++ + fmt.Printf("- Block %d missing in reference (present in current)\n", blockNum) + case !curOK: + totalMissingInCur++ + fmt.Printf("- Block %d missing in current (present in reference)\n", blockNum) + default: + totalCompared++ + diffs, refStellar, curStellar := compareSingleBlock(refBlk, curBlk) + if len(diffs) == 0 { + continue + } + totalDifferent++ + shortRef := refBlk.Id + if len(shortRef) > 12 { + shortRef = shortRef[:12] + "..." + } + fmt.Printf("- Block %d differs (ref id=%s): %d field(s)\n", blockNum, shortRef, len(diffs)) + for _, d := range diffs { + fmt.Printf(" · %s\n", d) + } + if showDiff { + printJSONDiff(blockNum, refStellar, curStellar) + } + if stopOnFirstDiff { + return stopErr + } + } + } + return nil + }) + if walkErr != nil && !errors.Is(walkErr, stopErr) { + return fmt.Errorf("walking reference bundles: %w", walkErr) + } + + // Catch bundles that exist only in the current store — the + // reference walk alone would silently skip them and the tool + // would falsely report a clean diff. + if !errors.Is(walkErr, stopErr) { + curOnlyErr := curStore.Walk(ctx, check.WalkBlockPrefix(blockRange, mergedBundleSize), func(filename string) error { + if visited[filename] { + return nil + } + fileStart, err := strconv.ParseUint(filename, 10, 64) + if err != nil { + return nil + } + if fileStart >= stopBlock { + return dstore.StopIteration + } + if fileStart+mergedBundleSize <= startBlock { + return nil + } + logger.Debug("current-only bundle", zap.String("file", filename), zap.Uint64("file_start", fileStart)) + + curMap, err := readMergedBundle(ctx, curStore, filename, startBlock, stopBlock, sanitizeCur, ignoreNonDet) + if err != nil { + return fmt.Errorf("reading current-only bundle %s: %w", filename, err) + } + for blockNum := max(startBlock, fileStart); blockNum < min(stopBlock, fileStart+mergedBundleSize); blockNum++ { + if _, ok := curMap[blockNum]; !ok { + continue + } + totalMissingInRef++ + fmt.Printf("- Block %d missing in reference (present in current)\n", blockNum) + if stopOnFirstDiff { + return stopErr + } + } + return nil + }) + if curOnlyErr != nil && !errors.Is(curOnlyErr, stopErr) { + return fmt.Errorf("walking current-only bundles: %w", curOnlyErr) + } + } + + fmt.Println() + fmt.Printf("Summary: %d compared, %d different, %d missing in current, %d missing in reference\n", + totalCompared, totalDifferent, totalMissingInCur, totalMissingInRef) + if totalDifferent == 0 && totalMissingInCur == 0 && totalMissingInRef == 0 { + fmt.Println("✅ Block ranges match.") + } + + return nil + } +} + +func sanitizeNote(on bool) string { + if on { + return " (sanitize: ConvertBrokenHash)" + } + return "" +} + +// readMergedBundle reads a 100-block merged file and returns a map +// keyed by block number. Blocks outside [startBlock, stopBlock) are +// dropped. When sanitize is true, broken Hash / PreviousLedgerHash are +// recovered and bstream Id/ParentId rewritten before the block lands +// in the map. +func readMergedBundle(ctx context.Context, store dstore.Store, filename string, startBlock, stopBlock uint64, sanitize, stripNonDet bool) (map[uint64]*pbbstream.Block, error) { + reader, err := store.OpenObject(ctx, filename) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", filename, err) + } + defer reader.Close() + + blockReader, err := bstream.NewDBinBlockReader(reader) + if err != nil { + return nil, fmt.Errorf("creating block reader: %w", err) + } + + out := make(map[uint64]*pbbstream.Block) + for { + blk, err := blockReader.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("reading block: %w", err) + } + if blk.Number < startBlock || blk.Number >= stopBlock { + continue + } + if sanitize { + if err := sanitizeBlockInPlace(blk); err != nil { + return nil, fmt.Errorf("sanitizing block %d in %s: %w", blk.Number, filename, err) + } + } + if stripNonDet { + if err := stripNonDeterministicInPlace(blk); err != nil { + return nil, fmt.Errorf("stripping non-deterministic events from block %d in %s: %w", blk.Number, filename, err) + } + } + out[blk.Number] = blk + } + return out, nil +} + +// sanitizeBlockInPlace runs fix.ConvertBrokenHash on the stellar +// payload's Hash and PreviousLedgerHash, then rewrites bstream Id / +// ParentId to match and re-marshals the payload. +func sanitizeBlockInPlace(blk *pbbstream.Block) error { + var stellarBlk pbstellar.Block + if err := blk.Payload.UnmarshalTo(&stellarBlk); err != nil { + return fmt.Errorf("unmarshalling payload: %w", err) + } + + if stellarBlk.Header == nil { + return fmt.Errorf("stellar block %d: nil Header (malformed or unexpected version)", stellarBlk.Number) + } + + recoveredHash, err := fix.ConvertBrokenHash(stellarBlk.Hash) + if err != nil { + return fmt.Errorf("recover Hash: %w", err) + } + recoveredPrev, err := fix.ConvertBrokenHash(stellarBlk.Header.PreviousLedgerHash) + if err != nil { + return fmt.Errorf("recover PreviousLedgerHash: %w", err) + } + + stellarBlk.Hash = recoveredHash + stellarBlk.Header.PreviousLedgerHash = recoveredPrev + + blk.Id = hex.EncodeToString(recoveredHash) + blk.ParentId = hex.EncodeToString(recoveredPrev) + + newPayload, err := anypb.New(&stellarBlk) + if err != nil { + return fmt.Errorf("re-marshalling stellar payload: %w", err) + } + blk.Payload = newPayload + return nil +} + +// stripNonDeterministicInPlace removes diagnostic events whose values +// vary across fetcher implementations (currently +// core_metrics.invoke_time_nsecs — wall-clock invocation duration) +// from the in-memory block, then re-marshals the payload so subsequent +// UnmarshalTo calls see the filtered version. Stored data on disk is +// left untouched. +func stripNonDeterministicInPlace(blk *pbbstream.Block) error { + var stellarBlk pbstellar.Block + if err := blk.Payload.UnmarshalTo(&stellarBlk); err != nil { + return fmt.Errorf("unmarshalling payload: %w", err) + } + + mutated := false + for _, tx := range stellarBlk.Transactions { + if tx.Events == nil || len(tx.Events.DiagnosticEventsXdr) == 0 { + continue + } + filtered := tx.Events.DiagnosticEventsXdr[:0] + for _, raw := range tx.Events.DiagnosticEventsXdr { + if utils.IsNonDeterministicDiagnosticEventBytes(raw) { + mutated = true + continue + } + filtered = append(filtered, raw) + } + tx.Events.DiagnosticEventsXdr = filtered + } + + if !mutated { + return nil + } + + newPayload, err := anypb.New(&stellarBlk) + if err != nil { + return fmt.Errorf("re-marshalling stellar payload: %w", err) + } + blk.Payload = newPayload + return nil +} + +// compareSingleBlock returns a list of differing fields between two +// bstream blocks plus the unmarshalled stellar payloads (for optional +// JSON diff printing). proto.Equal handles deep equality of the +// payload — we additionally surface a few top-level field names so +// the output is actionable. +func compareSingleBlock(ref, cur *pbbstream.Block) ([]string, *pbstellar.Block, *pbstellar.Block) { + var diffs []string + + if ref.Number != cur.Number { + diffs = append(diffs, fmt.Sprintf("bstream.Number: %d vs %d", ref.Number, cur.Number)) + } + if ref.Id != cur.Id { + diffs = append(diffs, fmt.Sprintf("bstream.Id: %s vs %s", ref.Id, cur.Id)) + } + if ref.ParentId != cur.ParentId { + diffs = append(diffs, fmt.Sprintf("bstream.ParentId: %s vs %s", ref.ParentId, cur.ParentId)) + } + + var refStellar, curStellar pbstellar.Block + if err := ref.Payload.UnmarshalTo(&refStellar); err != nil { + diffs = append(diffs, fmt.Sprintf("unmarshal reference payload: %s", err)) + return diffs, nil, nil + } + if err := cur.Payload.UnmarshalTo(&curStellar); err != nil { + diffs = append(diffs, fmt.Sprintf("unmarshal current payload: %s", err)) + return diffs, &refStellar, nil + } + + if !proto.Equal(&refStellar, &curStellar) { + // Top-level field hints. proto.Equal already told us they + // differ; these lines just say where. + if !bytesEq(refStellar.Hash, curStellar.Hash) { + diffs = append(diffs, fmt.Sprintf("pbstellar.Hash: %x vs %x", refStellar.Hash, curStellar.Hash)) + } + if refStellar.Header != nil && curStellar.Header != nil { + if !bytesEq(refStellar.Header.PreviousLedgerHash, curStellar.Header.PreviousLedgerHash) { + diffs = append(diffs, fmt.Sprintf("pbstellar.Header.PreviousLedgerHash: %x vs %x", refStellar.Header.PreviousLedgerHash, curStellar.Header.PreviousLedgerHash)) + } + if refStellar.Header.LedgerVersion != curStellar.Header.LedgerVersion { + diffs = append(diffs, fmt.Sprintf("pbstellar.Header.LedgerVersion: %d vs %d", refStellar.Header.LedgerVersion, curStellar.Header.LedgerVersion)) + } + if refStellar.Header.TotalCoins != curStellar.Header.TotalCoins { + diffs = append(diffs, fmt.Sprintf("pbstellar.Header.TotalCoins: %d vs %d", refStellar.Header.TotalCoins, curStellar.Header.TotalCoins)) + } + if refStellar.Header.BaseFee != curStellar.Header.BaseFee { + diffs = append(diffs, fmt.Sprintf("pbstellar.Header.BaseFee: %d vs %d", refStellar.Header.BaseFee, curStellar.Header.BaseFee)) + } + if refStellar.Header.BaseReserve != curStellar.Header.BaseReserve { + diffs = append(diffs, fmt.Sprintf("pbstellar.Header.BaseReserve: %d vs %d", refStellar.Header.BaseReserve, curStellar.Header.BaseReserve)) + } + } + if len(refStellar.Transactions) != len(curStellar.Transactions) { + diffs = append(diffs, fmt.Sprintf("pbstellar.Transactions count: %d vs %d", len(refStellar.Transactions), len(curStellar.Transactions))) + } else { + perTx := compareTransactionSlices(refStellar.Transactions, curStellar.Transactions) + diffs = append(diffs, perTx...) + } + + // Generic catch-all for anything we did not name above. + if len(diffs) == 0 { + diffs = append(diffs, "payloads differ (no top-level field surfaced; see --diff for full JSON)") + } + } + + return diffs, &refStellar, &curStellar +} + +// compareTransactionSlices matches transactions by hash and reports +// per-transaction field-level differences. Returns one entry per +// differing field so the user can see exactly what changed. +func compareTransactionSlices(ref, cur []*pbstellar.Transaction) []string { + var diffs []string + refByHash := make(map[string]*pbstellar.Transaction, len(ref)) + refIdx := make(map[string]int, len(ref)) + for i, tx := range ref { + h := fmt.Sprintf("%x", tx.Hash) + refByHash[h] = tx + refIdx[h] = i + } + curByHash := make(map[string]*pbstellar.Transaction, len(cur)) + for _, tx := range cur { + curByHash[fmt.Sprintf("%x", tx.Hash)] = tx + } + + for h, refTx := range refByHash { + curTx, ok := curByHash[h] + if !ok { + diffs = append(diffs, fmt.Sprintf("tx %s (index %d): missing in current", h, refIdx[h])) + continue + } + if !proto.Equal(refTx, curTx) { + if refTx.Status != curTx.Status { + diffs = append(diffs, fmt.Sprintf("tx %s (index %d): Status %s vs %s", h, refIdx[h], refTx.Status, curTx.Status)) + } + if refTx.ApplicationOrder != curTx.ApplicationOrder { + diffs = append(diffs, fmt.Sprintf("tx %s (index %d): ApplicationOrder %d vs %d", h, refIdx[h], refTx.ApplicationOrder, curTx.ApplicationOrder)) + } + if !bytesEq(refTx.EnvelopeXdr, curTx.EnvelopeXdr) { + diffs = append(diffs, fmt.Sprintf("tx %s (index %d): EnvelopeXdr differs", h, refIdx[h])) + } + if !bytesEq(refTx.ResultXdr, curTx.ResultXdr) { + diffs = append(diffs, fmt.Sprintf("tx %s (index %d): ResultXdr differs", h, refIdx[h])) + } + if !proto.Equal(refTx.Events, curTx.Events) { + diffs = append(diffs, fmt.Sprintf("tx %s (index %d): Events differ", h, refIdx[h])) + if refTx.Events == nil || curTx.Events == nil { + diffs = append(diffs, fmt.Sprintf(" tx %s: one side has nil Events (ref nil=%v cur nil=%v)", h, refTx.Events == nil, curTx.Events == nil)) + } else { + rTE, cTE := len(refTx.Events.TransactionEventsXdr), len(curTx.Events.TransactionEventsXdr) + rDE, cDE := len(refTx.Events.DiagnosticEventsXdr), len(curTx.Events.DiagnosticEventsXdr) + rCE, cCE := len(refTx.Events.ContractEventsXdr), len(curTx.Events.ContractEventsXdr) + if rTE != cTE { + diffs = append(diffs, fmt.Sprintf(" tx %s: TransactionEventsXdr count %d vs %d", h, rTE, cTE)) + } else { + for i := 0; i < rTE; i++ { + if !bytesEq(refTx.Events.TransactionEventsXdr[i], curTx.Events.TransactionEventsXdr[i]) { + diffs = append(diffs, fmt.Sprintf(" tx %s: TransactionEventsXdr[%d] differs (ref %d B / cur %d B)", h, i, len(refTx.Events.TransactionEventsXdr[i]), len(curTx.Events.TransactionEventsXdr[i]))) + } + } + } + if rDE != cDE { + diffs = append(diffs, fmt.Sprintf(" tx %s: DiagnosticEventsXdr count %d vs %d", h, rDE, cDE)) + } else { + for i := 0; i < rDE; i++ { + if !bytesEq(refTx.Events.DiagnosticEventsXdr[i], curTx.Events.DiagnosticEventsXdr[i]) { + diffs = append(diffs, fmt.Sprintf(" tx %s: DiagnosticEventsXdr[%d] differs (ref %d B / cur %d B)", h, i, len(refTx.Events.DiagnosticEventsXdr[i]), len(curTx.Events.DiagnosticEventsXdr[i]))) + diffs = append(diffs, fmt.Sprintf(" ref hex: %x", refTx.Events.DiagnosticEventsXdr[i])) + diffs = append(diffs, fmt.Sprintf(" cur hex: %x", curTx.Events.DiagnosticEventsXdr[i])) + } + } + } + if rCE != cCE { + diffs = append(diffs, fmt.Sprintf(" tx %s: ContractEventsXdr group count %d vs %d", h, rCE, cCE)) + } else { + for i := 0; i < rCE; i++ { + ri := refTx.Events.ContractEventsXdr[i] + ci := curTx.Events.ContractEventsXdr[i] + if len(ri.Events) != len(ci.Events) { + diffs = append(diffs, fmt.Sprintf(" tx %s: ContractEventsXdr[%d].Events count %d vs %d", h, i, len(ri.Events), len(ci.Events))) + continue + } + for j := 0; j < len(ri.Events); j++ { + if !bytesEq(ri.Events[j], ci.Events[j]) { + diffs = append(diffs, fmt.Sprintf(" tx %s: ContractEventsXdr[%d].Events[%d] differs (ref %d B / cur %d B)", h, i, j, len(ri.Events[j]), len(ci.Events[j]))) + } + } + } + } + } + } + } + } + for h, curTx := range curByHash { + if _, ok := refByHash[h]; !ok { + diffs = append(diffs, fmt.Sprintf("tx %x: missing in reference (present in current)", curTx.Hash)) + } + } + return diffs +} + +func printJSONDiff(blockNum uint64, ref, cur *pbstellar.Block) { + if ref == nil || cur == nil { + return + } + marshaller := protojson.MarshalOptions{Multiline: true, Indent: " ", EmitUnpopulated: false} + refJSON, err := marshaller.Marshal(ref) + if err != nil { + fmt.Printf(" ! marshalling reference: %s\n", err) + return + } + curJSON, err := marshaller.Marshal(cur) + if err != nil { + fmt.Printf(" ! marshalling current: %s\n", err) + return + } + fmt.Printf(" --- reference (block %d) ---\n%s\n", blockNum, string(refJSON)) + fmt.Printf(" --- current (block %d) ---\n%s\n", blockNum, string(curJSON)) +} + +// runOneblockCompare is a test-only path: reads per-block dbin files +// from both stores into in-memory maps, then runs the same +// compareSingleBlock used by the merged-bundle path. +func runOneblockCompare(ctx context.Context, refStore, curStore dstore.Store, startBlock, stopBlock uint64, sanitizeRef, sanitizeCur, stripNonDet, showDiff, stopOnFirstDiff bool) error { + refMap, err := loadOneblockRange(ctx, refStore, startBlock, stopBlock, sanitizeRef, stripNonDet) + if err != nil { + return fmt.Errorf("loading reference oneblocks: %w", err) + } + curMap, err := loadOneblockRange(ctx, curStore, startBlock, stopBlock, sanitizeCur, stripNonDet) + if err != nil { + return fmt.Errorf("loading current oneblocks: %w", err) + } + + var totalCompared, totalDifferent, totalMissingInCur, totalMissingInRef int + + for n := startBlock; n < stopBlock; n++ { + refBlk, refOK := refMap[n] + curBlk, curOK := curMap[n] + switch { + case !refOK && !curOK: + continue + case !refOK: + totalMissingInRef++ + fmt.Printf("- Block %d missing in reference (present in current)\n", n) + if stopOnFirstDiff { + goto done + } + case !curOK: + totalMissingInCur++ + fmt.Printf("- Block %d missing in current (present in reference)\n", n) + if stopOnFirstDiff { + goto done + } + default: + totalCompared++ + diffs, refS, curS := compareSingleBlock(refBlk, curBlk) + if len(diffs) == 0 { + continue + } + totalDifferent++ + shortRef := refBlk.Id + if len(shortRef) > 12 { + shortRef = shortRef[:12] + "..." + } + fmt.Printf("- Block %d differs (ref id=%s): %d field(s)\n", n, shortRef, len(diffs)) + for _, d := range diffs { + fmt.Printf(" · %s\n", d) + } + if showDiff { + printJSONDiff(n, refS, curS) + } + if stopOnFirstDiff { + goto done + } + } + } +done: + fmt.Println() + fmt.Printf("Summary: %d compared, %d different, %d missing in current, %d missing in reference\n", + totalCompared, totalDifferent, totalMissingInCur, totalMissingInRef) + if totalDifferent == 0 && totalMissingInCur == 0 && totalMissingInRef == 0 { + fmt.Println("✅ Block ranges match.") + } + return nil +} + +func loadOneblockRange(ctx context.Context, store dstore.Store, start, stop uint64, sanitize, stripNonDet bool) (map[uint64]*pbbstream.Block, error) { + out := make(map[uint64]*pbbstream.Block) + err := store.Walk(ctx, "", func(filename string) error { + if len(filename) < 10 { + return nil + } + n, err := strconv.ParseUint(filename[:10], 10, 64) + if err != nil { + return nil + } + if n < start { + return nil + } + if n >= stop { + return dstore.StopIteration + } + reader, err := store.OpenObject(ctx, filename) + if err != nil { + return fmt.Errorf("open %s: %w", filename, err) + } + defer reader.Close() + br, err := bstream.NewDBinBlockReader(reader) + if err != nil { + return fmt.Errorf("reader %s: %w", filename, err) + } + blk, err := br.Read() + if err != nil { + return fmt.Errorf("read %s: %w", filename, err) + } + if sanitize { + if err := sanitizeBlockInPlace(blk); err != nil { + return fmt.Errorf("sanitize block %d: %w", n, err) + } + } + if stripNonDet { + if err := stripNonDeterministicInPlace(blk); err != nil { + return fmt.Errorf("strip non-deterministic events from block %d: %w", n, err) + } + } + out[n] = blk + return nil + }) + return out, err +} + +func bytesEq(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/cmd/firestellar/tool_decode_block.go b/cmd/firestellar/tool_decode_block.go index 4e34333..56a34c6 100644 --- a/cmd/firestellar/tool_decode_block.go +++ b/cmd/firestellar/tool_decode_block.go @@ -1,5 +1,16 @@ package main +// FIXME(go-json-experiment): this file (and go.mod) pin +// github.com/go-json-experiment/json to the Oct 2023 snapshot +// (v0.0.0-20231013223334-54c864be5b8d) because that is the version +// firehose-core@v1.14.x still expects in its own +// firehose-core/json/marshallers.go. The Jan 2025+ snapshots renamed +// NewMarshalers -> JoinMarshalers and MarshalFuncV2 -> MarshalToFunc; +// keeping the old names here is a temporary shim. When firehose-core +// bumps its pin / renames its call sites, flip the names below back +// to JoinMarshalers / MarshalToFunc and bump go-json-experiment in +// go.mod. + import ( "encoding/hex" "errors" @@ -81,11 +92,11 @@ func runDecodeBlockE(cmd *cobra.Command, args []string) error { } out, err := json.Marshal(stellarBlock, - json.WithMarshalers(json.JoinMarshalers( - json.MarshalToFunc(marshalBytes), - json.MarshalToFunc(marshalAccountId), - json.MarshalToFunc(marshalTransaction), - json.MarshalToFunc(marshalMuxedAccount), + json.WithMarshalers(json.NewMarshalers( + json.MarshalFuncV2(marshalBytes), + json.MarshalFuncV2(marshalAccountId), + json.MarshalFuncV2(marshalTransaction), + json.MarshalFuncV2(marshalMuxedAccount), )), jsontext.WithIndent(" "), ) @@ -158,10 +169,10 @@ func marshalTransaction(e *jsontext.Encoder, value *pbstellar.Transaction, optio } out, err := json.Marshal(trx, - json.WithMarshalers(json.JoinMarshalers( - json.MarshalToFunc(marshalBytes), - json.MarshalToFunc(marshalAccountId), - json.MarshalToFunc(marshalMuxedAccount), + json.WithMarshalers(json.NewMarshalers( + json.MarshalFuncV2(marshalBytes), + json.MarshalFuncV2(marshalAccountId), + json.MarshalFuncV2(marshalMuxedAccount), )), ) if err != nil { diff --git a/cmd/tools/fix/tools_fix_block_hashes.go b/cmd/tools/fix/tools_fix_block_hashes.go new file mode 100644 index 0000000..8b98ea4 --- /dev/null +++ b/cmd/tools/fix/tools_fix_block_hashes.go @@ -0,0 +1,225 @@ +// Package fix carries one-shot maintenance commands. fix-block-hashes +// rewrites merged-blocks produced by the v1 RPC fetcher, which stored +// 48 bytes of garbage in pbstellar.Block.Hash and +// pbstellar.Block.Header.PreviousLedgerHash because it decoded the +// stellar-rpc hex hash as base64. The original 32-byte hash is +// recoverable per-block by re-encoding the garbage bytes back to +// base64 (yielding the original 64-char hex string) and hex-decoding +// that — no external source needed. See ConvertBrokenHash below. +// +// pbbstream.Block.Id and ParentId on v1 blocks are accidentally the +// correct hex hash strings already (the v1 convertBlock did +// base64.Encode(base64.Decode(hex)) = hex, an identity round-trip on +// 64-char base64 alphabet input), so they are kept and the fix +// cross-checks against them. +package fix + +import ( + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + + "github.com/spf13/cobra" + "github.com/streamingfast/bstream" + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + "github.com/streamingfast/dstore" + firecore "github.com/streamingfast/firehose-core" + "github.com/streamingfast/firehose-core/cmd/tools/check" + "github.com/streamingfast/firehose-core/types" + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/anypb" +) + +const stellarHashLen = 32 + +// NewToolsFixBlockHashesCmd builds the cobra command. +func NewToolsFixBlockHashesCmd(logger *zap.Logger) *cobra.Command { + return &cobra.Command{ + Use: "fix-block-hashes ", + Short: "Recover ledger Hash and PreviousLedgerHash on v1-fetcher merged blocks by reversing the bad base64-decode-of-hex encoding.", + Long: `The legacy RPC fetcher decoded stellar-rpc's hex ledger hash as base64, +which silently succeeded (hex chars are a subset of the base64 alphabet) +and stored 48 bytes of garbage in pbstellar.Block.Hash and +pbstellar.Block.Header.PreviousLedgerHash. The original 32-byte hash is +recoverable per-block by base64-encoding the garbage bytes (which +returns the original 64-char hex string thanks to base64 +round-tripping) and then hex-decoding the result. No external data +source is required. + +The bstream Block.Id / ParentId were accidentally correct on v1 blocks +(same round-trip identity), so this command uses them as a +cross-check: the recovered hash must equal hex.Decode(src.Id), else +the command bails on the block. + +The block range must start at a 100-block boundary so the destination +bundle layout stays aligned with the rest of the store. Non-overlapping +ranges can run in parallel — there is no shared state.`, + Args: cobra.ExactArgs(3), + RunE: runFixBlockHashesE(logger), + } +} + +func runFixBlockHashesE(logger *zap.Logger) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + srcStore, err := dstore.NewDBinStore(args[0]) + if err != nil { + return fmt.Errorf("creating source merged-blocks store: %w", err) + } + + destStore, err := dstore.NewDBinStore(args[1]) + if err != nil { + return fmt.Errorf("creating destination merged-blocks store: %w", err) + } + + blockRange, err := types.GetBlockRangeFromArg(args[2]) + if err != nil { + return fmt.Errorf("parsing block range: %w", err) + } + if !blockRange.IsResolved() { + return fmt.Errorf("block range must be closed (got %s)", blockRange.String()) + } + if blockRange.Start%100 != 0 { + return fmt.Errorf("block range start %d is not aligned to a 100-block bundle boundary", blockRange.Start) + } + + startBlock := uint64(blockRange.Start) + stopBlock := blockRange.MustGetStopBlock() + + mergeWriter := &firecore.MergedBlocksWriter{ + Store: destStore, + TweakBlock: func(b *pbbstream.Block) (*pbbstream.Block, error) { return b, nil }, + Logger: logger, + } + + walkErr := srcStore.Walk(ctx, check.WalkBlockPrefix(blockRange, 100), func(filename string) error { + fileStart := firecore.MustParseUint64(filename) + if fileStart >= stopBlock { + return dstore.StopIteration + } + if fileStart+100 <= startBlock { + return nil + } + + logger.Info("processing merged-blocks file", zap.String("filename", filename)) + + rc, err := srcStore.OpenObject(ctx, filename) + if err != nil { + return fmt.Errorf("opening %s: %w", filename, err) + } + defer rc.Close() + + br, err := bstream.NewDBinBlockReader(rc) + if err != nil { + return fmt.Errorf("creating block reader for %s: %w", filename, err) + } + + for { + blk, err := br.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading block from %s: %w", filename, err) + } + + if blk.Number < startBlock { + continue + } + if blk.Number >= stopBlock { + break + } + + fixed, err := FixBlock(blk) + if err != nil { + return fmt.Errorf("fixing block %d (%s): %w", blk.Number, blk.Id, err) + } + + if err := mergeWriter.ProcessBlock(fixed, nil); err != nil { + if errors.Is(err, io.EOF) { + return dstore.StopIteration + } + return fmt.Errorf("write fixed block %d: %w", fixed.Number, err) + } + } + + return nil + }) + + if walkErr != nil && !errors.Is(walkErr, io.EOF) { + return walkErr + } + + return nil + } +} + +// FixBlock applies the byte-level recovery to one v1 block. Exported so +// the test suite and ad-hoc tooling can reuse it. +func FixBlock(src *pbbstream.Block) (*pbbstream.Block, error) { + var srcStellar pbstellar.Block + if err := src.Payload.UnmarshalTo(&srcStellar); err != nil { + return nil, fmt.Errorf("unmarshal src payload: %w", err) + } + if srcStellar.Header == nil { + return nil, fmt.Errorf("src block %d has nil header", src.Number) + } + + recoveredHash, err := ConvertBrokenHash(srcStellar.Hash) + if err != nil { + return nil, fmt.Errorf("recovering ledger hash: %w", err) + } + recoveredPrev, err := ConvertBrokenHash(srcStellar.Header.PreviousLedgerHash) + if err != nil { + return nil, fmt.Errorf("recovering previous-ledger hash: %w", err) + } + + // Cross-check: v1 bstream.Id/ParentId are the correct hex strings + // (round-trip identity). If they disagree with our recovery, something + // about the input is not the shape we expect — fail loud. + if got := hex.EncodeToString(recoveredHash); got != src.Id { + return nil, fmt.Errorf("recovered hash %s does not match bstream Id %s for block %d", got, src.Id, src.Number) + } + if got := hex.EncodeToString(recoveredPrev); got != src.ParentId { + return nil, fmt.Errorf("recovered previous hash %s does not match bstream ParentId %s for block %d", got, src.ParentId, src.Number) + } + + srcStellar.Hash = recoveredHash + srcStellar.Header.PreviousLedgerHash = recoveredPrev + + newPayload, err := anypb.New(&srcStellar) + if err != nil { + return nil, fmt.Errorf("repacking payload: %w", err) + } + + return &pbbstream.Block{ + Number: src.Number, + Id: src.Id, + ParentId: src.ParentId, + Timestamp: src.Timestamp, + LibNum: src.LibNum, + ParentNum: src.ParentNum, + Payload: newPayload, + }, nil +} + +// ConvertBrokenHash reverses the v1 fetcher's accidental +// base64.Decode(hex_string) by base64-encoding the stored garbage +// bytes (giving back the original hex string) and hex-decoding the +// result. Returns an error if the recovered hash is not 32 bytes, +// which is the only valid length for a Stellar ledger hash. +func ConvertBrokenHash(broken []byte) ([]byte, error) { + hexStr := base64.StdEncoding.EncodeToString(broken) + recovered, err := hex.DecodeString(hexStr) + if err != nil { + return nil, fmt.Errorf("hex-decoding recovered string %q: %w", hexStr, err) + } + if len(recovered) != stellarHashLen { + return nil, fmt.Errorf("recovered hash length %d, expected %d", len(recovered), stellarHashLen) + } + return recovered, nil +} diff --git a/cmd/tools/fix/tools_fix_block_hashes_test.go b/cmd/tools/fix/tools_fix_block_hashes_test.go new file mode 100644 index 0000000..9e1c8f2 --- /dev/null +++ b/cmd/tools/fix/tools_fix_block_hashes_test.go @@ -0,0 +1,142 @@ +package fix + +import ( + "encoding/base64" + "encoding/hex" + "testing" + + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" +) + +// A real-looking 32-byte Stellar ledger hash (lowercase hex, 64 chars). +const knownHashHex = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff" +const knownPrevHex = "ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100" + +// brokenBytes reproduces what the v1 RPC fetcher stored in +// pbstellar.Block.Hash and pbstellar.Block.Header.PreviousLedgerHash: +// it decoded the hex hash string as base64 (which silently succeeds +// because hex chars are a subset of the base64 alphabet) yielding 48 +// bytes of garbage. +func brokenBytes(t *testing.T, hexStr string) []byte { + t.Helper() + out, err := base64.StdEncoding.DecodeString(hexStr) + require.NoError(t, err) + require.Len(t, out, 48, "v1 garbage bytes are 48 bytes long") + return out +} + +func TestConvertBrokenHash_RoundTrip(t *testing.T) { + broken := brokenBytes(t, knownHashHex) + + recovered, err := ConvertBrokenHash(broken) + require.NoError(t, err) + require.Len(t, recovered, stellarHashLen) + + want, err := hex.DecodeString(knownHashHex) + require.NoError(t, err) + require.Equal(t, want, recovered, "recovered bytes must equal the original 32-byte hash") +} + +func TestConvertBrokenHash_RejectsWrongLength(t *testing.T) { + // 32 bytes input (looks like a valid hash) would base64-encode to 44 + // chars, which is not a valid hex length — hex.DecodeString returns + // an error, so ConvertBrokenHash surfaces that. + _, err := ConvertBrokenHash(make([]byte, 32)) + require.Error(t, err) +} + +func TestFixBlock_RecoversBothHashes(t *testing.T) { + src := buildBrokenBlock(t, 60_000_001, knownHashHex, knownPrevHex) + + fixed, err := FixBlock(src) + require.NoError(t, err) + + var stellar pbstellar.Block + require.NoError(t, fixed.Payload.UnmarshalTo(&stellar)) + + wantHash, _ := hex.DecodeString(knownHashHex) + wantPrev, _ := hex.DecodeString(knownPrevHex) + require.Equal(t, wantHash, stellar.Hash) + require.Equal(t, wantPrev, stellar.Header.PreviousLedgerHash) + + // bstream.Id/ParentId on the v1 block are already the correct hex + // strings (base64.Encode(base64.Decode(hex)) is identity over the + // 64-char hex alphabet), so they pass through untouched. + require.Equal(t, knownHashHex, fixed.Id) + require.Equal(t, knownPrevHex, fixed.ParentId) +} + +func TestFixBlock_PreservesNonHashFields(t *testing.T) { + src := buildBrokenBlock(t, 60_000_002, knownHashHex, knownPrevHex) + + var origStellar pbstellar.Block + require.NoError(t, src.Payload.UnmarshalTo(&origStellar)) + + fixed, err := FixBlock(src) + require.NoError(t, err) + + require.Equal(t, src.Number, fixed.Number) + require.Equal(t, src.Timestamp, fixed.Timestamp) + require.Equal(t, src.LibNum, fixed.LibNum) + require.Equal(t, src.ParentNum, fixed.ParentNum) + + var fixedStellar pbstellar.Block + require.NoError(t, fixed.Payload.UnmarshalTo(&fixedStellar)) + + require.Equal(t, origStellar.Version, fixedStellar.Version) + require.Equal(t, origStellar.Header.LedgerVersion, fixedStellar.Header.LedgerVersion) + require.Equal(t, origStellar.Header.TotalCoins, fixedStellar.Header.TotalCoins) + require.Equal(t, origStellar.Header.BaseFee, fixedStellar.Header.BaseFee) + require.Equal(t, origStellar.Header.BaseReserve, fixedStellar.Header.BaseReserve) + require.Len(t, fixedStellar.Transactions, len(origStellar.Transactions)) +} + +func TestFixBlock_FailsWhenIdMismatch(t *testing.T) { + src := buildBrokenBlock(t, 60_000_003, knownHashHex, knownPrevHex) + src.Id = "deadbeef" // intentionally wrong + + _, err := FixBlock(src) + require.Error(t, err) + require.Contains(t, err.Error(), "does not match bstream Id") +} + +// buildBrokenBlock constructs a pbbstream.Block in exactly the shape +// the v1 RPC fetcher produced: pbstellar.Block.Hash and +// Header.PreviousLedgerHash hold base64.Decode(hex) garbage, while the +// bstream Id/ParentId hold the original hex string (because +// base64.Encode(base64.Decode(hex)) is identity for 64-char hex +// alphabet inputs). +func buildBrokenBlock(t *testing.T, num uint64, hashHex, prevHex string) *pbbstream.Block { + t.Helper() + + stellar := &pbstellar.Block{ + Number: num, + Version: 1, + Hash: brokenBytes(t, hashHex), + Header: &pbstellar.Header{ + LedgerVersion: 23, + PreviousLedgerHash: brokenBytes(t, prevHex), + TotalCoins: 105_443_902_087_300_000, + BaseFee: 100, + BaseReserve: 5_000_000, + }, + Transactions: []*pbstellar.Transaction{ + {ApplicationOrder: 1}, + }, + } + + payload, err := anypb.New(stellar) + require.NoError(t, err) + + return &pbbstream.Block{ + Number: num, + Id: hashHex, + ParentId: prevHex, + LibNum: num - 1, + ParentNum: num - 1, + Payload: payload, + } +} diff --git a/cursor/cursor.go b/cursor/cursor.go new file mode 100644 index 0000000..b7ef50d --- /dev/null +++ b/cursor/cursor.go @@ -0,0 +1,90 @@ +// Package cursor persists the last fired block to disk so a fetcher can +// resume across restarts. The on-disk schema (cursor.json) matches +// firehose-core/blockpoller, so a single state-dir can be shared +// between fetchers that use this package and ones backed by the +// upstream blockpoller. Stellar is final at close, so Lib == +// LastFiredBlock and the Blocks fork-history slice stays empty. +package cursor + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" +) + +type BlockRef struct { + Id string `json:"id"` + Num uint64 `json:"num"` +} + +type BlockRefWithPrev struct { + BlockRef + PrevBlockId string `json:"previous_ref_id"` +} + +// State is the on-disk cursor schema. The outer fields intentionally +// carry no JSON tags so they serialize as Lib / LastFiredBlock / Blocks, +// matching firehose-core/blockpoller's unexported stateFile struct +// byte-for-byte. Adding snake_case tags here would silently break the +// shared --state-dir compatibility we advertise in the README. +type State struct { + Lib BlockRef + LastFiredBlock BlockRefWithPrev + Blocks []BlockRefWithPrev +} + +const fileName = "cursor.json" + +func path(stateDir string) string { + return filepath.Join(stateDir, fileName) +} + +// Load returns the persisted state, or (nil, nil) if stateDir is empty +// or the file does not yet exist. +func Load(stateDir string) (*State, error) { + if stateDir == "" { + return nil, nil + } + data, err := os.ReadFile(path(stateDir)) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read cursor: %w", err) + } + var s State + if err := json.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("decode cursor: %w", err) + } + return &s, nil +} + +// Save records blk as the last fired block. No-op when stateDir is +// empty. +func Save(stateDir string, blk *pbbstream.Block) error { + if stateDir == "" { + return nil + } + if err := os.MkdirAll(stateDir, 0o755); err != nil { + return fmt.Errorf("mkdir state dir: %w", err) + } + s := State{ + Lib: BlockRef{Id: blk.Id, Num: blk.Number}, + LastFiredBlock: BlockRefWithPrev{ + BlockRef: BlockRef{Id: blk.Id, Num: blk.Number}, + PrevBlockId: blk.ParentId, + }, + Blocks: []BlockRefWithPrev{}, + } + data, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("marshal cursor: %w", err) + } + if err := os.WriteFile(path(stateDir), data, 0o644); err != nil { + return fmt.Errorf("write cursor: %w", err) + } + return nil +} diff --git a/devel/captive-core.sh b/devel/captive-core.sh new file mode 100755 index 0000000..b40be9a --- /dev/null +++ b/devel/captive-core.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Run firehose-stellar with the Captive Core fetcher. +# Requires: stellar-core binary on $PATH (override with STELLAR_CORE_BIN env var). + +set -e + +START_BLOCK="${START_BLOCK:-2391485}" +NETWORK="${NETWORK:-testnet}" + +if [[ -n "${STELLAR_CORE_BIN:-}" ]]; then + if [[ ! -x "${STELLAR_CORE_BIN}" ]]; then + echo "ERROR: STELLAR_CORE_BIN=${STELLAR_CORE_BIN} is not an executable file" >&2 + exit 1 + fi +else + STELLAR_CORE_BIN="$(command -v stellar-core || true)" + if [[ -z "${STELLAR_CORE_BIN}" ]]; then + echo "ERROR: stellar-core binary not found in PATH." >&2 + echo "Install it (e.g. 'brew install stellar-core' or download from https://github.com/stellar/stellar-core/releases)," >&2 + echo "or set STELLAR_CORE_BIN=/path/to/stellar-core before re-running." >&2 + exit 1 + fi +fi + +echo "Using stellar-core binary: ${STELLAR_CORE_BIN}" + +firecore start reader-node merger \ + --config-file= \ + --log-format=stackdriver \ + --log-to-file=false \ + --data-dir=data-cc \ + --common-auto-max-procs \ + --common-auto-mem-limit-percent=90 \ + --common-one-block-store-url=data-cc/oneblock \ + --common-first-streamable-block="${START_BLOCK}" \ + --reader-node-data-dir=data-cc/oneblock \ + --reader-node-working-dir=data-cc/work \ + --reader-node-readiness-max-latency=600s \ + --reader-node-debug-firehose-logs=false \ + --reader-node-blocks-chan-capacity=1000 \ + --reader-node-grpc-listen-addr=:9101 \ + --reader-node-manager-api-addr=:8180 \ + --merger-grpc-listen-addr=:10112 \ + --reader-node-path=./devel/firestellar \ + --reader-node-arguments="fetch captive-core ${START_BLOCK} --stellar-core-bin ${STELLAR_CORE_BIN} --stellar-core-network ${NETWORK}" diff --git a/devel/poller.sh b/devel/poller.sh index a30c12f..c14cb73 100755 --- a/devel/poller.sh +++ b/devel/poller.sh @@ -6,7 +6,7 @@ firecore start reader-node merger \ --common-auto-max-procs \ --common-auto-mem-limit-percent=90 \ --common-one-block-store-url=data/oneblock \ - --common-first-streamable-block=487500 \ + --common-first-streamable-block=2391485 \ --reader-node-data-dir=data/oneblock \ --reader-node-working-dir=data/work \ --reader-node-readiness-max-latency=600s \ @@ -15,6 +15,4 @@ firecore start reader-node merger \ --reader-node-grpc-listen-addr=:9001 \ --reader-node-manager-api-addr=:8080 \ --reader-node-path=firestellar \ - --reader-node-arguments='fetch rpc 487500 \ - --state-dir data \ - --endpoints https://soroban-testnet.stellar.org/' + --reader-node-arguments='fetch rpc 2391485 --state-dir data --stellar-rpc-network=testnet --endpoints https://soroban-testnet.stellar.org/' diff --git a/go.mod b/go.mod index 0104107..9142ab1 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.26.0 replace github.com/jhump/protoreflect => github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d require ( - github.com/bobg/go-generics/v3 v3.7.0 - github.com/go-json-experiment/json v0.0.0-20250116043007-0640c115aea5 + github.com/bobg/go-generics/v3 v3.5.0 + github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d // FIXME pinned to Oct 2023 snapshot to match firehose-core's internal json/marshallers.go (old API names NewMarshalers / MarshalFuncV2). Bump when firehose-core upstream renames its call sites to JoinMarshalers / MarshalToFunc and bumps this pin itself; see cmd/firestellar/tool_decode_block.go header. github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 github.com/spf13/cobra v1.10.2 - github.com/stellar/go-stellar-sdk v0.5.0 // bumped in https://github.com/stellar/stellar-rpc/pull/623/changes#diff-33ef32bf6c23acb95f5902d7097b7a1d5128ca061167ec0716715b0b9eeaa5f6 + github.com/stellar/go-stellar-sdk v0.5.0 // matches main; originally bumped in https://github.com/stellar/stellar-rpc/pull/623/changes#diff-33ef32bf6c23acb95f5902d7097b7a1d5128ca061167ec0716715b0b9eeaa5f6 github.com/streamingfast/bstream v0.0.2-0.20260402095814-607e840ece3d github.com/streamingfast/cli v0.0.4-0.20250815192146-d8a233ec3d0b github.com/streamingfast/dhttp v0.1.3-0.20251218140957-6d46b8f12eb1 @@ -21,6 +21,8 @@ require ( google.golang.org/protobuf v1.36.11 ) +require github.com/sirupsen/logrus v1.9.3 + require ( buf.build/gen/go/bufbuild/reflect/connectrpc/go v1.16.1-20240117202343-bf8f65e8876c.1 // indirect buf.build/gen/go/bufbuild/reflect/protocolbuffers/go v1.33.0-20240117202343-bf8f65e8876c.1 // indirect @@ -155,7 +157,6 @@ require ( github.com/sercand/kuberesolver/v5 v5.1.1 // indirect github.com/sethvargo/go-retry v0.2.3 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect diff --git a/go.sum b/go.sum index c28b839..5d75807 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d h1:fSlGu5ePbkjBidXuj2O5j9EcYrVB5Cr6/wdkYyDgxZk= github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d/go.mod h1:yCBkgASmKHgUOFjK9h1sOytUVgA+JkQjqj3xYP4AdWY= -github.com/bobg/go-generics/v3 v3.7.0 h1:4SJHDWqONTRcA8al6491VW/ys6061bPCcTcI7YnIHPc= -github.com/bobg/go-generics/v3 v3.7.0/go.mod h1:wGlMLQER92clsh3cJoQjbUtUEJ03FoxnGhZjaWhf4fM= +github.com/bobg/go-generics/v3 v3.5.0 h1:OdBXzCRCO4e3Z7FQz1maEN2Q5LFYHc7vIK8EXcS4xQQ= +github.com/bobg/go-generics/v3 v3.5.0/go.mod h1:wGlMLQER92clsh3cJoQjbUtUEJ03FoxnGhZjaWhf4fM= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -284,8 +284,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= -github.com/go-json-experiment/json v0.0.0-20250116043007-0640c115aea5 h1:MZu1NBx4m+lYVrt55nFcKb3Fze2F7LUwJXyywvNm11A= -github.com/go-json-experiment/json v0.0.0-20250116043007-0640c115aea5/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s= +github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d h1:zqfo2jECgX5eYQseB/X+uV4Y5ocGOG/vG/LTztUCyPA= +github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/rpc/fetcher.go b/rpc/fetcher.go index 24d2702..04f668d 100644 --- a/rpc/fetcher.go +++ b/rpc/fetcher.go @@ -4,12 +4,13 @@ import ( "context" "encoding/base64" "encoding/hex" + "errors" "fmt" + "io" "strconv" "time" "github.com/stellar/go-stellar-sdk/ingest" - "github.com/stellar/go-stellar-sdk/network" "github.com/stellar/go-stellar-sdk/xdr" pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" "github.com/streamingfast/firehose-stellar/decoder" @@ -37,8 +38,8 @@ type Fetcher struct { decoder *decoder.Decoder transactionFetchLimit int - logger *zap.Logger - isMainnet bool + logger *zap.Logger + networkPassphrase string // Statistics acquisitionTimes []time.Duration @@ -50,7 +51,12 @@ type Fetcher struct { statsTicker *time.Ticker } -func NewFetcher(fetchInterval, latestBlockRetryInterval time.Duration, transactionFetchLimit int, isMainnet bool, logger *zap.Logger) *Fetcher { +// NewFetcher constructs an rpc fetcher. networkPassphrase MUST match the +// passphrase the rpc endpoint serves (Public, Testnet, Standalone, etc.) — +// the Stellar SDK uses it to recompute transaction hashes from ledger +// metadata, so a mismatch produces "unknown tx hash in LedgerCloseMeta" +// errors when reading transactions out of fetched ledgers. +func NewFetcher(fetchInterval, latestBlockRetryInterval time.Duration, transactionFetchLimit int, networkPassphrase string, logger *zap.Logger) *Fetcher { f := &Fetcher{ fetchInterval: fetchInterval, latestBlockRetryInterval: latestBlockRetryInterval, @@ -58,7 +64,7 @@ func NewFetcher(fetchInterval, latestBlockRetryInterval time.Duration, transacti decoder: decoder.NewDecoder(logger), transactionFetchLimit: transactionFetchLimit, logger: logger, - isMainnet: isMainnet, + networkPassphrase: networkPassphrase, acquisitionTimes: make([]time.Duration, 0, 50), conversionTimes: make([]time.Duration, 0, 50), totalTimes: make([]time.Duration, 0, 50), @@ -222,15 +228,14 @@ func (f *Fetcher) Fetch(ctx context.Context, client *Client, requestBlockNum uin } } - ledgerHashBytes, err := base64.StdEncoding.DecodeString(ledger[0].Hash) + // stellar-rpc returns ledger.hash as a hex string; previous_ledger_hash on + // the LedgerHeader is already raw 32 bytes. + ledgerHashBytes, err := hex.DecodeString(ledger[0].Hash) if err != nil { return nil, false, fmt.Errorf("decoding ledger hash: %w", err) } - previousLedgerHashBytes, err := base64.StdEncoding.DecodeString(ledgerHeader.Header.PreviousLedgerHash.HexString()) - if err != nil { - return nil, false, fmt.Errorf("decoding previous ledger hash: %w", err) - } + previousLedgerHashBytes := ledgerHeader.Header.PreviousLedgerHash[:] stellarBlk := &pbstellar.Block{ Number: ledger[0].Sequence, @@ -313,11 +318,7 @@ func (f *Fetcher) extractTransactionsFromLedgerMetadata(ledgerMetadata *xdr.Ledg // Use the Stellar SDK's LedgerTransactionReader to extract transactions from ledger metadata // This is the proper way to access transaction data from LedgerCloseMeta - passphrase := network.PublicNetworkPassphrase - if !f.isMainnet { - passphrase = network.TestNetworkPassphrase - } - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(passphrase, *ledgerMetadata) + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(f.networkPassphrase, *ledgerMetadata) if err != nil { return nil, fmt.Errorf("failed to create ledger transaction reader: %w", err) } @@ -327,7 +328,7 @@ func (f *Fetcher) extractTransactionsFromLedgerMetadata(ledgerMetadata *xdr.Ledg for { tx, err := reader.Read() if err != nil { - if err.Error() == "EOF" { + if errors.Is(err, io.EOF) { break } return nil, fmt.Errorf("failed to read transaction: %w", err) @@ -453,8 +454,11 @@ func convertBlock(stellarBlk *pbstellar.Block) (*pbbstream.Block, error) { return nil, fmt.Errorf("unable to create anypb: %w", err) } - stellarBlockHash := base64.StdEncoding.EncodeToString(stellarBlk.Hash) - previousStellarBlockHash := base64.StdEncoding.EncodeToString(stellarBlk.Header.PreviousLedgerHash) + // Hex-encode IDs so the strings are filesystem-safe — firecore mindreader + // uses Block.Id in one-block filenames and treats '/' (which appears in + // standard base64 of 32-byte hashes) as a path separator. + stellarBlockHash := hex.EncodeToString(stellarBlk.Hash) + previousStellarBlockHash := hex.EncodeToString(stellarBlk.Header.PreviousLedgerHash) return &pbbstream.Block{ Number: stellarBlk.Number, diff --git a/rpc/fetcher_test.go b/rpc/fetcher_test.go index 50e2dad..308f245 100644 --- a/rpc/fetcher_test.go +++ b/rpc/fetcher_test.go @@ -2,6 +2,9 @@ package rpc import ( "context" + "encoding/base64" + "encoding/hex" + "strings" "testing" "time" @@ -9,13 +12,47 @@ import ( "github.com/stretchr/testify/require" ) +func Test_convertBlock_HexEncodesIDs(t *testing.T) { + // Bytes chosen so standard base64 of these 32-byte hashes contains '/' — + // guards against any regression back to base64, which would corrupt + // firecore one-block filenames (path-separator collision). + hash := []byte{ + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, + 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, + } + prev := []byte{ + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0xab, 0xcd, 0xef, 0x12, 0x34, 0x56, 0x78, 0x90, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, + } + require.Contains(t, base64.StdEncoding.EncodeToString(hash), "/", "test fixture invariant: base64(hash) should contain '/' for this regression test to be meaningful") + + stellarBlk := &pbstellar.Block{ + Number: 12345, + Hash: hash, + Header: &pbstellar.Header{PreviousLedgerHash: prev}, + } + + b, err := convertBlock(stellarBlk) + require.NoError(t, err) + + require.Equal(t, hex.EncodeToString(hash), b.Id) + require.Equal(t, hex.EncodeToString(prev), b.ParentId) + for _, id := range []string{b.Id, b.ParentId} { + require.False(t, strings.ContainsAny(id, "/+="), "block id must not contain base64-only chars: %q", id) + } +} + func Test_Fetch(t *testing.T) { c := NewClient(RPC_MAINNET_ENDPOINT, testLog, testTracer) ledger, err := c.GetLatestLedger(context.Background()) require.NoError(t, err) - f := NewFetcher(time.Second, time.Second, 200, c.rpcEndpoint == RPC_MAINNET_ENDPOINT, testLog) + f := NewFetcher(time.Second, time.Second, 200, passphraseFor(c.rpcEndpoint), testLog) b, _, err := f.Fetch(context.Background(), c, uint64(ledger.Sequence)) require.NoError(t, err) @@ -29,7 +66,7 @@ func Test_FetchSpecificLedger(t *testing.T) { const BLOCK_TO_FETCH = uint64(61322487) c := NewClient(RPC_MAINNET_ENDPOINT, testLog, testTracer) - f := NewFetcher(time.Second, time.Second, 200, c.rpcEndpoint == RPC_MAINNET_ENDPOINT, testLog) + f := NewFetcher(time.Second, time.Second, 200, passphraseFor(c.rpcEndpoint), testLog) b, _, err := f.Fetch(context.Background(), c, BLOCK_TO_FETCH) require.NoError(t, err) @@ -48,7 +85,7 @@ func Test_FetchSpecificLedger_Testnet(t *testing.T) { const EXPECTED_TRANSACTION_COUNT = 3 c := NewClient(RPC_TESTNET_ENDPOINT, testLog, testTracer) - f := NewFetcher(time.Second, time.Second, 200, c.rpcEndpoint == RPC_MAINNET_ENDPOINT, testLog) + f := NewFetcher(time.Second, time.Second, 200, passphraseFor(c.rpcEndpoint), testLog) b, _, err := f.Fetch(context.Background(), c, BLOCK_TO_FETCH) require.NoError(t, err) @@ -67,7 +104,7 @@ func Test_FetchSpecificLedger_ProtocolUpgrade23_MetadataV2(t *testing.T) { const BLOCK_TO_FETCH = uint64(2063) c := NewClient(RPC_TESTNET_ENDPOINT, testLog, testTracer) - f := NewFetcher(time.Second, time.Second, 200, c.rpcEndpoint == RPC_MAINNET_ENDPOINT, testLog) + f := NewFetcher(time.Second, time.Second, 200, passphraseFor(c.rpcEndpoint), testLog) b, _, err := f.Fetch(context.Background(), c, BLOCK_TO_FETCH) require.NoError(t, err) diff --git a/rpc/init_test.go b/rpc/init_test.go index db32276..2374f69 100644 --- a/rpc/init_test.go +++ b/rpc/init_test.go @@ -1,9 +1,22 @@ package rpc -import "github.com/streamingfast/logging" +import ( + "github.com/stellar/go-stellar-sdk/network" + "github.com/streamingfast/logging" +) var testLog, testTracer = logging.PackageLogger("rpc", "github.com/streamingfast/firehose-stellar/rpc") func init() { logging.InstantiateLoggers() } + +// passphraseFor returns the matching network passphrase for a given test +// rpc endpoint. Mirrors the resolution that production code does via the +// --stellar-rpc-network flag, but keeps the tests self-contained. +func passphraseFor(endpoint string) string { + if endpoint == RPC_MAINNET_ENDPOINT { + return network.PublicNetworkPassphrase + } + return network.TestNetworkPassphrase +} diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..9c152f5 --- /dev/null +++ b/test/README.md @@ -0,0 +1,170 @@ +# test/ — battlefield integration tests for firehose-stellar + +Drives end-to-end scenarios against firestellar's two fetcher backends and asserts the structural transaction view against committed snapshots. + +> Captive-core is the supported backend going forward; the RPC poller is kept here for cross-backend regression checks only and is no longer actively developed. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ go test ./test/scenarios/... │ +│ │ +│ ┌─────────────────────────────┐ ┌────────────────────────┐ +│ │ InProcessCaptiveCoreFetcher │ │ InProcessRPCFetcher │ +│ │ → captivecore.Backend (lib)│ │ → rpc.Fetcher (lib) │ +│ └─────────────┬───────────────┘ └─────────────┬──────────┘ +│ │ │ +└────────────────┼───────────────────────────────────┼────────┘ + │ peer + history │ HTTP + │ :11625, :1570 │ :8000/soroban/rpc + ▼ ▼ + ┌────────────────────────────────────────────────────┐ + │ stellar/quickstart container (the chain) │ + │ - stellar-core validator │ + │ - horizon + friendbot + soroban-rpc on :8000 │ + └────────────────────────────────────────────────────┘ + ▲ + │ submit tx + │ + ┌────────────────────────────────────────────────────┐ + │ scenarios_test.go (RunScenario) │ + └────────────────────────────────────────────────────┘ +``` + +Both fetchers run **in-process** — firestellar's `rpc` and `captivecore` packages are called as libraries. The only container is stellar/quickstart, which IS the chain. + +Why: +- **No drift.** Tests build against the same module — flag/API changes can't desync. +- **Cross-backend diff always on.** Both fetchers run per scenario, runner asserts they agree. +- **Fast loop.** `go test` cold-start is ~30s (quickstart boot only). + +## Run the test suite + +```bash +go test ./test/scenarios/... -v +go test ./test/scenarios/... -run Payment +SNAPSHOTS_UPDATE=. go test ./test/scenarios/... -v # regen all snapshots +SNAPSHOTS_UPDATE=payment/native go test ./test/scenarios/... # regen one +``` + +`TestMain` brings up quickstart (resetting it to a clean chain) before the suite and tears it down after. Set `BATTLEFIELD_MANAGE_STACK=0` to opt out (manual lifecycle, see below). + +### Logs + +Chatty fetcher output (stellar-core subprocess + rpc.Fetcher zap logs) is redirected to file so the test terminal stays readable. Tail from another shell: + +```bash +tail -f test/.data/fetchers.log # stellar-core + zap fetcher logs +tail -f test/.data/compose.log # docker compose output (quickstart up/down) +``` + +### Cross-backend signal + +Default fetcher set is `{captive-core, poller}` — both run per scenario, runner diffs their views. Captive-core is the primary backend; the poller stays in the suite as a regression check. A divergence between fetchers fails the test before snapshot comparison runs, so the error message tells you which fetcher emitted bad output. + +If `stellar-core` is not on `$PATH`, the captive-core fetcher silently disables itself and only the poller runs (cross-diff becomes a no-op; snapshot is the only assertion). Install stellar-core to run the supported path. + +## Prerequisites + +- `docker` + `docker compose` plugin (for quickstart) +- `go` 1.26+ +- `stellar-core` binary on `$PATH` (for captive-core in-process fetcher) + - Required minimum: **`26.1.0-3210.427aa3978`** (SDF May 2026 critical security advisory) + - macOS: `brew upgrade stellar/sdf/stellar-core` (or `brew install` for first-time) + - Linux: `apt install stellar-core` from SDF apt repo (https://apt.stellar.org); run `apt update && apt install --only-upgrade stellar-core` on existing hosts to pick up the patched build + - Override location via `STELLAR_CORE_BIN=/path/to/stellar-core` + +The stellar-core version must be protocol-compatible with the quickstart image — same major version is the safest match. The compose stack pulls `stellar/quickstart:testing` with `pull_policy: always` (override via `QUICKSTART_PULL_POLICY=missing`) so the bundled stellar-core stays current with SDF's patched release. + +## Snapshot regen + determinism + +```bash +SNAPSHOTS_UPDATE=. go test ./test/scenarios/... +``` + +Only inherently non-deterministic fields are templated as `$name` placeholders: tx hash, ledger sequence, `createdAt` timestamp, signatures, signature hints, sequence numbers. Everything else — including account addresses, SAC contract IDs, op codes, amounts — is compared as a literal so any drift fails byte-equality directly. + +### Why account addresses are deterministic + +Keypairs derive from `sha256(t.Name() + "/" + accountName)` (see `runner.Config.AccountSeedScope`). The same `(test name, role)` pair always produces the same `G…` address across runs. Two consequences: + +1. Snapshots can commit literal `G…` strings instead of templated placeholders. +2. **Renaming a `Test…` function invalidates that scenario's snapshot.** Workflow: + ```bash + SNAPSHOTS_UPDATE=^$ go test ./test/scenarios/... + ``` + +Same applies to any test-code change that alters submitted operations — snapshot drift is the intended signal. + +## Manual stack control + +For fast iteration across many test cycles, opt out of TestMain's lifecycle: + +```bash +test/scripts/dev/up.sh # boot quickstart +BATTLEFIELD_MANAGE_STACK=0 go test ./test/scenarios/... -v # use it +test/scripts/dev/down.sh # tear down +``` + +`test/scripts/dev/reset.sh` restarts quickstart for a clean chain without rebooting docker daemon state. + +`BATTLEFIELD_MANAGE_STACK=0` tells TestMain to trust the pre-existing stack. With it unset (default), `go test` owns the lifecycle. + +## Configuration + +| Var | Default | Meaning | +|---|---|---| +| `BATTLEFIELD_MANAGE_STACK` | `1` | TestMain brings quickstart up/down | +| `AUTO_RESET` | `1` | reset chain before tests; `0` to reuse current state | +| `KEEP_RUNNING` | `0` | leave quickstart up after tests finish | +| `SKIP_TESTS` | `0` | bring quickstart up, skip the test body | +| `DEBUG` | `0` | stream raw `docker compose` output to stderr | +| `SNAPSHOTS_UPDATE` | unset | regex: regenerate matching snapshots | +| `STELLAR_CORE_BIN` | from `$PATH` | override captive-core's stellar-core binary location | +| `BATTLEFIELD_SNAPSHOTS` | `snapshots` | snapshot root | +| `BATTLEFIELD_WAIT_TIMEOUT` | `90s` | poll deadline per ledger | +| `BATTLEFIELD_DATA_DIR` | `test/.data` | dev-stack state dir | + +The dev image is amd64-only (SDF only ships amd64 stellar-core packages); on Apple Silicon, quickstart runs under Rosetta. The host-side stellar-core (for the in-process captive-core fetcher) runs natively on whatever arch it was built for. + +## Layout + +``` +test/ +├── lib/ +│ ├── devstack/ docker-compose lifecycle (quickstart up/down) +│ ├── firehose/ Fetcher iface + in-process impls (RPC, CaptiveCore) +│ ├── runner/ submit → fetch → diff fetchers → snapshot +│ ├── snapshot/ load + $var resolve + deepcompare + SNAPSHOTS_UPDATE +│ ├── stellar/ keypair + horizon + txnbuild helpers +│ └── xdr/ envelope/result/events → JSON-friendly maps +├── scenarios/ *_test.go scenarios (TestMain owns lifecycle) +├── snapshots/ /.expected.json +└── scripts/dev/ docker-compose.yml + up/down/reset.sh + configs/ +``` + +Library code under `firehose-stellar/captivecore/` and `firehose-stellar/rpc/` (one module level up) is the in-process surface battlefield calls. + +## Scenarios + +| File | What it covers | +|---|---| +| `scenarios_test.go` | native payment, double-send, create_account, manage_data, asset issuance + trustline, multi-op, account_merge, failed-trustline | +| `edge_test.go` | fee bump, multi-sig, manage_sell_offer, bump_sequence | +| `soroban_test.go` | invoke, contract events, cross-contract, diagnostic events. **Placeholder** — gated behind `BATTLEFIELD_SOROBAN=1`. | + +### TODO + +- Soroban contract suite (deploy + invoke + events + panic) +- V0 envelope (`EnvelopeTypeEnvelopeTypeTxV0`) +- Sponsorship sandwich (`BeginSponsoringFutureReserves` / op / `End…`) +- Clawback + asset trustline flags + +## Troubleshooting + +- **`stellar-core: command not found`** — install via brew/apt, or set `STELLAR_CORE_BIN=/path/to/stellar-core`. Captive-core is the supported backend, so install it for full coverage; the legacy poller still runs without it. +- **`bind: Address already in use`** — captive-core listens on a peer port; the follower config uses `PEER_PORT=11626` to avoid colliding with quickstart's `:11625`. If you've changed quickstart's host port mapping, update the follower config. +- **`stack didn't produce blocks`** — `docker compose -f test/scripts/dev/docker-compose.yml logs quickstart`. +- **`fetcher disagreement for X`** — congrats, you found a real divergence between captive-core and the poller. The error includes the JSON path that differs. +- **`docker compose` plugin missing** — install Docker Desktop or the compose-plugin package. diff --git a/test/lib/devstack/devstack.go b/test/lib/devstack/devstack.go new file mode 100644 index 0000000..6b1af31 --- /dev/null +++ b/test/lib/devstack/devstack.go @@ -0,0 +1,392 @@ +// Package devstack manages the test chain (stellar/quickstart docker +// container) from Go. TestMain calls Reset/Up before scenarios run and +// Down when they finish, so plain `go test` drives the full lifecycle. +// +// The actual containers + Dockerfile + entrypoint config live under +// test/scripts/dev/. This package shells `docker compose` via +// exec.Command — same orchestration the bash scripts do, just in Go. +package devstack + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" +) + +// Config controls the quickstart lifecycle. Optional fields have +// reasonable defaults — call DefaultConfig() and tweak. +type Config struct { + // ComposeFile is the absolute path to test/scripts/dev/docker-compose.yml. + // Empty triggers an upward search from cwd. + ComposeFile string + + // ProjectName maps to docker compose's COMPOSE_PROJECT_NAME — also + // the prefix for container names and the network name. + ProjectName string + + // DataRoot is the host dir used for compose logs + captive-core + // working files. Defaults to /.data, where is + // the directory three levels above ComposeFile — i.e. the repo's + // test/ dir when ComposeFile is test/scripts/dev/docker-compose.yml. + // Not the repo root; the name reflects the actual derivation. + DataRoot string + + // QuickstartImage overrides docker-compose.yml's + // stellar/quickstart:testing default. + QuickstartImage string + + // HorizonPort is where the host reaches quickstart's horizon / + // soroban-rpc / friendbot. + HorizonPort int + + // SorobanReadyTimeout caps how long Up waits for soroban-rpc and + // friendbot to come online. + SorobanReadyTimeout time.Duration + + // PollInterval is the poll cadence for the readiness probes. + PollInterval time.Duration + + // Debug streams docker compose's stdout/stderr to Stderr instead + // of writing it to LogFile. + Debug bool + + // LogFile collects compose stdout/stderr when Debug=false. + // Defaults to /compose.log. Truncated on each Up(). + LogFile string + + // Stdout / Stderr receive progress lines (`>> starting quickstart`). + Stdout io.Writer + Stderr io.Writer +} + +// DefaultConfig returns a Config with timeout/output defaults filled. +// ComposeFile + DataRoot resolved at New() time. +func DefaultConfig() Config { + return Config{ + ProjectName: "battlefield-stellar", + HorizonPort: 8000, + SorobanReadyTimeout: 120 * time.Second, + PollInterval: 2 * time.Second, + Stdout: os.Stderr, + Stderr: os.Stderr, + } +} + +// Stack drives one quickstart lifecycle. +type Stack struct { + cfg Config + testRoot string +} + +// New validates cfg, resolves ComposeFile + DataRoot + LogFile if blank, +// and returns a ready Stack. +func New(cfg Config) (*Stack, error) { + if cfg.ProjectName == "" { + cfg.ProjectName = "battlefield-stellar" + } + if cfg.HorizonPort == 0 { + cfg.HorizonPort = 8000 + } + if cfg.SorobanReadyTimeout == 0 { + cfg.SorobanReadyTimeout = 120 * time.Second + } + if cfg.PollInterval == 0 { + cfg.PollInterval = 2 * time.Second + } + if cfg.Stdout == nil { + cfg.Stdout = os.Stderr + } + if cfg.Stderr == nil { + cfg.Stderr = os.Stderr + } + + if cfg.ComposeFile == "" { + found, err := findComposeFile() + if err != nil { + return nil, err + } + cfg.ComposeFile = found + } + + testRoot := filepath.Dir(filepath.Dir(filepath.Dir(cfg.ComposeFile))) + if cfg.DataRoot == "" { + cfg.DataRoot = filepath.Join(testRoot, ".data") + } + if err := os.MkdirAll(cfg.DataRoot, 0o755); err != nil { + return nil, fmt.Errorf("mkdir data root: %w", err) + } + if cfg.LogFile == "" { + cfg.LogFile = filepath.Join(cfg.DataRoot, "compose.log") + } + + return &Stack{cfg: cfg, testRoot: testRoot}, nil +} + +// ComposeFile returns the resolved docker-compose.yml path. +func (s *Stack) ComposeFile() string { return s.cfg.ComposeFile } + +// DataRoot returns the resolved host data directory. +func (s *Stack) DataRoot() string { return s.cfg.DataRoot } + +// Up starts quickstart and waits for soroban-rpc + friendbot to come +// online. Idempotent. +func (s *Stack) Up(ctx context.Context) error { + if err := s.requireDocker(ctx); err != nil { + return err + } + if err := s.truncateLogFile(); err != nil { + return err + } + + s.logf(">> starting quickstart (logs: %s)", s.cfg.LogFile) + if err := s.compose(ctx, "up", "-d", "--build", "quickstart"); err != nil { + return fmt.Errorf("compose up quickstart: %w", err) + } + + if _, err := s.waitForSoroban(ctx); err != nil { + return err + } + + // Friendbot lives behind horizon's reverse proxy; the proxy returns + // 502 Bad Gateway for ~10-30s after a fresh boot even though + // soroban-rpc is already ticking. Wait until friendbot answers + // with its own 4xx (not a gateway error). + if err := s.waitForFriendbot(ctx); err != nil { + return err + } + + s.logf(">> quickstart ready (horizon http://localhost:%d)", s.cfg.HorizonPort) + return nil +} + +// Down tears quickstart down. Idempotent. +func (s *Stack) Down(ctx context.Context) error { + if _, err := exec.LookPath("docker"); err != nil { + s.logf(">> docker not present; nothing to do") + return nil + } + s.logf(">> stopping quickstart") + return s.compose(ctx, "down", "--remove-orphans") +} + +// Reset is Down + Up. Used between runs so each scenario gets a clean +// deterministic chain. +func (s *Stack) Reset(ctx context.Context) error { + _ = s.Down(ctx) + return s.Up(ctx) +} + +// IsRunning reports whether the quickstart container is currently up. +func (s *Stack) IsRunning(ctx context.Context) (bool, error) { + if _, err := exec.LookPath("docker"); err != nil { + return false, nil + } + var out bytes.Buffer + cmd := exec.CommandContext(ctx, "docker", "compose", "-f", s.cfg.ComposeFile, "ps", "-q", "--status=running") + cmd.Env = append(os.Environ(), s.composeEnv()...) + cmd.Stdout = &out + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return false, nil // treat any failure as "not running" + } + return strings.TrimSpace(out.String()) != "", nil +} + +func (s *Stack) requireDocker(ctx context.Context) error { + if _, err := exec.LookPath("docker"); err != nil { + return fmt.Errorf("docker not found: install Docker Desktop or the docker CLI") + } + cmd := exec.CommandContext(ctx, "docker", "compose", "version") + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return fmt.Errorf("'docker compose' plugin not found: %w", err) + } + return nil +} + +func (s *Stack) truncateLogFile() error { + f, err := os.Create(s.cfg.LogFile) + if err != nil { + return fmt.Errorf("truncate log file: %w", err) + } + return f.Close() +} + +// composeEnv is the env-var slice docker compose needs. +func (s *Stack) composeEnv() []string { + env := []string{ + "COMPOSE_PROJECT_NAME=" + s.cfg.ProjectName, + "BATTLEFIELD_DATA_DIR=" + s.cfg.DataRoot, + } + if s.cfg.QuickstartImage != "" { + env = append(env, "QUICKSTART_IMAGE="+s.cfg.QuickstartImage) + } + if s.cfg.HorizonPort != 0 && s.cfg.HorizonPort != 8000 { + env = append(env, "QUICKSTART_HORIZON_PORT="+strconv.Itoa(s.cfg.HorizonPort)) + } + return env +} + +// compose runs `docker compose `, routing output to the log file +// (or stderr when Debug=true). +func (s *Stack) compose(ctx context.Context, args ...string) error { + full := append([]string{"compose", "-f", s.cfg.ComposeFile}, args...) + cmd := exec.CommandContext(ctx, "docker", full...) + cmd.Env = append(os.Environ(), s.composeEnv()...) + + if s.cfg.Debug { + cmd.Stdout = s.cfg.Stderr + cmd.Stderr = s.cfg.Stderr + return cmd.Run() + } + + logF, err := os.OpenFile(s.cfg.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o644) + if err != nil { + return fmt.Errorf("open log file: %w", err) + } + defer logF.Close() + cmd.Stdout = logF + cmd.Stderr = logF + return cmd.Run() +} + +// waitForSoroban polls getLatestLedger until the chain is producing +// ledgers (>=2). Returns the latest ledger seen. +func (s *Stack) waitForSoroban(ctx context.Context) (uint64, error) { + deadline := time.Now().Add(s.cfg.SorobanReadyTimeout) + url := fmt.Sprintf("http://localhost:%d/soroban/rpc", s.cfg.HorizonPort) + body := []byte(`{"jsonrpc":"2.0","id":1,"method":"getLatestLedger"}`) + + s.log(">> waiting for soroban-rpc") + var lastErr error + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return 0, ctx.Err() + default: + } + latest, err := sorobanLatestLedger(ctx, url, body) + if err == nil && latest >= 2 { + s.logf(" ✓ (ledger %d)", latest) + return latest, nil + } + lastErr = err + s.log(".") + time.Sleep(s.cfg.PollInterval) + } + return 0, fmt.Errorf("soroban-rpc didn't come online within %s (last err: %v); see %s", + s.cfg.SorobanReadyTimeout, lastErr, s.cfg.LogFile) +} + +// waitForFriendbot polls horizon's friendbot endpoint until it returns +// a non-5xx response. 502 means horizon's reverse proxy can't reach +// friendbot yet; 4xx means friendbot is actually responding. +func (s *Stack) waitForFriendbot(ctx context.Context) error { + deadline := time.Now().Add(s.cfg.SorobanReadyTimeout) + url := fmt.Sprintf("http://localhost:%d/friendbot", s.cfg.HorizonPort) + + s.log(">> waiting for friendbot") + var lastStatus int + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err == nil { + resp, err := http.DefaultClient.Do(req) + if err == nil { + lastStatus = resp.StatusCode + resp.Body.Close() + if resp.StatusCode < 500 { + s.logf(" ✓ (status %d)", resp.StatusCode) + return nil + } + } + } + s.log(".") + time.Sleep(s.cfg.PollInterval) + } + return fmt.Errorf("friendbot didn't come online within %s (last status: %d); see %s", + s.cfg.SorobanReadyTimeout, lastStatus, s.cfg.LogFile) +} + +func sorobanLatestLedger(ctx context.Context, url string, body []byte) (uint64, error) { + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body)) + if err != nil { + return 0, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return 0, fmt.Errorf("status %d", resp.StatusCode) + } + var out struct { + Result struct { + Sequence uint64 `json:"sequence"` + } `json:"result"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return 0, err + } + return out.Result.Sequence, nil +} + +// findComposeFile locates test/scripts/dev/docker-compose.yml. Tries: +// 1. walking upward from cwd for test/scripts/dev/docker-compose.yml +// 2. walking upward from cwd for scripts/dev/docker-compose.yml (when +// cwd is already inside test/, since go test sets cwd to pkg dir) +// 3. runtime.Caller fallback (this file's known location) +func findComposeFile() (string, error) { + if cwd, err := os.Getwd(); err == nil { + if found := walkUpFor(cwd, filepath.Join("test", "scripts", "dev", "docker-compose.yml")); found != "" { + return found, nil + } + if found := walkUpFor(cwd, filepath.Join("scripts", "dev", "docker-compose.yml")); found != "" { + return found, nil + } + } + if _, file, _, ok := runtime.Caller(0); ok { + // file is .../test/lib/devstack/devstack.go — test root three up. + testRoot := filepath.Dir(filepath.Dir(filepath.Dir(file))) + candidate := filepath.Join(testRoot, "scripts", "dev", "docker-compose.yml") + if _, err := os.Stat(candidate); err == nil { + return candidate, nil + } + } + return "", fmt.Errorf("could not locate test/scripts/dev/docker-compose.yml") +} + +func walkUpFor(start, rel string) string { + dir := start + for { + candidate := filepath.Join(dir, rel) + if _, err := os.Stat(candidate); err == nil { + return candidate + } + parent := filepath.Dir(dir) + if parent == dir { + return "" + } + dir = parent + } +} + +func (s *Stack) log(msg string) { fmt.Fprint(s.cfg.Stdout, msg) } +func (s *Stack) logf(f string, a ...any) { fmt.Fprintf(s.cfg.Stdout, f+"\n", a...) } diff --git a/test/lib/firehose/fetcher.go b/test/lib/firehose/fetcher.go new file mode 100644 index 0000000..1d6db9d --- /dev/null +++ b/test/lib/firehose/fetcher.go @@ -0,0 +1,69 @@ +// Package firehose abstracts a source of decoded pbstellar.Block values. +// Two implementations: +// +// - InProcessRPCFetcher: wraps the rpc.Fetcher library +// - InProcessCaptiveCoreFetcher: wraps the captivecore.Backend library +// +// The runner pulls each ledger from every configured fetcher and asserts +// they agree (cross-backend diff) before snapshot comparison. +package firehose + +import ( + "context" + "fmt" + "strings" + "time" + + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" +) + +// unmarshalStellarBlock unwraps the pbstellar.Block payload from a +// pbbstream.Block. Used by the in-process fetcher impls to convert the +// firestellar library's return type into the shape the runner expects. +func unmarshalStellarBlock(b *pbbstream.Block) (*pbstellar.Block, error) { + var out pbstellar.Block + if err := b.Payload.UnmarshalTo(&out); err != nil { + return nil, fmt.Errorf("unmarshal stellar payload: %w", err) + } + return &out, nil +} + +// Fetcher is a source of decoded stellar blocks. +type Fetcher interface { + // Name identifies the fetcher ("poller", "captive-core"). + Name() string + // FetchBlock returns the block at ledger or an error if not yet available. + FetchBlock(ctx context.Context, ledger uint64) (*pbstellar.Block, error) + // Close releases the fetcher's resources. + Close() error +} + +// WaitForBlock polls a fetcher with backoff until either the block becomes +// available or the context is cancelled. Useful right after submitting a tx, +// when the firehose tail is a few seconds behind horizon. +func WaitForBlock(ctx context.Context, f Fetcher, ledger uint64, every time.Duration) (*pbstellar.Block, error) { + t := time.NewTicker(every) + defer t.Stop() + for { + blk, err := f.FetchBlock(ctx, ledger) + if err == nil { + return blk, nil + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("[%s] waiting for ledger %d: %w", f.Name(), ledger, ctx.Err()) + case <-t.C: + } + } +} + +// FindTxByHash scans a block for a transaction with the matching hex hash. +func FindTxByHash(block *pbstellar.Block, hexHash string) (*pbstellar.Transaction, int, error) { + for i, tx := range block.Transactions { + if fmt.Sprintf("%x", tx.Hash) == strings.ToLower(hexHash) { + return tx, i, nil + } + } + return nil, -1, fmt.Errorf("transaction %s not found in ledger %d", hexHash, block.Number) +} diff --git a/test/lib/firehose/inprocess_cc.go b/test/lib/firehose/inprocess_cc.go new file mode 100644 index 0000000..8465c74 --- /dev/null +++ b/test/lib/firehose/inprocess_cc.go @@ -0,0 +1,101 @@ +// In-process captive-core fetcher. +// +// Wraps github.com/streamingfast/firehose-stellar/captivecore directly: +// supervises a stellar-core subprocess from the test runner, no docker +// container. Requires the stellar-core binary on the host (homebrew on +// macOS, apt on Linux). +// +// Same library used by `firestellar fetch captive-core`. Calling it +// directly skips the firecore reader-node + merger plumbing the +// production CLI needs. +package firehose + +import ( + "context" + "fmt" + "sync" + + "github.com/streamingfast/firehose-stellar/captivecore" + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" +) + +// InProcessCaptiveCoreFetcher implements Fetcher by driving a captivecore.Backend +// in-process. The first FetchBlock call lazily calls PrepareRange with that +// ledger as the start; subsequent calls fetch via the same prepared range. +// +// stellar-core runs as a subprocess of the test runner — defer Close() +// to terminate it cleanly. +type InProcessCaptiveCoreFetcher struct { + name string + backend *captivecore.Backend + + mu sync.Mutex + rangePrepared bool + preparedLedger uint64 +} + +// NewInProcessCaptiveCoreFetcher constructs a fetcher. The Config mirrors +// captivecore.Config; this is just a thin wrapper that adds a Name and +// the lazy PrepareRange behavior battlefield needs. +func NewInProcessCaptiveCoreFetcher(name string, cfg captivecore.Config) (*InProcessCaptiveCoreFetcher, error) { + if name == "" { + name = "captive-core" + } + backend, err := captivecore.New(cfg) + if err != nil { + return nil, fmt.Errorf("inprocess captive-core: %w", err) + } + return &InProcessCaptiveCoreFetcher{ + name: name, + backend: backend, + }, nil +} + +// Name returns the fetcher identifier. +func (f *InProcessCaptiveCoreFetcher) Name() string { return f.name } + +// FetchBlock returns the stellar block at ledger. On first call, +// PrepareRange is invoked with this ledger as the start — meaning the +// stellar-core subprocess catches up from scratch. Subsequent calls +// hit the prepared range directly. +// +// If the caller asks for a ledger BEFORE the prepared range, returns an +// error — captive-core can't rewind without restarting stellar-core. +// Most battlefield scenarios submit a tx and immediately fetch its +// ledger, so the natural call pattern is monotonically increasing +// ledger numbers, which works fine. +func (f *InProcessCaptiveCoreFetcher) FetchBlock(ctx context.Context, ledger uint64) (*pbstellar.Block, error) { + f.mu.Lock() + if !f.rangePrepared { + // First call dictates the range start. PrepareRange spawns + // stellar-core and waits for it to catch up to `ledger` — can + // take 5-30s depending on archive replay distance. + if err := f.backend.PrepareRange(ctx, ledger); err != nil { + f.mu.Unlock() + return nil, fmt.Errorf("inprocess captive-core prepare range from %d: %w", ledger, err) + } + f.rangePrepared = true + f.preparedLedger = ledger + } else if ledger < f.preparedLedger { + f.mu.Unlock() + return nil, fmt.Errorf("inprocess captive-core: ledger %d is below prepared start %d (captive-core can't rewind)", + ledger, f.preparedLedger) + } + f.mu.Unlock() + + bstreamBlock, err := f.backend.GetBlock(ctx, ledger) + if err != nil { + return nil, fmt.Errorf("inprocess captive-core fetch ledger %d: %w", ledger, err) + } + return unmarshalStellarBlock(bstreamBlock) +} + +// Close terminates the stellar-core subprocess. Idempotent. +func (f *InProcessCaptiveCoreFetcher) Close() error { + if f.backend == nil { + return nil + } + err := f.backend.Close() + f.backend = nil + return err +} diff --git a/test/lib/firehose/inprocess_rpc.go b/test/lib/firehose/inprocess_rpc.go new file mode 100644 index 0000000..6e476b8 --- /dev/null +++ b/test/lib/firehose/inprocess_rpc.go @@ -0,0 +1,141 @@ +// In-process RPC fetcher. +// +// Wraps github.com/streamingfast/firehose-stellar/rpc directly: no container, +// no firecore reader-node, no dbin files. Tests instantiate an InProcessRPCFetcher +// pointed at the host-side soroban-rpc (the quickstart container) and ask +// for blocks one at a time. +// +// This is the production rpc.Fetcher used by `firestellar fetch rpc`, just +// called from Go instead of from a long-running firecore subprocess. +package firehose + +import ( + "context" + "fmt" + "sync" + "time" + + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" + "github.com/streamingfast/firehose-stellar/rpc" + "github.com/streamingfast/logging" + "go.uber.org/zap" +) + +// inProcessTracer is the package-level Tracer required by rpc.NewClient. +// The logging package only exposes Tracer construction via PackageLogger, +// so we register one for this package once and pass it through. +var _, inProcessTracer = logging.PackageLogger("test_inprocess", "github.com/streamingfast/firehose-stellar/test/lib/firehose") + +// InProcessRPCFetcher implements Fetcher by calling rpc.Fetcher.Fetch in +// the caller's process. Useful for integration tests that want byte- +// equivalent output to the production poller without the docker overhead. +type InProcessRPCFetcher struct { + name string + client *rpc.Client + fetcher *rpc.Fetcher + + // rpc.Fetcher caches the last-seen ledger and won't drive its + // internal "wait for chain to catch up" loop concurrently. Serialize + // FetchBlock calls so concurrent tests are safe. + mu sync.Mutex +} + +// InProcessRPCConfig configures an InProcessRPCFetcher. Defaults are +// tuned for a local quickstart container (fast ticks, no retry pressure). +type InProcessRPCConfig struct { + // Name is the fetcher identifier reported via Name(). Defaults to "rpc". + Name string + + // RPCEndpoint is the soroban-rpc URL, e.g. + // "http://localhost:8000/soroban/rpc". Required. + RPCEndpoint string + + // NetworkPassphrase must match the chain — Stellar uses it to + // recompute transaction hashes from ledger metadata. For a local + // quickstart in --local mode this is the standalone passphrase + // "Standalone Network ; February 2017". + NetworkPassphrase string + + // FetchInterval controls the rpc.Fetcher's internal pacing between + // successive ledger reads. Defaults to 1s. + FetchInterval time.Duration + + // LatestBlockRetryInterval controls how often the fetcher polls for + // the chain's latest ledger when waiting for a future block. + // Defaults to 500ms (quickstart ticks every 5s). + LatestBlockRetryInterval time.Duration + + // TransactionFetchLimit caps the number of transactions per + // getTransactions paginated call. Defaults to 200 (rpc default). + TransactionFetchLimit int + + // Logger receives fetcher events. Optional — defaults to a no-op + // zap logger when nil. + Logger *zap.Logger +} + +// NewInProcessRPCFetcher constructs a fetcher ready to call FetchBlock. +// Validates required fields, fills defaults, and dials the rpc.Client. +func NewInProcessRPCFetcher(cfg InProcessRPCConfig) (*InProcessRPCFetcher, error) { + if cfg.RPCEndpoint == "" { + return nil, fmt.Errorf("inprocess rpc: RPCEndpoint is required") + } + if cfg.NetworkPassphrase == "" { + return nil, fmt.Errorf("inprocess rpc: NetworkPassphrase is required") + } + name := cfg.Name + if name == "" { + name = "rpc" + } + if cfg.FetchInterval == 0 { + cfg.FetchInterval = 1 * time.Second + } + if cfg.LatestBlockRetryInterval == 0 { + cfg.LatestBlockRetryInterval = 500 * time.Millisecond + } + if cfg.TransactionFetchLimit == 0 { + cfg.TransactionFetchLimit = 200 + } + logger := cfg.Logger + if logger == nil { + logger = zap.NewNop() + } + + client := rpc.NewClient(cfg.RPCEndpoint, logger, inProcessTracer) + fetcher := rpc.NewFetcher( + cfg.FetchInterval, + cfg.LatestBlockRetryInterval, + cfg.TransactionFetchLimit, + cfg.NetworkPassphrase, + logger, + ) + return &InProcessRPCFetcher{ + name: name, + client: client, + fetcher: fetcher, + }, nil +} + +// Name returns the fetcher identifier. +func (f *InProcessRPCFetcher) Name() string { return f.name } + +// FetchBlock pulls one ledger from soroban-rpc. If `skipped` is true (the +// rpc fetcher's signal that the requested ledger doesn't exist on chain) +// it's surfaced as an error so the caller's retry loop kicks in. +func (f *InProcessRPCFetcher) FetchBlock(ctx context.Context, ledger uint64) (*pbstellar.Block, error) { + f.mu.Lock() + defer f.mu.Unlock() + + bstreamBlock, skipped, err := f.fetcher.Fetch(ctx, f.client, ledger) + if err != nil { + return nil, fmt.Errorf("inprocess rpc fetch ledger %d: %w", ledger, err) + } + if skipped { + return nil, fmt.Errorf("inprocess rpc: ledger %d skipped (gap or pre-genesis)", ledger) + } + return unmarshalStellarBlock(bstreamBlock) +} + +// Close is a no-op for the rpc fetcher — there's no long-lived process +// or connection to release. Provided to satisfy the Fetcher interface. +func (f *InProcessRPCFetcher) Close() error { return nil } diff --git a/test/lib/runner/runner.go b/test/lib/runner/runner.go new file mode 100644 index 0000000..454620e --- /dev/null +++ b/test/lib/runner/runner.go @@ -0,0 +1,264 @@ +// Package runner submits a transaction, pulls the resulting block from +// every configured firehose Fetcher, asserts they agree, and compares the +// canonical view against a JSON snapshot. +package runner + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/txnbuild" + + "github.com/streamingfast/firehose-stellar/test/lib/firehose" + "github.com/streamingfast/firehose-stellar/test/lib/snapshot" + "github.com/streamingfast/firehose-stellar/test/lib/stellar" + libxdr "github.com/streamingfast/firehose-stellar/test/lib/xdr" +) + +type Config struct { + // Fetchers is the list of firehose backends the runner reads blocks + // from. Required; TestMain typically constructs poller + + // captive-core in-process fetchers and assigns them here. + Fetchers []firehose.Fetcher + + SnapshotRoot string // base dir for snapshots/ + WaitTimeout time.Duration // poll deadline per ledger + WaitInterval time.Duration + + // AccountSeedScope, when non-empty, makes keypair generation + // deterministic: NewFundedAccount("foo") derives from + // sha256(scope + "/" + name). Tests set this to t.Name() so account + // addresses are byte-stable across runs (snapshots commit literal + // G... strings). + AccountSeedScope string +} + +// DefaultConfig fills snapshot + timeout fields from env vars. Fetchers +// is left to the caller — set it before calling New. +func DefaultConfig() Config { + return Config{ + SnapshotRoot: envOr("BATTLEFIELD_SNAPSHOTS", "snapshots"), + WaitTimeout: envDuration("BATTLEFIELD_WAIT_TIMEOUT", 90*time.Second), + WaitInterval: envDuration("BATTLEFIELD_WAIT_INTERVAL", 2*time.Second), + } +} + +type Runner struct { + Config Config + Stellar *stellar.Client + Fetchers []firehose.Fetcher +} + +func New(cfg Config) (*Runner, error) { + if len(cfg.Fetchers) == 0 { + return nil, fmt.Errorf("runner: Config.Fetchers is required") + } + sc, err := stellar.NewClient() + if err != nil { + return nil, err + } + return &Runner{ + Config: cfg, + Stellar: sc, + Fetchers: cfg.Fetchers, + }, nil +} + +// NewFundedAccount creates a friendbot-funded account. +// +// If Config.AccountSeedScope is set, the keypair is derived +// deterministically from sha256(scope + "/" + name) — see Config docs. +// Otherwise it's freshly random. The deterministic case is the supported +// one for snapshot tests: addresses appear as literal G... strings in +// committed snapshots and remain byte-stable across runs. +func (r *Runner) NewFundedAccount(name string) (*keypair.Full, error) { + kp, err := r.deriveKeypair(name) + if err != nil { + return nil, err + } + if err := r.Stellar.FundAccount(kp.Address()); err != nil { + return nil, fmt.Errorf("fund %s: %w", kp.Address(), err) + } + return kp, nil +} + +func (r *Runner) MustNewFundedAccount(name string) *keypair.Full { + kp, err := r.NewFundedAccount(name) + if err != nil { + panic(fmt.Errorf("fund %q: %w", name, err)) + } + return kp +} + +// NewUnfundedAccount returns a keypair derived from `name` without +// calling friendbot. Same determinism rules as NewFundedAccount. +// +// Use this for the target of a CreateAccount operation — friendbot- +// funding it first would make CreateAccount fail with op_already_exists. +func (r *Runner) NewUnfundedAccount(name string) (*keypair.Full, error) { + kp, err := r.deriveKeypair(name) + if err != nil { + return nil, err + } + return kp, nil +} + +func (r *Runner) MustNewUnfundedAccount(name string) *keypair.Full { + kp, err := r.NewUnfundedAccount(name) + if err != nil { + panic(err) + } + return kp +} + +// deriveKeypair produces a keypair for the logical role `name`. If a +// seed scope is configured, the seed is sha256(scope + "/" + name) so +// keypairs are identical across runs for the same (scope, name) — this +// makes SAC contract IDs (which are hashes of issuer pubkeys) byte- +// stable across the record-then-validate workflow. +func (r *Runner) deriveKeypair(name string) (*keypair.Full, error) { + if r.Config.AccountSeedScope == "" { + kp, err := keypair.Random() + if err != nil { + return nil, fmt.Errorf("generate keypair %q: %w", name, err) + } + return kp, nil + } + seed := sha256.Sum256([]byte(r.Config.AccountSeedScope + "/" + name)) + kp, err := keypair.FromRawSeed(seed) + if err != nil { + return nil, fmt.Errorf("derive keypair %q from scope %q: %w", name, r.Config.AccountSeedScope, err) + } + return kp, nil +} + +// RunScenario submits the operations, waits for the resulting ledger to land +// in every configured fetcher, decodes the transaction from each, asserts +// they all agree, and compares the canonical view against the snapshot at +// /.expected.json. +func (r *Runner) RunScenario(id string, ops []txnbuild.Operation, source *keypair.Full, extraSigners ...*keypair.Full) error { + resp, err := r.Stellar.SubmitOps(source, ops, extraSigners...) + if err != nil { + return fmt.Errorf("submit %s: %w", id, err) + } + return r.AssertScenario(id, uint64(resp.Ledger), resp.Hash) +} + +// AssertScenario runs the post-submission half of RunScenario against a tx +// hash that was produced by something other than `SubmitOps` — e.g. a fee +// bump submission, a multi-sig flow, or a horizon-bypassing direct submit. +func (r *Runner) AssertScenario(id string, ledger uint64, txHash string) error { + views, canonical, err := r.fetchAndDecodeFromAll(ledger, txHash) + if err != nil { + return err + } + if err := r.assertFetchersAgree(id, views); err != nil { + return err + } + return r.compareAgainstSnapshot(id, canonical, ledger) +} + +// fetchAndDecodeFromAll fetches the ledger from every fetcher, finds the +// transaction by hash in each block, decodes it structurally, and returns +// (per-fetcher view, canonical view, error). The canonical view is taken +// from the first fetcher in the configured order. +func (r *Runner) fetchAndDecodeFromAll(ledger uint64, txHash string) (map[string]*libxdr.TxView, *libxdr.TxView, error) { + views := map[string]*libxdr.TxView{} + var canonical *libxdr.TxView + + for _, f := range r.Fetchers { + ctx, cancel := context.WithTimeout(context.Background(), r.Config.WaitTimeout) + block, err := firehose.WaitForBlock(ctx, f, ledger, r.Config.WaitInterval) + cancel() + if err != nil { + return nil, nil, fmt.Errorf("fetcher %s wait for ledger %d: %w", f.Name(), ledger, err) + } + + tx, _, err := firehose.FindTxByHash(block, txHash) + if err != nil { + return nil, nil, fmt.Errorf("fetcher %s: %w", f.Name(), err) + } + + view, err := libxdr.FromTransaction(tx) + if err != nil { + return nil, nil, fmt.Errorf("fetcher %s decode: %w", f.Name(), err) + } + views[f.Name()] = view + if canonical == nil { + canonical = view + } + } + return views, canonical, nil +} + +// assertFetchersAgree compares the structural view emitted by every fetcher +// against the canonical (first) view. Any difference is a fetcher bug and +// fails the scenario regardless of snapshot status. +func (r *Runner) assertFetchersAgree(id string, views map[string]*libxdr.TxView) error { + if len(views) <= 1 { + return nil + } + + canonicalName := r.Fetchers[0].Name() + canonical := views[canonicalName] + for _, f := range r.Fetchers[1:] { + other := views[f.Name()] + if err := snapshot.DiffViews(canonical, other); err != nil { + return fmt.Errorf("fetcher disagreement for %s: %s vs %s:\n%w", + id, canonicalName, f.Name(), err) + } + } + return nil +} + +func (r *Runner) compareAgainstSnapshot(id string, view *libxdr.TxView, ledger uint64) error { + snap, err := snapshot.Load(filepath.Join(r.Config.SnapshotRoot, id+".expected.json")) + if err != nil { + return err + } + // Account addresses are deterministic across runs (derived from + // sha256(t.Name()+"/"+name) when AccountSeedScope is set), so we let + // them appear as literal G... strings in snapshots instead of + // templating them. This makes the snapshot diff stronger: any drift + // in an emitted address fails the byte-equality check directly. + // + // Renaming a Test… function changes the scope and therefore every + // derived address, so the affected snapshot must be regenerated: + // SNAPSHOTS_UPDATE=^$ go test ./test/scenarios/... + snap.Bind("ledger", strconv.FormatUint(ledger, 10)) + snap.Bind("hash", view.Hash) + snap.Bind("createdAt", view.CreatedAt) + return snap.Compare(view) +} + +// Close releases all fetcher resources. +func (r *Runner) Close() { + for _, f := range r.Fetchers { + _ = f.Close() + } +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envDuration(key string, def time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil { + return def + } + return d +} diff --git a/test/lib/snapshot/diff.go b/test/lib/snapshot/diff.go new file mode 100644 index 0000000..32990b6 --- /dev/null +++ b/test/lib/snapshot/diff.go @@ -0,0 +1,49 @@ +package snapshot + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// DiffViews structurally compares two arbitrary values (typically *xdr.TxView +// instances coming from different backends) and returns nil iff they are +// deep-equal. Errors include a path-prefixed list of differences to make +// backend disagreements debuggable. +// +// Unlike Snapshot.Compare, this does no $var substitution — both inputs are +// expected to already be in the same canonical form. +func DiffViews(a, b any) error { + aMap, err := toGeneric(a) + if err != nil { + return fmt.Errorf("normalize a: %w", err) + } + bMap, err := toGeneric(b) + if err != nil { + return fmt.Errorf("normalize b: %w", err) + } + diffs := diff("", aMap, bMap) + if len(diffs) == 0 { + return nil + } + return fmt.Errorf("%s", strings.Join(diffs, "\n - ")) +} + +func toGeneric(v any) (any, error) { + blob, err := json.Marshal(v) + if err != nil { + return nil, err + } + // UseNumber so large integers survive as json.Number instead of + // being coerced to float64. Without this, backend diffs above 2^53 + // (Stellar stroop amounts, sequence numbers) silently compare equal + // even when the underlying ints differ. + dec := json.NewDecoder(bytes.NewReader(blob)) + dec.UseNumber() + var out any + if err := dec.Decode(&out); err != nil { + return nil, err + } + return out, nil +} diff --git a/test/lib/snapshot/snapshot.go b/test/lib/snapshot/snapshot.go new file mode 100644 index 0000000..fdd69db --- /dev/null +++ b/test/lib/snapshot/snapshot.go @@ -0,0 +1,239 @@ +// Package snapshot provides JSON expected-vs-actual comparison with $var +// templating, modeled on the battlefield-ethereum snapshot library. +// +// Workflow: +// +// expected, err := snapshot.Load("snapshots/payment/native.expected.json") +// expected.Bind("source", srcAddr) +// expected.Bind("dest", destAddr) +// if err := expected.Compare(actual); err != nil { … } +// +// When SNAPSHOTS_UPDATE matches the snapshot path (regex), Compare writes +// `actual` to the expected file with bound variables re-substituted as $var +// placeholders, instead of returning a diff. +package snapshot + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" +) + +type Snapshot struct { + Path string + Expected any + bindings map[string]string +} + +// Load reads an expected snapshot from disk. A missing file is treated as an +// empty expectation (useful for first-run regeneration via SNAPSHOTS_UPDATE). +func Load(path string) (*Snapshot, error) { + s := &Snapshot{Path: path, bindings: map[string]string{}} + + blob, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return s, nil + } + return nil, fmt.Errorf("read snapshot %s: %w", path, err) + } + // UseNumber so large integers in the snapshot file (sequence numbers, + // stroop amounts) keep their precision instead of being decoded as + // float64 — matches how `normalize` decodes the actual side. + dec := json.NewDecoder(bytes.NewReader(blob)) + dec.UseNumber() + if err := dec.Decode(&s.Expected); err != nil { + return nil, fmt.Errorf("parse snapshot %s: %w", path, err) + } + return s, nil +} + +// Bind associates a $var placeholder with a runtime value. During Compare, +// every occurrence of the value in `actual` is replaced with `$name` before +// the deep equality check, so snapshots stay stable across runs. +func (s *Snapshot) Bind(name, value string) { + if value == "" { + return + } + s.bindings["$"+name] = value +} + +// Compare normalizes `actual` by substituting bound values with their $var +// placeholders, then deep-compares against the expected JSON. +// +// If the SNAPSHOTS_UPDATE env var is set and matches s.Path as a regex, the +// normalized actual is written to disk and Compare returns nil. +func (s *Snapshot) Compare(actual any) error { + normalized := s.normalize(actual) + + if pattern := os.Getenv("SNAPSHOTS_UPDATE"); pattern != "" { + matched, err := regexp.MatchString(pattern, s.Path) + if err != nil { + return fmt.Errorf("invalid SNAPSHOTS_UPDATE regex %q: %w", pattern, err) + } + if matched { + return s.write(normalized) + } + } + + if s.Expected == nil { + return fmt.Errorf("no snapshot at %s — set SNAPSHOTS_UPDATE=%s to record one", s.Path, regexp.QuoteMeta(s.Path)) + } + + diffs := diff("", s.Expected, normalized) + if len(diffs) == 0 { + return nil + } + return fmt.Errorf("snapshot mismatch %s:\n - %s", s.Path, strings.Join(diffs, "\n - ")) +} + +func (s *Snapshot) write(normalized any) error { + if err := os.MkdirAll(filepath.Dir(s.Path), 0o755); err != nil { + return fmt.Errorf("mkdir for %s: %w", s.Path, err) + } + blob, err := json.MarshalIndent(normalized, "", " ") + if err != nil { + return fmt.Errorf("marshal snapshot: %w", err) + } + if err := os.WriteFile(s.Path, append(blob, '\n'), 0o644); err != nil { + return fmt.Errorf("write %s: %w", s.Path, err) + } + return nil +} + +func (s *Snapshot) normalize(v any) any { + blob, err := json.Marshal(v) + if err != nil { + return v + } + // UseNumber so large integers survive the JSON round-trip as + // json.Number; otherwise Stellar sequence numbers / stroop amounts + // > 2^53 lose precision and snapshot diffs silently mis-compare. + dec := json.NewDecoder(bytes.NewReader(blob)) + dec.UseNumber() + var copy any + if err := dec.Decode(©); err != nil { + return v + } + reps := sortedReplacements(s.bindings) + return walk(copy, reps) +} + +func walk(v any, reps []replacement) any { + switch x := v.(type) { + case map[string]any: + out := map[string]any{} + for k, val := range x { + out[k] = walk(val, reps) + } + return out + case []any: + out := make([]any, len(x)) + for i, val := range x { + out[i] = walk(val, reps) + } + return out + case string: + return substitute(x, reps) + default: + return v + } +} + +type replacement struct { + from string + to string +} + +// sortedReplacements returns longest-value-first so that overlapping bindings +// don't shadow each other (e.g. "ABCDEF" replaces before "ABC"). +func sortedReplacements(bindings map[string]string) []replacement { + out := make([]replacement, 0, len(bindings)) + for placeholder, value := range bindings { + out = append(out, replacement{from: value, to: placeholder}) + } + sort.Slice(out, func(i, j int) bool { return len(out[i].from) > len(out[j].from) }) + return out +} + +func substitute(s string, reps []replacement) string { + for _, r := range reps { + if r.from == "" { + continue + } + s = strings.ReplaceAll(s, r.from, r.to) + } + return s +} + +func diff(path string, expected, actual any) []string { + if reflect.DeepEqual(expected, actual) { + return nil + } + switch e := expected.(type) { + case map[string]any: + a, ok := actual.(map[string]any) + if !ok { + return []string{fmt.Sprintf("%s: expected object, got %T", displayPath(path), actual)} + } + var out []string + seen := map[string]bool{} + for k, ev := range e { + seen[k] = true + out = append(out, diff(joinPath(path, k), ev, a[k])...) + } + for k, av := range a { + if seen[k] { + continue + } + out = append(out, fmt.Sprintf("%s: unexpected key with value %s", displayPath(joinPath(path, k)), summary(av))) + } + return out + case []any: + a, ok := actual.([]any) + if !ok { + return []string{fmt.Sprintf("%s: expected array, got %T", displayPath(path), actual)} + } + if len(e) != len(a) { + return []string{fmt.Sprintf("%s: array length differs — expected %d, actual %d", displayPath(path), len(e), len(a))} + } + var out []string + for i := range e { + out = append(out, diff(fmt.Sprintf("%s[%d]", path, i), e[i], a[i])...) + } + return out + default: + return []string{fmt.Sprintf("%s: expected %s, actual %s", displayPath(path), summary(expected), summary(actual))} + } +} + +func joinPath(p, k string) string { + if p == "" { + return k + } + return p + "." + k +} + +func displayPath(p string) string { + if p == "" { + return "" + } + return p +} + +func summary(v any) string { + blob, err := json.Marshal(v) + if err != nil { + return fmt.Sprintf("%v", v) + } + if len(blob) > 200 { + return string(blob[:200]) + "…" + } + return string(blob) +} diff --git a/test/lib/snapshot/snapshot_test.go b/test/lib/snapshot/snapshot_test.go new file mode 100644 index 0000000..7dcd8c6 --- /dev/null +++ b/test/lib/snapshot/snapshot_test.go @@ -0,0 +1,81 @@ +package snapshot + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCompareWithBindings(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "case.expected.json") + + expected := `{ + "from": "$source", + "to": "$dest", + "amount": "10", + "hash": "$hash" +}` + if err := os.WriteFile(path, []byte(expected), 0o644); err != nil { + t.Fatal(err) + } + + s, err := Load(path) + if err != nil { + t.Fatal(err) + } + s.Bind("source", "GAAA") + s.Bind("dest", "GBBB") + s.Bind("hash", "abcd") + + actual := map[string]any{ + "from": "GAAA", + "to": "GBBB", + "amount": "10", + "hash": "abcd", + } + if err := s.Compare(actual); err != nil { + t.Fatalf("expected match: %v", err) + } + + mismatch := map[string]any{ + "from": "GAAA", + "to": "GCCC", // diverges + "amount": "10", + "hash": "abcd", + } + err = s.Compare(mismatch) + if err == nil { + t.Fatal("expected mismatch error") + } + if !strings.Contains(err.Error(), "to:") { + t.Fatalf("expected diff to mention `to:`, got: %v", err) + } +} + +func TestSnapshotsUpdate(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "regen.expected.json") + + t.Setenv("SNAPSHOTS_UPDATE", ".") + + s, err := Load(path) + if err != nil { + t.Fatal(err) + } + s.Bind("source", "GAAA") + + actual := map[string]any{"from": "GAAA", "amount": "1"} + if err := s.Compare(actual); err != nil { + t.Fatal(err) + } + + blob, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(blob), `"$source"`) { + t.Fatalf("expected $source placeholder in regenerated snapshot, got:\n%s", blob) + } +} diff --git a/test/lib/stellar/client.go b/test/lib/stellar/client.go new file mode 100644 index 0000000..126f0ae --- /dev/null +++ b/test/lib/stellar/client.go @@ -0,0 +1,171 @@ +// Package stellar wraps the stellar/go-stellar-sdk to provide the high-level +// operations battlefield scenarios need: account creation/funding, +// transaction building/signing/submission, and minting custom assets. +// +// The dev stack runs a stellar/quickstart docker in --local mode, so the +// client is hardcoded to its standalone-network endpoints (horizon at +// :8000, friendbot at :8000/friendbot). +package stellar + +import ( + "fmt" + "io" + "net/http" + "time" + + "github.com/stellar/go-stellar-sdk/clients/horizonclient" + "github.com/stellar/go-stellar-sdk/keypair" + hProtocol "github.com/stellar/go-stellar-sdk/protocols/horizon" + "github.com/stellar/go-stellar-sdk/txnbuild" +) + +// StandaloneNetworkPassphrase is the canonical passphrase the stellar/quickstart +// docker image uses in --local mode. There is no constant for it in the +// stellar SDK, so we hardcode the official string. +const StandaloneNetworkPassphrase = "Standalone Network ; February 2017" + +type Client struct { + Horizon *horizonclient.Client + NetworkPhrase string + FriendbotURL string + httpClient *http.Client +} + +// NewClient returns a Client wired to the local stellar/quickstart standalone +// network: horizon at http://localhost:8000, friendbot at the same host. +func NewClient() (*Client, error) { + httpClient := &http.Client{Timeout: 30 * time.Second} + return &Client{ + Horizon: &horizonclient.Client{ + HorizonURL: "http://localhost:8000/", + HTTP: httpClient, + }, + NetworkPhrase: StandaloneNetworkPassphrase, + FriendbotURL: "http://localhost:8000/friendbot", + httpClient: httpClient, + }, nil +} + +// FundAccount asks friendbot to fund the given address. +// +// Retries transient 5xx responses with capped exponential backoff. The +// quickstart friendbot is prone to 502 Bad Gateway in the seconds +// immediately after a chain reset — supervisord brings up the +// horizon/friendbot/soroban-rpc stack with some startup race that takes +// 10–30s to settle even after horizon's `/` endpoint returns 200. So we +// retry generously (up to ~90s of retries) before giving up. +// +// 4xx errors are treated as permanent (e.g. address already funded). +func (c *Client) FundAccount(address string) error { + const maxAttempts = 15 + const maxDelay = 8 * time.Second + delay := 500 * time.Millisecond + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + resp, err := c.httpClient.Get(c.FriendbotURL + "?addr=" + address) + if err != nil { + lastErr = fmt.Errorf("friendbot request: %w", err) + } else { + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode < 400 { + return nil + } + lastErr = fmt.Errorf("friendbot returned %d: %s", resp.StatusCode, truncate(body, 200)) + // 4xx (bad request, address already funded) is permanent — bail. + if resp.StatusCode < 500 { + return lastErr + } + } + + if attempt < maxAttempts { + time.Sleep(delay) + delay *= 2 + if delay > maxDelay { + delay = maxDelay + } + } + } + return fmt.Errorf("friendbot %s after %d attempts: %w", address, maxAttempts, lastErr) +} + +func truncate(b []byte, n int) string { + if len(b) <= n { + return string(b) + } + return string(b[:n]) + "…" +} + +// LoadAccount fetches the on-chain account state for the given address. +func (c *Client) LoadAccount(address string) (hProtocol.Account, error) { + return c.Horizon.AccountDetail(horizonclient.AccountRequest{AccountID: address}) +} + +// SubmitOps builds, signs and submits a transaction with the given operations, +// signed by the given source keypair (and any extra cosigners). Returns the +// horizon submit response, which includes the canonical transaction hash. +func (c *Client) SubmitOps(source *keypair.Full, ops []txnbuild.Operation, extraSigners ...*keypair.Full) (hProtocol.Transaction, error) { + srcAccount, err := c.LoadAccount(source.Address()) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("load source account: %w", err) + } + + tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ + SourceAccount: &srcAccount, + IncrementSequenceNum: true, + BaseFee: txnbuild.MinBaseFee, + Preconditions: txnbuild.Preconditions{TimeBounds: txnbuild.NewInfiniteTimeout()}, + Operations: ops, + }) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("build tx: %w", err) + } + + signers := append([]*keypair.Full{source}, extraSigners...) + for _, s := range signers { + tx, err = tx.Sign(c.NetworkPhrase, s) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("sign tx with %s: %w", s.Address(), err) + } + } + + resp, err := c.Horizon.SubmitTransaction(tx) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("submit tx: %w", err) + } + return resp, nil +} + +// SubmitOpsExpectFail builds and submits a transaction expected to fail at +// horizon submission. It returns nil only when horizon itself rejects the +// transaction (the expected case). Non-submission errors (build/sign, horizon +// unreachable) are propagated so callers don't get false positives. If the +// transaction unexpectedly succeeds, an error is returned. +func (c *Client) SubmitOpsExpectFail(source *keypair.Full, ops []txnbuild.Operation, extraSigners ...*keypair.Full) error { + resp, err := c.SubmitOps(source, ops, extraSigners...) + if err != nil { + if _, ok := err.(*horizonclient.Error); ok { + return nil + } + if wrapped := unwrapHorizonError(err); wrapped != nil { + return nil + } + return fmt.Errorf("submit tx (expected horizon failure): %w", err) + } + return fmt.Errorf("expected submission to fail, got hash=%s ledger=%d", resp.Hash, resp.Ledger) +} + +func unwrapHorizonError(err error) *horizonclient.Error { + for err != nil { + if he, ok := err.(*horizonclient.Error); ok { + return he + } + u, ok := err.(interface{ Unwrap() error }) + if !ok { + return nil + } + err = u.Unwrap() + } + return nil +} diff --git a/test/lib/stellar/feebump.go b/test/lib/stellar/feebump.go new file mode 100644 index 0000000..bc841f8 --- /dev/null +++ b/test/lib/stellar/feebump.go @@ -0,0 +1,53 @@ +package stellar + +import ( + "fmt" + + "github.com/stellar/go-stellar-sdk/keypair" + hProtocol "github.com/stellar/go-stellar-sdk/protocols/horizon" + "github.com/stellar/go-stellar-sdk/txnbuild" +) + +// SubmitFeeBump wraps an inner-signed transaction in a fee-bump envelope +// signed by `payer`, then submits it. Useful for surfacing fee-bump-specific +// envelope encoding (the firehose poller and captive-core paths historically +// disagreed on FeeBumpTransactionEnvelope decoding). +func (c *Client) SubmitFeeBump(payer *keypair.Full, innerSource *keypair.Full, ops []txnbuild.Operation) (hProtocol.Transaction, error) { + innerAccount, err := c.LoadAccount(innerSource.Address()) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("load inner account: %w", err) + } + inner, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ + SourceAccount: &innerAccount, + IncrementSequenceNum: true, + BaseFee: txnbuild.MinBaseFee, + Preconditions: txnbuild.Preconditions{TimeBounds: txnbuild.NewInfiniteTimeout()}, + Operations: ops, + }) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("build inner: %w", err) + } + inner, err = inner.Sign(c.NetworkPhrase, innerSource) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("sign inner: %w", err) + } + + bump, err := txnbuild.NewFeeBumpTransaction(txnbuild.FeeBumpTransactionParams{ + Inner: inner, + FeeAccount: payer.Address(), + BaseFee: txnbuild.MinBaseFee * 10, + }) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("build fee bump: %w", err) + } + bump, err = bump.Sign(c.NetworkPhrase, payer) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("sign fee bump: %w", err) + } + + resp, err := c.Horizon.SubmitFeeBumpTransaction(bump) + if err != nil { + return hProtocol.Transaction{}, fmt.Errorf("submit fee bump: %w", err) + } + return resp, nil +} diff --git a/test/lib/stellar/ops.go b/test/lib/stellar/ops.go new file mode 100644 index 0000000..bb30d9f --- /dev/null +++ b/test/lib/stellar/ops.go @@ -0,0 +1,42 @@ +package stellar + +import ( + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/txnbuild" +) + +// Payment builds a payment operation. Pass txnbuild.NativeAsset{} for XLM +// or txnbuild.CreditAsset{Code, Issuer} for a custom asset. +func Payment(destination, amount string, asset txnbuild.Asset) *txnbuild.Payment { + return &txnbuild.Payment{Destination: destination, Amount: amount, Asset: asset} +} + +// CreateAccount builds a CreateAccount operation, used for funding a new +// account from an existing one (the alternative to friendbot). +func CreateAccount(destination, startingBalance string) *txnbuild.CreateAccount { + return &txnbuild.CreateAccount{Destination: destination, Amount: startingBalance} +} + +// ChangeTrust builds a trustline-creation operation for the given credit asset. +// Trustlines apply only to non-native assets, so the parameter type rules out +// txnbuild.NativeAsset at compile time. +func ChangeTrust(asset txnbuild.CreditAsset, limit string) *txnbuild.ChangeTrust { + return &txnbuild.ChangeTrust{Line: txnbuild.ChangeTrustAssetWrapper{Asset: asset}, Limit: limit} +} + +// ManageData stores a key/value pair on an account. Useful for "boring" +// scenarios that exercise no asset movement. +func ManageData(name, value string) *txnbuild.ManageData { + return &txnbuild.ManageData{Name: name, Value: []byte(value)} +} + +// AccountMerge merges the source account into the destination, sweeping its +// XLM balance. +func AccountMerge(destination string) *txnbuild.AccountMerge { + return &txnbuild.AccountMerge{Destination: destination} +} + +// CreditAsset is a convenience constructor. +func CreditAsset(code string, issuer *keypair.Full) txnbuild.CreditAsset { + return txnbuild.CreditAsset{Code: code, Issuer: issuer.Address()} +} diff --git a/test/lib/xdr/decode.go b/test/lib/xdr/decode.go new file mode 100644 index 0000000..0faf442 --- /dev/null +++ b/test/lib/xdr/decode.go @@ -0,0 +1,488 @@ +// Package xdr decodes Stellar XDR payloads (envelope, result, events) into +// JSON-friendly Go values. Battlefield snapshots compare structurally rather +// than byte-for-byte so that signatures, sequence numbers and other run-local +// noise can be normalized. +package xdr + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/stellar/go-stellar-sdk/strkey" + sdkxdr "github.com/stellar/go-stellar-sdk/xdr" + + pbstellar "github.com/streamingfast/firehose-stellar/pb/sf/stellar/type/v1" +) + +// TxView is the structural projection of a single transaction we feed into +// snapshot comparison. Field names mirror the Stellar XDR vocabulary so +// snapshot diffs are recognizable. +type TxView struct { + Hash string `json:"hash"` + Status string `json:"status"` + ApplicationOrder uint64 `json:"applicationOrder"` + CreatedAt string `json:"createdAt"` + Envelope map[string]any `json:"envelope"` + Result map[string]any `json:"result"` + Events map[string]any `json:"events,omitempty"` +} + +// FromTransaction decodes a firehose transaction into the structural view. +func FromTransaction(tx *pbstellar.Transaction) (*TxView, error) { + envelope, err := decodeEnvelope(tx.EnvelopeXdr) + if err != nil { + return nil, fmt.Errorf("envelope: %w", err) + } + result, err := decodeResult(tx.ResultXdr) + if err != nil { + return nil, fmt.Errorf("result: %w", err) + } + events, err := decodeEvents(tx.Events) + if err != nil { + return nil, fmt.Errorf("events: %w", err) + } + view := &TxView{ + Hash: fmt.Sprintf("%x", tx.Hash), + Status: tx.Status.String(), + ApplicationOrder: tx.ApplicationOrder, + Envelope: envelope, + Result: result, + Events: events, + } + if tx.CreatedAt != nil { + view.CreatedAt = tx.CreatedAt.AsTime().UTC().Format("2006-01-02T15:04:05Z") + } + return view, nil +} + +func decodeEnvelope(blob []byte) (map[string]any, error) { + var env sdkxdr.TransactionEnvelope + if err := env.UnmarshalBinary(blob); err != nil { + return nil, fmt.Errorf("unmarshal envelope: %w", err) + } + return roundtrip(env) +} + +func decodeResult(blob []byte) (map[string]any, error) { + var res sdkxdr.TransactionResult + if err := res.UnmarshalBinary(blob); err != nil { + return nil, fmt.Errorf("unmarshal result: %w", err) + } + return roundtrip(res) +} + +func decodeEvents(events *pbstellar.Events) (map[string]any, error) { + if events == nil { + return nil, nil + } + out := map[string]any{} + + if diags := decodeDiagnosticEvents(events.DiagnosticEventsXdr); diags != nil { + out["diagnostic"] = diags + } + if txs := decodeTxEvents(events.TransactionEventsXdr); txs != nil { + out["transaction"] = txs + } + if contracts := decodeContractEvents(events.ContractEventsXdr); contracts != nil { + out["contract"] = contracts + } + + if len(out) == 0 { + return nil, nil + } + return out, nil +} + +func decodeDiagnosticEvents(blobs [][]byte) []any { + if len(blobs) == 0 { + return nil + } + var out []any + for _, blob := range blobs { + var ev sdkxdr.DiagnosticEvent + if err := ev.UnmarshalBinary(blob); err != nil { + out = append(out, map[string]any{"_decodeError": err.Error()}) + continue + } + v, err := roundtrip(ev) + if err != nil { + out = append(out, map[string]any{"_decodeError": err.Error()}) + continue + } + out = append(out, v) + } + return out +} + +func decodeTxEvents(blobs [][]byte) []any { + if len(blobs) == 0 { + return nil + } + var out []any + for _, blob := range blobs { + var ev sdkxdr.TransactionEvent + if err := ev.UnmarshalBinary(blob); err != nil { + out = append(out, map[string]any{"_decodeError": err.Error()}) + continue + } + v, err := roundtrip(ev) + if err != nil { + out = append(out, map[string]any{"_decodeError": err.Error()}) + continue + } + out = append(out, v) + } + return out +} + +func decodeContractEvents(groups []*pbstellar.ContractEvent) []any { + var out []any + for _, group := range groups { + var perOp []any + for _, blob := range group.Events { + var ev sdkxdr.ContractEvent + if err := ev.UnmarshalBinary(blob); err != nil { + perOp = append(perOp, map[string]any{"_decodeError": err.Error()}) + continue + } + v, err := roundtrip(ev) + if err != nil { + perOp = append(perOp, map[string]any{"_decodeError": err.Error()}) + continue + } + perOp = append(perOp, v) + } + if perOp != nil { + out = append(out, perOp) + } + } + if len(out) == 0 { + return nil + } + return out +} + +// roundtrip serializes the value to JSON and back to map[string]any so the +// snapshot library has a uniform shape to walk, then applies three +// normalization passes that make snapshots stable and readable: +// +// 1. normalizeAccountIDs collapses `{Ed25519: [32 bytes]}` shapes to +// their strkey "G…" form so $name-substitution can match them. +// +// 2. normalizeDynamicFields replaces per-run-varying fields (signatures, +// signature hints, sequence numbers, ledger seqs, timestamps) with +// fixed placeholder strings. Keypairs and addresses are deterministic +// across runs because runner.Config.AccountSeedScope derives them +// from the test name (so SAC contract IDs are stable too). The drift +// left to template is in horizon-assigned values (transaction +// sequence numbers, ledger numbers, createdAt) and in the signature +// bytes themselves — Ed25519 signatures are deterministic per +// (key, message), but every run produces a different message because +// the templated seqnum and ledger flow into the envelope. +// +// 3. stripNulls removes nil entries from objects. The Go SDK marshals +// discriminated unions as a struct with one populated field plus null +// siblings for every other variant — pruning the nulls cuts snapshot +// size by ~70% and keeps diffs surgical. +func roundtrip(v any) (map[string]any, error) { + blob, err := json.Marshal(v) + if err != nil { + return nil, err + } + // UseNumber so large int64/uint64 values (sequence numbers, stroop + // amounts > 2^53) survive the round-trip as json.Number instead of + // being silently coerced to float64. + dec := json.NewDecoder(bytes.NewReader(blob)) + dec.UseNumber() + var out map[string]any + if err := dec.Decode(&out); err != nil { + return nil, err + } + normalizeAccountIDs(out) + normalizeDynamicFields(out) + stripNulls(out) + return out, nil +} + +// stripNulls walks `v` recursively and deletes any map entry whose value is +// nil (Go's json.Unmarshal renders JSON null as a Go nil interface). Stellar +// XDR discriminated unions are heavy with these — only one variant is set +// per union, the rest marshal to null. +func stripNulls(v any) { + switch x := v.(type) { + case map[string]any: + for k, val := range x { + if val == nil { + delete(x, k) + continue + } + stripNulls(val) + } + case []any: + for _, val := range x { + stripNulls(val) + } + } +} + +// normalizeDynamicFields walks `v` recursively and replaces per-run- +// varying fields with stable placeholder strings: +// +// - DecoratedSignature: {Hint: [4 bytes], Signature: } +// becomes {Hint: "$hint", Signature: "$signature"} +// - Transaction.SeqNum (uint64) → "$seqNum" +// - BumpSequenceOp.BumpTo (uint64) → "$bumpTo" +// - InnerResultPair.TransactionHash ([32]B) → "$innerTransactionHash" +// +// Tests submit a fresh transaction every run; the horizon-assigned +// next-sequence-number for that account is inherently per-run noise. +// Keypairs are deterministic when runner.Config.AccountSeedScope is set +// (the default in the suite, scoped to t.Name()), so account addresses +// and SAC contract IDs derived from them are byte-stable across runs. +// When AccountSeedScope is empty, keypairs are random and those derived +// values vary per run — only the placeholder-protected fields above +// stay stable in either mode. +// +// We test that the field is *present and well-shaped*, not that the +// bytes match across runs — backend equivalence at the byte level is +// reasoned about transitively from the envelope (Ed25519 signatures and +// SAC contract IDs are deterministic functions of their inputs, so +// matching envelopes ⇒ matching signatures and SAC IDs). +func normalizeDynamicFields(v any) { + switch x := v.(type) { + case map[string]any: + if isDecoratedSignature(x) { + x["Hint"] = "$hint" + x["Signature"] = "$signature" + return + } + for k, val := range x { + if placeholder, ok := dynamicFieldPlaceholder(k, val); ok { + x[k] = placeholder + continue + } + normalizeDynamicFields(val) + } + case []any: + for _, val := range x { + normalizeDynamicFields(val) + } + } +} + +// numericByte extracts a byte value from a JSON-decoded numeric. Real +// decoded payloads use json.Number (roundtrip enables UseNumber so +// integer precision survives), but the helper still accepts float64 +// for hand-built test fixtures and other callers that pre-date the +// json.Number switch. +func numericByte(v any) (byte, bool) { + switch n := v.(type) { + case json.Number: + i, err := n.Int64() + if err != nil { + return 0, false + } + return byte(i), true + case float64: + return byte(n), true + } + return 0, false +} + +// isNumericValue reports whether v is one of the JSON numeric shapes +// (json.Number from UseNumber decoding, or float64 from a fixture). +func isNumericValue(v any) bool { + switch v.(type) { + case json.Number, float64: + return true + } + return false +} + +// dynamicFieldPlaceholder maps a (key, value) pair to its placeholder +// string if it's one of the known per-run-varying fields. Returns +// ("", false) otherwise. +func dynamicFieldPlaceholder(key string, val any) (string, bool) { + switch key { + case "SeqNum": + if isNumericValue(val) { + return "$seqNum", true + } + case "BumpTo": + if isNumericValue(val) { + return "$bumpTo", true + } + case "OfferId": + // OfferId is a per-network monotonic counter assigned when a + // ManageSellOffer/ManageBuyOffer creates a new offer. Stable + // within a run, varies across runs. + if isNumericValue(val) { + return "$offerId", true + } + case "TransactionHash": + if isThirtyTwoByteArray(val) { + return "$innerTransactionHash", true + } + case "ContractId": + // SAC contract IDs are deterministic functions of (passphrase, + // asset, issuer pubkey). With deterministic test keypairs (set + // via runner.Config.AccountSeedScope) the issuer is stable + // across runs, so the contract ID is too. Render it as the + // canonical "C…" strkey instead of a placeholder so the snapshot + // shows real, debuggable values. + if encoded, ok := tryEncodeStrkey(val, strkey.VersionByteContract); ok { + return encoded, true + } + } + return "", false +} + +// isThirtyTwoByteArray returns true if v is a JSON-decoded `[]any` of +// 32 number-typed elements (the shape of a [32]byte field after a +// JSON round-trip). +func isThirtyTwoByteArray(v any) bool { + arr, ok := v.([]any) + if !ok || len(arr) != 32 { + return false + } + for _, b := range arr { + if !isNumericValue(b) { + return false + } + } + return true +} + +// tryEncodeStrkey treats v as a 32-byte JSON-decoded array and encodes +// it with the given strkey version (e.g. VersionByteContract for "C…", +// VersionByteAccountID for "G…"). Returns ("", false) if the shape +// doesn't match. +func tryEncodeStrkey(v any, version strkey.VersionByte) (string, bool) { + if !isThirtyTwoByteArray(v) { + return "", false + } + arr := v.([]any) + bytes := make([]byte, 32) + for i, b := range arr { + bb, ok := numericByte(b) + if !ok { + return "", false + } + bytes[i] = bb + } + encoded, err := strkey.Encode(version, bytes) + if err != nil { + return "", false + } + return encoded, true +} + +// isDecoratedSignature matches the JSON shape Go's encoder produces for +// xdr.DecoratedSignature: a 2-key map with a 4-byte Hint array and a +// base64 Signature string. +func isDecoratedSignature(m map[string]any) bool { + if len(m) != 2 { + return false + } + hint, hasHint := m["Hint"].([]any) + if !hasHint || len(hint) != 4 { + return false + } + sig, hasSig := m["Signature"].(string) + if !hasSig || sig == "" { + return false + } + return true +} + +// normalizeAccountIDs walks `v` recursively and replaces any nested +// `{Ed25519: [32 ints]}` shape with the canonical strkey "G…" string. +// +// Stellar XDR represents AccountID, MuxedAccount, and PublicKey as a +// discriminated union whose only currently-used variant carries the raw +// ed25519 public key. Marshaled to JSON without a Type discriminator +// (the SDK omits zero-valued types), they all share the +// `{Ed25519: byte-array}` shape, so a single normalizer covers them all. +// +// Unsupported variants (Med25519 muxed accounts) are left untouched — +// they have additional fields and don't match the single-key pattern. +func normalizeAccountIDs(v any) { + switch x := v.(type) { + case map[string]any: + for k, val := range x { + if encoded, ok := tryEncodeAccountID(val); ok { + x[k] = encoded + continue + } + normalizeAccountIDs(val) + } + case []any: + for i, val := range x { + if encoded, ok := tryEncodeAccountID(val); ok { + x[i] = encoded + continue + } + normalizeAccountIDs(val) + } + } +} + +// tryEncodeAccountID returns the strkey form of an object whose only +// payload is a 32-byte Ed25519 public key, or ("", false) if the value +// isn't that shape. The pattern matches: +// +// {Ed25519: [32 bytes]} // AccountId, PublicKey +// {Type: 0, Ed25519: [32 bytes]} // MuxedAccount (KeyTypeEd25519) +// {Type: 0, Ed25519: [32 bytes], Med25519: null} // ditto, with explicit nulls +// +// The 32-byte length check guards against rewriting unrelated fields +// (signature blobs are 64 bytes; contract IDs are 32 bytes but live +// inside `ContractId.Hash`, not under `Ed25519`). +// +// If a `Type` field is present it must be numeric and zero (the only +// stellar discriminator under which Ed25519 means "an account address"). +// All other map keys must be either absent or hold the JSON value `nil` +// — that's how Go's json.Marshal renders nil pointers in the +// alternative-variant fields of a discriminated union. +func tryEncodeAccountID(v any) (string, bool) { + m, ok := v.(map[string]any) + if !ok { + return "", false + } + raw, ok := m["Ed25519"].([]any) + if !ok || len(raw) != 32 { + return "", false + } + for k, val := range m { + switch k { + case "Ed25519": + continue + case "Type": + b, ok := numericByte(val) + if !ok || b != 0 { + return "", false + } + default: + // Sibling fields (Med25519, MuxedAccountId, etc.) must be + // explicit nulls — non-nil siblings mean we're looking at a + // different variant of the union. + if val != nil { + return "", false + } + } + } + bytes := make([]byte, 32) + for i, b := range raw { + bb, ok := numericByte(b) + if !ok { + return "", false + } + bytes[i] = bb + } + encoded, err := strkey.Encode(strkey.VersionByteAccountID, bytes) + if err != nil { + return "", false + } + return encoded, true +} diff --git a/test/lib/xdr/decode_test.go b/test/lib/xdr/decode_test.go new file mode 100644 index 0000000..f4dcb62 --- /dev/null +++ b/test/lib/xdr/decode_test.go @@ -0,0 +1,183 @@ +package xdr + +import ( + "encoding/json" + "strconv" + "testing" +) + +func TestNormalizeAccountIDs(t *testing.T) { + // Synthesize a tree containing every shape we expect in real + // xdr-encoded JSON and verify each gets collapsed to a strkey string + // (or correctly skipped). Numbers use json.Number to match what + // roundtrip() produces at runtime (UseNumber-enabled decoder). + src := map[string]any{ + "envelope": map[string]any{ + // Shape A: MuxedAccount with explicit Type discriminator. This + // is what xdr.MuxedAccount marshals as via Go's json package. + "SourceAccount": map[string]any{ + "Type": json.Number("0"), + "Ed25519": jsonNumberSlice(make32Bytes(0x42)), + }, + "Other": "ignored", + }, + "events": []any{ + map[string]any{ + "Address": map[string]any{ + // Shape B: AccountId — just Ed25519. + "AccountId": map[string]any{ + "Ed25519": jsonNumberSlice(make32Bytes(0x37)), + }, + }, + }, + }, + // Shape C: a non-account-id variant of the same union. Type != 0 + // means a different discriminator (e.g. SignerKey HashX), so we + // must NOT collapse it. + "signerKey": map[string]any{ + "Type": json.Number("2"), + "Ed25519": jsonNumberSlice(make32Bytes(0x99)), + }, + // Shape D: 64-byte ed25519 (signature, not pubkey) — must remain. + "signature": map[string]any{ + "Ed25519": jsonNumberSlice(make([]byte, 64)), + }, + // Shape E: explicit-null sibling fields (Go json renders nil + // pointers as null), with Type=0 — should still collapse. + "muxedAccountWithNullSiblings": map[string]any{ + "Type": json.Number("0"), + "Ed25519": jsonNumberSlice(make32Bytes(0x55)), + "Med25519": nil, + }, + } + + normalizeAccountIDs(src) + + // SourceAccount should now be a strkey string starting with "G". + envelope := src["envelope"].(map[string]any) + got, ok := envelope["SourceAccount"].(string) + if !ok || len(got) != 56 || got[0] != 'G' { + t.Fatalf("envelope.SourceAccount: expected strkey 'G…' (56 chars), got %T = %v", envelope["SourceAccount"], envelope["SourceAccount"]) + } + + // AccountId nested inside Address should likewise be collapsed. + ev := src["events"].([]any)[0].(map[string]any) + addr := ev["Address"].(map[string]any) + if _, ok := addr["AccountId"].(string); !ok { + t.Fatalf("Address.AccountId: expected strkey string, got %T", addr["AccountId"]) + } + + // 64-byte signature must remain a byte array (not the AccountID shape). + sig := src["signature"].(map[string]any) + if _, ok := sig["Ed25519"].([]any); !ok { + t.Fatalf("signature.Ed25519: should still be byte array, got %T", sig["Ed25519"]) + } + + // Type != 0 (different union variant) must NOT be collapsed. + sk := src["signerKey"].(map[string]any) + if _, ok := sk["Ed25519"].([]any); !ok { + t.Fatalf("signerKey: Type=2 means non-account variant, must not collapse; got %T", sk["Ed25519"]) + } + + // Explicit-null siblings (e.g. Med25519: null) should still allow + // collapse because the variant is determined by Type=0. + if _, ok := src["muxedAccountWithNullSiblings"].(string); !ok { + t.Fatalf("muxedAccountWithNullSiblings: expected strkey, got %T", src["muxedAccountWithNullSiblings"]) + } +} + +// make32Bytes fills a 32-byte slice with the given byte value, easy to +// recognise in test failures. +func make32Bytes(b byte) []byte { + out := make([]byte, 32) + for i := range out { + out[i] = b + } + return out +} + +// jsonNumberSlice produces `[]any{json.Number, json.Number, …}` — the +// exact shape the runtime walker sees after roundtrip()'s UseNumber +// decode of an xdr Uint256 (a fixed [32]byte array, marshaled by Go as +// a JSON number array, not as base64 like a []byte slice would be). +func jsonNumberSlice(b []byte) []any { + out := make([]any, len(b)) + for i, v := range b { + out[i] = json.Number(strconv.Itoa(int(v))) + } + return out +} + +// TestNormalizeDynamicFields covers the per-run-varying fields that +// normalizeDynamicFields rewrites to placeholder strings. +func TestNormalizeDynamicFields(t *testing.T) { + src := map[string]any{ + "V1": map[string]any{ + "Tx": map[string]any{ + // SeqNum gets replaced. json.Number mirrors the runtime + // UseNumber decode and exercises numbers above 2^53 that + // would lose precision as float64. + "SeqNum": json.Number("9007199254740993"), + "Operations": []any{ + map[string]any{ + "Body": map[string]any{ + // BumpTo gets replaced. + "BumpSequenceOp": map[string]any{ + "BumpTo": json.Number("999"), + }, + }, + }, + }, + // Signature {Hint, Signature} also gets replaced (existing). + "Signatures": []any{ + map[string]any{ + "Hint": jsonNumberSlice([]byte{1, 2, 3, 4}), + "Signature": "AAAA", + }, + }, + }, + }, + "Result": map[string]any{ + "InnerResultPair": map[string]any{ + // TransactionHash 32-byte array gets replaced. + "TransactionHash": jsonNumberSlice(make32Bytes(0xAB)), + }, + }, + "events": []any{ + map[string]any{ + // ContractId 32-byte array gets replaced. + "ContractId": jsonNumberSlice(make32Bytes(0xCD)), + }, + }, + // Should NOT be touched: a 32-byte array under an unrelated key. + "randomHash": jsonNumberSlice(make32Bytes(0xEF)), + } + + normalizeDynamicFields(src) + + tx := src["V1"].(map[string]any)["Tx"].(map[string]any) + if got := tx["SeqNum"]; got != "$seqNum" { + t.Errorf("SeqNum: want $seqNum, got %v", got) + } + op := tx["Operations"].([]any)[0].(map[string]any)["Body"].(map[string]any)["BumpSequenceOp"].(map[string]any) + if got := op["BumpTo"]; got != "$bumpTo" { + t.Errorf("BumpTo: want $bumpTo, got %v", got) + } + sig := tx["Signatures"].([]any)[0].(map[string]any) + if got := sig["Signature"]; got != "$signature" { + t.Errorf("Signature: want $signature, got %v", got) + } + pair := src["Result"].(map[string]any)["InnerResultPair"].(map[string]any) + if got := pair["TransactionHash"]; got != "$innerTransactionHash" { + t.Errorf("TransactionHash: want $innerTransactionHash, got %v", got) + } + ev := src["events"].([]any)[0].(map[string]any) + contractStrkey, ok := ev["ContractId"].(string) + if !ok || len(contractStrkey) != 56 || contractStrkey[0] != 'C' { + t.Errorf("ContractId: expected 'C…' strkey (56 chars), got %T = %v", ev["ContractId"], ev["ContractId"]) + } + // randomHash must remain a byte array — not a known dynamic field key. + if _, ok := src["randomHash"].([]any); !ok { + t.Errorf("randomHash: should remain a byte array, got %T", src["randomHash"]) + } +} diff --git a/test/scenarios/edge_test.go b/test/scenarios/edge_test.go new file mode 100644 index 0000000..2accfbe --- /dev/null +++ b/test/scenarios/edge_test.go @@ -0,0 +1,121 @@ +// Edge-case scenarios that exercise transaction shapes most likely to +// surface differences between the poller (RPC-based) and captive-core +// (stellar-core-direct) firehose backends: +// +// - V1 envelope with multi-sig (multiple decorated signatures) +// - Fee-bump envelope (FeeBumpTransactionEnvelope wraps an inner tx) +// - SetOptions adding a signer (changes account thresholds) +// - ManageSellOffer (DEX op, exercises offer events) +// - BumpSequence (boring op, useful as a sanity benchmark) +// +// Soroban-specific scenarios live in soroban_test.go (TODO). +package scenarios + +import ( + "testing" + + "github.com/stellar/go-stellar-sdk/txnbuild" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/streamingfast/firehose-stellar/test/lib/stellar" +) + +func TestFeeBumpPayment(t *testing.T) { + r := newRunner(t) + payer := r.MustNewFundedAccount("payer") + innerSrc := r.MustNewFundedAccount("inner_source") + dest := r.MustNewFundedAccount("dest") + + resp, err := r.Stellar.SubmitFeeBump(payer, innerSrc, []txnbuild.Operation{ + stellar.Payment(dest.Address(), "3", txnbuild.NativeAsset{}), + }) + if err != nil { + t.Fatal(err) + } + if err := r.AssertScenario("fee_bump/payment", uint64(resp.Ledger), resp.Hash); err != nil { + t.Fatal(err) + } +} + +func TestMultiSigPayment(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + cosigner := r.MustNewFundedAccount("cosigner") + dest := r.MustNewFundedAccount("dest") + + // Add cosigner as a 1-weight signer; lower master to 1 and bump + // thresholds to 2 so that BOTH master and cosigner are required. + masterWeight := txnbuild.Threshold(1) + med := txnbuild.Threshold(2) + high := txnbuild.Threshold(2) + if err := r.RunScenario("multisig/setup", []txnbuild.Operation{ + &txnbuild.SetOptions{ + MasterWeight: &masterWeight, + MediumThreshold: &med, + HighThreshold: &high, + Signer: &txnbuild.Signer{ + Address: cosigner.Address(), + Weight: txnbuild.Threshold(1), + }, + }, + }, src); err != nil { + t.Fatal(err) + } + + // Now require both signatures to push a payment through. + if err := r.RunScenario("multisig/payment", []txnbuild.Operation{ + stellar.Payment(dest.Address(), "1", txnbuild.NativeAsset{}), + }, src, cosigner); err != nil { + t.Fatal(err) + } +} + +func TestManageSellOffer(t *testing.T) { + r := newRunner(t) + issuer := r.MustNewFundedAccount("issuer") + seller := r.MustNewFundedAccount("seller") + asset := stellar.CreditAsset("OFFR", issuer) + + if err := r.RunScenario("offer/setup_trustline", []txnbuild.Operation{ + stellar.ChangeTrust(asset, "1000"), + }, seller); err != nil { + t.Fatal(err) + } + if err := r.RunScenario("offer/issue_to_seller", []txnbuild.Operation{ + stellar.Payment(seller.Address(), "100", asset), + }, issuer); err != nil { + t.Fatal(err) + } + + // Place a passive offer: 100 OFFR for native, 1 OFFR = 0.5 XLM. + if err := r.RunScenario("offer/manage_sell", []txnbuild.Operation{ + &txnbuild.ManageSellOffer{ + Selling: asset, + Buying: txnbuild.NativeAsset{}, + Amount: "10", + Price: xdr.Price{N: 1, D: 2}, + }, + }, seller); err != nil { + t.Fatal(err) + } +} + +func TestBumpSequence(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + + current, err := r.Stellar.LoadAccount(src.Address()) + if err != nil { + t.Fatal(err) + } + seq, err := current.GetSequenceNumber() + if err != nil { + t.Fatal(err) + } + + if err := r.RunScenario("seq/bump", []txnbuild.Operation{ + &txnbuild.BumpSequence{BumpTo: seq + 100}, + }, src); err != nil { + t.Fatal(err) + } +} diff --git a/test/scenarios/main_test.go b/test/scenarios/main_test.go new file mode 100644 index 0000000..3698130 --- /dev/null +++ b/test/scenarios/main_test.go @@ -0,0 +1,287 @@ +// main_test.go drives the dev-stack lifecycle from `go test`. +// +// Both fetchers (rpc poller + captive-core) run in-process via the +// rpc and captivecore library packages. The only container managed +// here is stellar/quickstart — the chain itself. +// +// stellar-core (the C++ binary) is required on PATH for the captive- +// core fetcher. macOS: `brew install stellar/sdf/stellar-core`. Linux: +// SDF apt repo. +// +// Env vars: +// +// BATTLEFIELD_MANAGE_STACK=0 skip docker-compose lifecycle entirely +// (assume quickstart already running) +// AUTO_RESET=0 reuse the current chain instead of +// restarting quickstart for a clean slate +// KEEP_RUNNING=1 leave quickstart up after tests finish +// SKIP_TESTS=1 bring quickstart up, exit without +// running tests +// STELLAR_CORE_BIN= override stellar-core binary location +// (default: exec.LookPath("stellar-core")) +// DEBUG=1 stream docker compose output to stderr +package scenarios + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "github.com/streamingfast/firehose-stellar/captivecore" + "github.com/streamingfast/firehose-stellar/test/lib/devstack" + "github.com/streamingfast/firehose-stellar/test/lib/firehose" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// sharedFetchers is populated by TestMain and consumed by newRunner. Both +// in-process fetchers are constructed once per `go test` invocation and +// shared across every scenario — important for captive-core, whose +// stellar-core subprocess startup costs 5-30s. +var sharedFetchers []firehose.Fetcher + +// standaloneNetworkPassphrase is the passphrase stellar/quickstart uses +// in --local mode. Pinned here so the in-process fetchers can recompute +// transaction hashes — must match the chain or all hash assertions break. +const standaloneNetworkPassphrase = "Standalone Network ; February 2017" + +// rpcEndpoint is where the in-process rpc fetcher reaches soroban-rpc. +// Goes through the port the quickstart container exposes (8000 by +// default, see devstack.Config.HorizonPort). +const rpcEndpoint = "http://localhost:8000/soroban/rpc" + +// historyArchiveURL is the SDF history server quickstart runs. Captive- +// core needs at least one archive URL to catch up — for a local +// standalone network, this is the bundled archive on the quickstart +// container. +const historyArchiveURL = "http://localhost:1570" + +func TestMain(m *testing.M) { + os.Exit(runTests(m)) +} + +func runTests(m *testing.M) int { + ctx := context.Background() + + // Stack lifecycle. With MANAGE_STACK=0 the caller is responsible + // for keeping quickstart running. + if envBool("BATTLEFIELD_MANAGE_STACK", true) { + cfg := devstack.DefaultConfig() + cfg.Debug = envBool("DEBUG", false) + + stack, err := devstack.New(cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "devstack init: %v\n", err) + return 2 + } + + exitCode := 2 + defer func() { + if envBool("KEEP_RUNNING", false) { + fmt.Fprintln(os.Stderr, ">> KEEP_RUNNING=1: leaving quickstart up") + return + } + if err := stack.Down(ctx); err != nil { + fmt.Fprintf(os.Stderr, "quickstart down: %v\n", err) + } + }() + + if envBool("AUTO_RESET", true) { + fmt.Fprintln(os.Stderr, ">> resetting quickstart (clean chain)") + if err := stack.Reset(ctx); err != nil { + fmt.Fprintf(os.Stderr, "quickstart reset: %v\n", err) + return 2 + } + } else { + running, _ := stack.IsRunning(ctx) + if running { + fmt.Fprintln(os.Stderr, ">> reusing running quickstart") + } else { + fmt.Fprintln(os.Stderr, ">> bringing up quickstart") + if err := stack.Up(ctx); err != nil { + fmt.Fprintf(os.Stderr, "quickstart up: %v\n", err) + return 2 + } + } + } + + if envBool("SKIP_TESTS", false) { + fmt.Fprintln(os.Stderr, ">> SKIP_TESTS=1, exiting after quickstart boot") + return 0 + } + + exitCode = runWithFetchers(m) + return exitCode + } + + // MANAGE_STACK=0 path: trust quickstart is already up. + return runWithFetchers(m) +} + +// runWithFetchers constructs the two in-process fetchers (rpc + captive- +// core), publishes them to scenarios via the sharedFetchers package var, +// runs the suite, and tears the fetchers down on exit. Quickstart must +// already be healthy at this point. +func runWithFetchers(m *testing.M) int { + // Redirect chatty fetcher logs (zap from rpc.Fetcher, logrus from + // stellar-core subprocess) to a file under .data/. Tail it from + // another terminal if you need to debug: + // tail -f test/.data/fetchers.log + logFile, err := openFetcherLogFile() + if err != nil { + fmt.Fprintf(os.Stderr, "open fetcher log file: %v\n", err) + return 2 + } + // Don't defer Close — stellar-core subprocess flushes log lines + // asynchronously after our Close() returns, racing with this. OS + // reclaims the fd on process exit anyway. + fmt.Fprintf(os.Stderr, ">> fetcher logs → %s\n", logFile.Name()) + + logger := newFileLogger(logFile) + + rpcF, err := firehose.NewInProcessRPCFetcher(firehose.InProcessRPCConfig{ + Name: "poller", + RPCEndpoint: rpcEndpoint, + NetworkPassphrase: standaloneNetworkPassphrase, + Logger: logger, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "build in-process rpc fetcher: %v\n", err) + return 2 + } + + ccBin, err := resolveStellarCoreBinary() + if err != nil { + // stellar-core unavailable on host → poller only. Tests will + // skip the cross-fetcher diff (only 1 fetcher configured) but + // snapshot assertion still runs. + fmt.Fprintf(os.Stderr, ">> stellar-core not found (%v); running poller only\n", err) + sharedFetchers = []firehose.Fetcher{rpcF} + defer rpcF.Close() + return m.Run() + } + + ccF, err := firehose.NewInProcessCaptiveCoreFetcher("captive-core", captivecore.Config{ + BinaryPath: ccBin, + NetworkPassphrase: standaloneNetworkPassphrase, + HistoryArchiveURLs: []string{historyArchiveURL}, + StellarCoreConfPath: lookupFollowerToml(), + // Keep stellar-core's working files (sqlite db, buckets, tmp) + // under test/.data/captive-core/ alongside the chain state. + StoragePath: captiveCoreStoragePath(), + LogOutput: logFile, + Logger: logger, + }) + if err != nil { + // Non-fatal — fall back to poller only. + fmt.Fprintf(os.Stderr, ">> captive-core fetcher init failed (%v); running poller only\n", err) + sharedFetchers = []firehose.Fetcher{rpcF} + defer rpcF.Close() + return m.Run() + } + + sharedFetchers = []firehose.Fetcher{rpcF, ccF} + defer func() { + _ = rpcF.Close() + _ = ccF.Close() + }() + return m.Run() +} + +// openFetcherLogFile creates (or truncates) the file where stellar-core +// and rpc.Fetcher zap logs land. Lives under /fetchers.log +// so it sits next to compose.log and captive-core/ working files. +func openFetcherLogFile() (*os.File, error) { + dataRoot := os.Getenv("BATTLEFIELD_DATA_DIR") + if dataRoot == "" { + dataRoot = "../.data" + } + abs, err := filepath.Abs(dataRoot) + if err != nil { + abs = dataRoot + } + if err := os.MkdirAll(abs, 0o755); err != nil { + return nil, err + } + return os.Create(filepath.Join(abs, "fetchers.log")) +} + +// newFileLogger builds a zap logger that writes to the given file in +// dev-friendly text format (timestamps, level colors stripped). +func newFileLogger(w *os.File) *zap.Logger { + encCfg := zap.NewDevelopmentEncoderConfig() + encCfg.EncodeLevel = zapcore.CapitalLevelEncoder + core := zapcore.NewCore( + zapcore.NewConsoleEncoder(encCfg), + zapcore.AddSync(w), + zapcore.DebugLevel, + ) + return zap.New(core) +} + +// resolveStellarCoreBinary picks the stellar-core binary location. Env +// override wins; otherwise consult $PATH. +func resolveStellarCoreBinary() (string, error) { + if v := os.Getenv("STELLAR_CORE_BIN"); v != "" { + return v, nil + } + return exec.LookPath("stellar-core") +} + +// captiveCoreStoragePath returns the data root that ledgerbackend will +// use as the parent for its `captive-core/` working directory. The SDK +// always appends `/captive-core` to whatever path is passed, so we hand +// over .data directly and end up with .data/captive-core/ — not +// .data/captive-core/captive-core/. +func captiveCoreStoragePath() string { + dataRoot := os.Getenv("BATTLEFIELD_DATA_DIR") + if dataRoot == "" { + // go test sets cwd to package dir (test/scenarios). Resolve + // ../.data which sits at test/.data, matching devstack. + dataRoot = "../.data" + } + abs, err := filepath.Abs(dataRoot) + if err != nil { + return dataRoot + } + _ = os.MkdirAll(abs, 0o755) + return abs +} + +// lookupFollowerToml returns the path to a captive-core toml that peers +// with quickstart from the host (localhost + docker-compose-exposed +// ports). go test sets cwd to the package dir, so the canonical +// relative path is ../scripts/dev/configs/... +func lookupFollowerToml() string { + for _, candidate := range []string{ + "../scripts/dev/configs/captive-core-follower-host.cfg", + "./scripts/dev/configs/captive-core-follower-host.cfg", + "test/scripts/dev/configs/captive-core-follower-host.cfg", + } { + if _, err := os.Stat(candidate); err == nil { + if abs, err := filepath.Abs(candidate); err == nil { + return abs + } + return candidate + } + } + return "" +} + +// envBool parses a 0/1 (or true/false) env var. Missing or unparseable +// → fallback. +func envBool(name string, fallback bool) bool { + v := os.Getenv(name) + if v == "" { + return fallback + } + b, err := strconv.ParseBool(v) + if err != nil { + return fallback + } + return b +} diff --git a/test/scenarios/scenarios_test.go b/test/scenarios/scenarios_test.go new file mode 100644 index 0000000..b8cf18c --- /dev/null +++ b/test/scenarios/scenarios_test.go @@ -0,0 +1,181 @@ +// Package scenarios is the battlefield-stellar test suite. Each test submits +// one or more transactions to a Stellar network, fetches the resulting block +// via firehose, and asserts the structural transaction view against a +// recorded snapshot. +// +// Run from the repo root with: +// +// go test ./test/scenarios/... -v +// +// Or to regenerate all snapshots: +// +// SNAPSHOTS_UPDATE=. go test ./test/scenarios/... -v +package scenarios + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stellar/go-stellar-sdk/txnbuild" + + "github.com/streamingfast/firehose-stellar/test/lib/runner" + "github.com/streamingfast/firehose-stellar/test/lib/stellar" +) + +// snapshotRoot points the runner at the repo-root snapshots/ directory rather +// than scenarios/snapshots, matching the layout in the design doc. +func snapshotRoot(t *testing.T) string { + t.Helper() + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + return filepath.Join(filepath.Dir(cwd), "snapshots") +} + +func newRunner(t *testing.T) *runner.Runner { + t.Helper() + cfg := runner.DefaultConfig() + cfg.SnapshotRoot = snapshotRoot(t) + // Seed keypair derivation off the test name so accounts are + // byte-identical across two runs of the same test (record from + // poller → validate from captive-core). Without this, every run + // gets fresh random keypairs and SAC contract IDs vary, defeating + // snapshot byte-stability. + cfg.AccountSeedScope = t.Name() + + // Use the in-process fetchers TestMain built. Shared across the + // suite; TestMain closes them after m.Run(). Per-test Close would + // tear down captive-core's stellar-core subprocess. + cfg.Fetchers = sharedFetchers + + r, err := runner.New(cfg) + if err != nil { + t.Fatalf("runner setup: %v", err) + } + // Do NOT call r.Close in cleanup — fetchers are shared across tests + // and owned by TestMain. r.Close would propagate to the shared + // captive-core fetcher and kill its stellar-core subprocess. + return r +} + +func TestNativePayment(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + dst := r.MustNewFundedAccount("dest") + + if err := r.RunScenario("payment/native", []txnbuild.Operation{ + stellar.Payment(dst.Address(), "10", txnbuild.NativeAsset{}), + }, src); err != nil { + t.Fatal(err) + } +} + +func TestDoubleNativePayment(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + dst := r.MustNewFundedAccount("dest") + + if err := r.RunScenario("payment/double", []txnbuild.Operation{ + stellar.Payment(dst.Address(), "5", txnbuild.NativeAsset{}), + stellar.Payment(dst.Address(), "7", txnbuild.NativeAsset{}), + }, src); err != nil { + t.Fatal(err) + } +} + +func TestCreateAccount(t *testing.T) { + r := newRunner(t) + funder := r.MustNewFundedAccount("funder") + // `target` must NOT be friendbot-funded — the CreateAccount operation + // is what brings it into existence. Friendbot-funding first would + // make the op fail with op_already_exists. + target := r.MustNewUnfundedAccount("target") + + if err := r.RunScenario("account/create", []txnbuild.Operation{ + stellar.CreateAccount(target.Address(), "5"), + }, funder); err != nil { + t.Fatal(err) + } +} + +func TestManageData(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + + if err := r.RunScenario("data/manage", []txnbuild.Operation{ + stellar.ManageData("battlefield", "hello"), + }, src); err != nil { + t.Fatal(err) + } +} + +func TestIssueAndSendAsset(t *testing.T) { + r := newRunner(t) + issuer := r.MustNewFundedAccount("issuer") + distributor := r.MustNewFundedAccount("distributor") + asset := stellar.CreditAsset("USDB", issuer) + + // Distributor accepts the trustline first. + if err := r.RunScenario("asset/trustline", []txnbuild.Operation{ + stellar.ChangeTrust(asset, "5000"), + }, distributor); err != nil { + t.Fatal(err) + } + + // Then the issuer mints into the distributor account. + if err := r.RunScenario("asset/issue", []txnbuild.Operation{ + stellar.Payment(distributor.Address(), "100", asset), + }, issuer); err != nil { + t.Fatal(err) + } +} + +func TestMultiOp(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + a := r.MustNewFundedAccount("recipientA") + b := r.MustNewFundedAccount("recipientB") + + if err := r.RunScenario("multi_op/payment_pair", []txnbuild.Operation{ + stellar.Payment(a.Address(), "1", txnbuild.NativeAsset{}), + stellar.Payment(b.Address(), "2", txnbuild.NativeAsset{}), + stellar.ManageData("note", "multi-op-test"), + }, src); err != nil { + t.Fatal(err) + } +} + +// TestFailedTransaction issues a payment from an account that does not exist +// on chain — the submission should be rejected. We assert the rejection +// rather than snapshotting a (non-existent) firehose block. +func TestFailedTransaction(t *testing.T) { + r := newRunner(t) + src := r.MustNewFundedAccount("source") + // Generate a destination keypair but skip funding so the payment fails + // at create-account-required (the destination has no trustline / no XLM + // reserve). For native payments this still succeeds and creates the + // account, so we instead try a custom asset payment with no trustline. + stranger := r.MustNewFundedAccount("stranger") + asset := stellar.CreditAsset("NOPE", src) + + err := r.Stellar.SubmitOpsExpectFail(src, []txnbuild.Operation{ + stellar.Payment(stranger.Address(), "1", asset), + }) + if err != nil { + t.Fatal(err) + } +} + +func TestAccountMerge(t *testing.T) { + r := newRunner(t) + dying := r.MustNewFundedAccount("dying") + heir := r.MustNewFundedAccount("heir") + + if err := r.RunScenario("account/merge", []txnbuild.Operation{ + stellar.AccountMerge(heir.Address()), + }, dying); err != nil { + t.Fatal(err) + } +} diff --git a/test/scenarios/soroban_test.go b/test/scenarios/soroban_test.go new file mode 100644 index 0000000..6e1daf7 --- /dev/null +++ b/test/scenarios/soroban_test.go @@ -0,0 +1,30 @@ +// Soroban scenarios — placeholders. These exercise transaction shapes that +// are the most-likely sources of poller↔captive-core divergence (diagnostic +// events, contract events, failed-but-included transactions). They are +// gated behind BATTLEFIELD_SOROBAN=1 because they need: +// +// - a deployed test contract (WASM under contracts/, deployed once and +// pinned via env or fixture) +// - a quickstart docker with --enable-soroban-rpc +// +// TODO(phase-2): port the substreams-stellar-soroban example contracts and +// add invoke/event/failure scenarios here. +package scenarios + +import ( + "os" + "testing" +) + +func skipUnlessSoroban(t *testing.T) { + t.Helper() + if os.Getenv("BATTLEFIELD_SOROBAN") != "1" { + t.Skip("set BATTLEFIELD_SOROBAN=1 to run soroban scenarios") + } +} + +func TestSorobanInvoke(t *testing.T) { skipUnlessSoroban(t); t.Skip("TODO: contract deploy + invoke") } +func TestSorobanFailedInvoke(t *testing.T) { skipUnlessSoroban(t); t.Skip("TODO: contract panics, tx INCLUDED with status=FAILED") } +func TestSorobanContractEvents(t *testing.T) { skipUnlessSoroban(t); t.Skip("TODO: emit multiple events from a single op") } +func TestSorobanCrossContract(t *testing.T) { skipUnlessSoroban(t); t.Skip("TODO: contract A calls contract B") } +func TestSorobanDiagnosticEvents(t *testing.T) { skipUnlessSoroban(t); t.Skip("TODO: poller and captive-core differ on diag event presence") } diff --git a/test/scripts/dev/configs/captive-core-follower-host.cfg b/test/scripts/dev/configs/captive-core-follower-host.cfg new file mode 100644 index 0000000..c09b205 --- /dev/null +++ b/test/scripts/dev/configs/captive-core-follower-host.cfg @@ -0,0 +1,42 @@ +# Captive-core configuration for the IN-PROCESS battlefield-stellar +# fetcher running on the host (not inside docker). +# +# Used by test/scenarios/main_test.go's InProcessCaptiveCoreFetcher when +# stellar-core is invoked directly from `go test`. The only difference +# from captive-core-follower.cfg is the peer + history addresses: from +# the host we can't resolve `quickstart` (the docker service hostname), +# so we use localhost + the ports docker-compose exposes. +# +# The captivecore.Config.HistoryArchiveURLs supplied by the Go caller +# overrides the HISTORY URL templated below — they must match, so keep +# both pointed at the host-mapped port (default 1570). + +NETWORK_PASSPHRASE="Standalone Network ; February 2017" + +# Peer + history reachable on the host via the ports docker-compose +# exposes (see test/scripts/dev/docker-compose.yml `ports:` section). +KNOWN_PEERS=["localhost:11625"] + +# Listen on a different peer port than the quickstart container — host +# port 11625 is already taken by the docker-compose port mapping, so +# binding 0.0.0.0:11625 from this stellar-core process would fail with +# EADDRINUSE. +PEER_PORT=11626 + +NODE_IS_VALIDATOR=false + +FAILURE_SAFETY=0 +UNSAFE_QUORUM=true + +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true + +[[HOME_DOMAINS]] +HOME_DOMAIN="local.stellar" +QUALITY="LOW" + +[[VALIDATORS]] +NAME="quickstart_validator" +HOME_DOMAIN="local.stellar" +PUBLIC_KEY="GCTI6HMWRH2QGMFKWVU5M5ZSOTKL7P7JAHZDMJJBKDHGWTEC4CJ7O3DU" +ADDRESS="localhost:11625" +HISTORY="curl -sf http://localhost:1570/{0} -o {1}" diff --git a/test/scripts/dev/configs/derive_pubkey.go b/test/scripts/dev/configs/derive_pubkey.go new file mode 100644 index 0000000..bc2c5dd --- /dev/null +++ b/test/scripts/dev/configs/derive_pubkey.go @@ -0,0 +1,26 @@ +//go:build ignore + +// Tiny helper to print the validator public key for the well-known +// quickstart --local NODE_SEED. Run from the repo root via: +// go run test/scripts/dev/configs/derive_pubkey.go +package main + +import ( + "fmt" + "os" + + "github.com/stellar/go-stellar-sdk/keypair" +) + +func main() { + seed := "SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2" + if len(os.Args) > 1 { + seed = os.Args[1] + } + kp, err := keypair.ParseFull(seed) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Printf("Seed: %s\nPublic: %s\n", kp.Seed(), kp.Address()) +} diff --git a/test/scripts/dev/configs/stellar-core-validator.cfg b/test/scripts/dev/configs/stellar-core-validator.cfg new file mode 100644 index 0000000..c768113 --- /dev/null +++ b/test/scripts/dev/configs/stellar-core-validator.cfg @@ -0,0 +1,69 @@ +# DOCUMENTATION ONLY — this file is NOT mounted into the running container. +# +# stellar-core configuration that documents what is running inside the +# stellar/quickstart --local validator (i.e. the quickstart-quickstart +# service in scripts/dev/docker-compose.yml). It is functionally identical +# to /opt/stellar-default/local/core/etc/stellar-core.cfg inside the +# quickstart image, with the placeholders __NETWORK__, __DATABASE__, +# __MANUAL_CLOSE__ resolved. +# +# Why ship it without mounting: +# +# We can't mount just this file at /opt/stellar/core/etc/stellar-core.cfg +# because quickstart's start script's copy_defaults() also needs to copy +# /opt/stellar-default/local/core/etc/config-settings/ (per-protocol JSON +# network upgrade settings). The skip-if-etc-exists check is dir-level, so +# mounting our cfg into a fresh etc/ would prevent config-settings/ from +# being copied and stellar-core would fail to start. +# +# Mounting the WHOLE etc/ (our cfg + a copy of bundled config-settings/) +# works but is fragile across quickstart image versions. +# +# So we rely on the bundled config — which uses the same well-known +# NODE_SEED below — and treat this file as the canonical record of what +# that config contains. +# +# Why this matters for cross-backend comparison: +# +# captive-core-follower.cfg (in this same dir) hardcodes the validator's +# public key derived from NODE_SEED below. To verify the derivation: +# +# go run test/scripts/dev/configs/derive_pubkey.go + +HTTP_PORT=11626 +PUBLIC_HTTP_PORT=true +LOG_FILE_PATH="/var/log/stellar-core/stellar-core-{datetime:%Y-%m-%d_%H-%M-%S}.log" + +# false = automatic block production every 5s; true = blocks only on demand +# via the /manualclose HTTP endpoint. Keep false for the dev loop. +MANUAL_CLOSE=false + +# Tells stellar-core to close ledgers as fast as possible regardless of +# wall-clock — turns the 5s ledger close target into ~1s in standalone mode. +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true + +NETWORK_PASSPHRASE="Standalone Network ; February 2017" + +# Well-known standalone seed. Public key: +# GCTI6HMWRH2QGMFKWVU5M5ZSOTKL7P7JAHZDMJJBKDHGWTEC4CJ7O3DU +# Same as the one quickstart bundles in /opt/stellar-default/local/core/etc. +NODE_SEED="SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2 self" +NODE_IS_VALIDATOR=true + +DATABASE="sqlite3:///opt/stellar/core/data/stellar.db" + +FAILURE_SAFETY=0 +UNSAFE_QUORUM=true + +# Single-validator quorum (the validator trusts only itself in standalone). +[QUORUM_SET] +THRESHOLD_PERCENT=100 +VALIDATORS=["$self"] + +# History archive served from a local directory; quickstart's start script +# also runs an nginx serving this dir at http://localhost:1570 so external +# captive-core peers can fetch backfill. +[HISTORY.vs] +get="cp /opt/stellar/history-archive/data/{0} {1}" +put="cp {0} /opt/stellar/history-archive/data/{1}" +mkdir="mkdir -p /opt/stellar/history-archive/data/{0}" diff --git a/test/scripts/dev/docker-compose.yml b/test/scripts/dev/docker-compose.yml new file mode 100644 index 0000000..caccfb0 --- /dev/null +++ b/test/scripts/dev/docker-compose.yml @@ -0,0 +1,60 @@ +# Dev stack for battlefield (firehose-stellar integration tests). +# +# Only one service: stellar/quickstart, which IS the chain (stellar-core +# validator + horizon + friendbot + soroban-rpc). The firestellar fetchers +# (poller and captive-core) run in-process from `go test` via the +# captivecore + rpc packages, so no firestellar container is needed. +# +# Up/down: +# docker compose -f test/scripts/dev/docker-compose.yml up -d +# docker compose -f test/scripts/dev/docker-compose.yml down +# +# Or via wrappers: +# test/scripts/dev/up.sh +# test/scripts/dev/down.sh +# +# Or — easiest — just `go test ./test/scenarios/...`; the suite's TestMain +# brings this stack up and tears it down around the run (set +# BATTLEFIELD_MANAGE_STACK=0 to opt out). + +services: + quickstart: + image: ${QUICKSTART_IMAGE:-stellar/quickstart:testing} + # Force a fresh pull each `up` so we don't keep running a locally + # cached quickstart whose bundled stellar-core predates the SDF + # May 2026 critical fix (>= 26.1.0-3210.427aa3978). Override with + # QUICKSTART_PULL_POLICY=missing to skip the pull in air-gapped envs. + pull_policy: ${QUICKSTART_PULL_POLICY:-always} + # quickstart is amd64-only; pin so we don't rely on docker's default + # platform inference (which warns on Apple Silicon). + platform: linux/amd64 + container_name: ${COMPOSE_PROJECT_NAME:-battlefield-stellar}-quickstart + command: + - --local + - --enable-soroban-rpc + # quickstart's bundled stellar-core.cfg uses a well-known fixed + # NODE_SEED, so the validator pubkey is a stable constant the + # in-process captive-core fetcher can hardcode in its follower + # config (see configs/captive-core-follower-host.cfg). + ports: + # Horizon + friendbot + soroban-rpc all multiplexed on :8000. + - "${QUICKSTART_HORIZON_PORT:-8000}:8000" + # stellar-core peer port — the in-process captive-core fetcher + # dials this to join the standalone network as a non-validating + # peer. + - "${QUICKSTART_PEER_PORT:-11625}:11625" + # History archive (stellar-core's HTTP file server) — captive-core + # needs at least one archive URL to catch up. Mapped to the host + # so the in-process fetcher can reach it. + - "${QUICKSTART_HISTORY_PORT:-1570}:1570" + healthcheck: + # `/friendbot` without `?addr=...` returns HTTP 400 — but it being + # reachable at all means horizon's reverse-proxy is up and friendbot + # is responding. We deliberately don't use curl -f so a 4xx counts + # as a successful probe (the only failure modes here are connection + # refused or no response, which exit non-zero on their own). + test: ["CMD-SHELL", "curl -sS -o /dev/null http://localhost:8000/friendbot"] + interval: 2s + timeout: 3s + retries: 90 + start_period: 5s diff --git a/test/scripts/dev/down.sh b/test/scripts/dev/down.sh new file mode 100755 index 0000000..901f7b8 --- /dev/null +++ b/test/scripts/dev/down.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Tear down the test chain. Idempotent. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml" +PROJECT="${COMPOSE_PROJECT_NAME:-battlefield-stellar}" + +if ! command -v docker >/dev/null 2>&1; then + echo ">> docker not present; nothing to do" + exit 0 +fi + +echo ">> stopping quickstart" +COMPOSE_PROJECT_NAME="$PROJECT" docker compose -f "$COMPOSE_FILE" down --remove-orphans diff --git a/test/scripts/dev/reset.sh b/test/scripts/dev/reset.sh new file mode 100755 index 0000000..65d3951 --- /dev/null +++ b/test/scripts/dev/reset.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Tear quickstart down and bring it back up for a clean chain. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +"$SCRIPT_DIR/down.sh" >/dev/null 2>&1 || true +"$SCRIPT_DIR/up.sh" diff --git a/test/scripts/dev/up.sh b/test/scripts/dev/up.sh new file mode 100755 index 0000000..8fedd3f --- /dev/null +++ b/test/scripts/dev/up.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Bring up the test chain (stellar/quickstart container). Idempotent. +# +# Useful for the manual flow: +# test/scripts/dev/up.sh +# BATTLEFIELD_MANAGE_STACK=0 go test ./test/scenarios/... -v +# test/scripts/dev/down.sh +# +# Most of the time you don't need this — `go test` auto-manages quickstart. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml" +PROJECT="${COMPOSE_PROJECT_NAME:-battlefield-stellar}" + +echo ">> starting quickstart" +COMPOSE_PROJECT_NAME="$PROJECT" docker compose -f "$COMPOSE_FILE" up -d --wait +echo ">> quickstart ready (horizon http://localhost:8000)" diff --git a/test/snapshots/account/create.expected.json b/test/snapshots/account/create.expected.json new file mode 100644 index 0000000..6dc1998 --- /dev/null +++ b/test/snapshots/account/create.expected.json @@ -0,0 +1,152 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "CreateAccountOp": { + "Destination": "GCPPTQDO52NQHEAPIOO5NGIZVZV3FFA4ED42CY4YCYDP6ZOMLNFLHTZQ", + "StartingBalance": 50000000 + }, + "Type": 0 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GD65IG7OO6TUGRY6WVFJDZAMSL5FKNKH7VSKV46A436QDWQD42PLCITV" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 50000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GD65IG7OO6TUGRY6WVFJDZAMSL5FKNKH7VSKV46A436QDWQD42PLCITV", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCPPTQDO52NQHEAPIOO5NGIZVZV3FFA4ED42CY4YCYDP6ZOMLNFLHTZQ", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GD65IG7OO6TUGRY6WVFJDZAMSL5FKNKH7VSKV46A436QDWQD42PLCITV", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "CreateAccountResult": { + "Code": 0 + }, + "Type": 0 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/account/merge.expected.json b/test/snapshots/account/merge.expected.json new file mode 100644 index 0000000..c9b6699 --- /dev/null +++ b/test/snapshots/account/merge.expected.json @@ -0,0 +1,150 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "Destination": "GCJNNRTGLQ5YW6DUXSS3JWRIW7WF2NU727OOFZPGIW3LF7SIPXDYLFTC", + "Type": 8 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GANNQUXUT3WVXIO2WTCSDSVLA3IMFUGN6NOI5KAYF22RWKGH3FXE6LGH" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 99999999900 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GANNQUXUT3WVXIO2WTCSDSVLA3IMFUGN6NOI5KAYF22RWKGH3FXE6LGH", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCJNNRTGLQ5YW6DUXSS3JWRIW7WF2NU727OOFZPGIW3LF7SIPXDYLFTC", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GANNQUXUT3WVXIO2WTCSDSVLA3IMFUGN6NOI5KAYF22RWKGH3FXE6LGH", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "AccountMergeResult": { + "Code": 0, + "SourceAccountBalance": 99999999900 + }, + "Type": 8 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/asset/issue.expected.json b/test/snapshots/asset/issue.expected.json new file mode 100644 index 0000000..45a72c9 --- /dev/null +++ b/test/snapshots/asset/issue.expected.json @@ -0,0 +1,157 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 1000000000, + "Asset": { + "AlphaNum4": { + "AssetCode": [ + 85, + 83, + 68, + 66 + ], + "Issuer": "GDVFWONJRLK7XU3BWZL3CXGE5A6SZYWUJO7P34SHZSPDFZ2I4SI55AH4" + }, + "Type": 1 + }, + "Destination": "GBOIJZ7T5MM2C46ZCEX2MPLFRQNYY64R2LYST2PQRIIE4EGMODID6Y7Y" + }, + "Type": 1 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDVFWONJRLK7XU3BWZL3CXGE5A6SZYWUJO7P34SHZSPDFZ2I4SI55AH4" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 1000000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "mint", + "Type": 15 + }, + { + "Address": { + "AccountId": "GBOIJZ7T5MM2C46ZCEX2MPLFRQNYY64R2LYST2PQRIIE4EGMODID6Y7Y", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "USDB:GDVFWONJRLK7XU3BWZL3CXGE5A6SZYWUJO7P34SHZSPDFZ2I4SI55AH4", + "Type": 14 + } + ] + } + }, + "ContractId": "CAK2UGQLFWYJ77M4A2ZJSBOAB2UIYCBAX6SQ2XOY4YADF5YR76ADGJ5D", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDVFWONJRLK7XU3BWZL3CXGE5A6SZYWUJO7P34SHZSPDFZ2I4SI55AH4", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/asset/trustline.expected.json b/test/snapshots/asset/trustline.expected.json new file mode 100644 index 0000000..2526c50 --- /dev/null +++ b/test/snapshots/asset/trustline.expected.json @@ -0,0 +1,116 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "ChangeTrustOp": { + "Limit": 50000000000, + "Line": { + "AlphaNum4": { + "AssetCode": [ + 85, + 83, + 68, + 66 + ], + "Issuer": "GDVFWONJRLK7XU3BWZL3CXGE5A6SZYWUJO7P34SHZSPDFZ2I4SI55AH4" + }, + "Type": 1 + } + }, + "Type": 6 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GBOIJZ7T5MM2C46ZCEX2MPLFRQNYY64R2LYST2PQRIIE4EGMODID6Y7Y" + } + } + }, + "events": { + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GBOIJZ7T5MM2C46ZCEX2MPLFRQNYY64R2LYST2PQRIIE4EGMODID6Y7Y", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "ChangeTrustResult": { + "Code": 0 + }, + "Type": 6 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/data/manage.expected.json b/test/snapshots/data/manage.expected.json new file mode 100644 index 0000000..7c9ced9 --- /dev/null +++ b/test/snapshots/data/manage.expected.json @@ -0,0 +1,105 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "ManageDataOp": { + "DataName": "battlefield", + "DataValue": "aGVsbG8=" + }, + "Type": 10 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GBKF72W6V7CUGOZTZ6PGE5ZYETTJSFWJEZNWTP3W6HGX3LQ7YVOU2UNO" + } + } + }, + "events": { + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GBKF72W6V7CUGOZTZ6PGE5ZYETTJSFWJEZNWTP3W6HGX3LQ7YVOU2UNO", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "ManageDataResult": { + "Code": 0 + }, + "Type": 10 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/fee_bump/payment.expected.json b/test/snapshots/fee_bump/payment.expected.json new file mode 100644 index 0000000..a0059f8 --- /dev/null +++ b/test/snapshots/fee_bump/payment.expected.json @@ -0,0 +1,185 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "FeeBump": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Ext": { + "V": 0 + }, + "Fee": 2000, + "FeeSource": "GDQ7QCZMPGHFMOWSYC5JGCJ54D2IENZAQX2OZMXIGQQOMPQWP3LPD55I", + "InnerTx": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 30000000, + "Asset": { + "Type": 0 + }, + "Destination": "GAYB6AQFII3K43DUL443N2IKEUSLTUQVBI64RGP4V5CLRLBVXILQFOUR" + }, + "Type": 1 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GAKX5BFUSTE57L5TUNFNFJBQJWC5U5R3WFKRCWVN2YV755VXTKU7RNGS" + } + } + } + } + }, + "Type": 5 + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 30000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GAKX5BFUSTE57L5TUNFNFJBQJWC5U5R3WFKRCWVN2YV755VXTKU7RNGS", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GAYB6AQFII3K43DUL443N2IKEUSLTUQVBI64RGP4V5CLRLBVXILQFOUR", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 200 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDQ7QCZMPGHFMOWSYC5JGCJ54D2IENZAQX2OZMXIGQQOMPQWP3LPD55I", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 200, + "Result": { + "Code": 1, + "InnerResultPair": { + "Result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 0, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + } + ] + } + }, + "TransactionHash": "$innerTransactionHash" + } + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/multi_op/payment_pair.expected.json b/test/snapshots/multi_op/payment_pair.expected.json new file mode 100644 index 0000000..e7a3f7e --- /dev/null +++ b/test/snapshots/multi_op/payment_pair.expected.json @@ -0,0 +1,239 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 300, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 10000000, + "Asset": { + "Type": 0 + }, + "Destination": "GBS2HTSVQGPKGHLVYLTLOURIFNZJ7SJLTEMV3N5KNBIGHSXCUCOU4BUI" + }, + "Type": 1 + } + }, + { + "Body": { + "PaymentOp": { + "Amount": 20000000, + "Asset": { + "Type": 0 + }, + "Destination": "GCZ372SZG33OCAPIZYBDIEAAYHOD6GYOMHNNBEXKEUTTTJ5A35CKSUXV" + }, + "Type": 1 + } + }, + { + "Body": { + "ManageDataOp": { + "DataName": "note", + "DataValue": "bXVsdGktb3AtdGVzdA==" + }, + "Type": 10 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GCPS7NNVVWSKZFKNO3A4NNVBIX6YRRSCDUK6YCQEJR2HJRITXU2E2AIX" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 10000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GCPS7NNVVWSKZFKNO3A4NNVBIX6YRRSCDUK6YCQEJR2HJRITXU2E2AIX", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GBS2HTSVQGPKGHLVYLTLOURIFNZJ7SJLTEMV3N5KNBIGHSXCUCOU4BUI", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ], + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 20000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GCPS7NNVVWSKZFKNO3A4NNVBIX6YRRSCDUK6YCQEJR2HJRITXU2E2AIX", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCZ372SZG33OCAPIZYBDIEAAYHOD6GYOMHNNBEXKEUTTTJ5A35CKSUXV", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 300 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GCPS7NNVVWSKZFKNO3A4NNVBIX6YRRSCDUK6YCQEJR2HJRITXU2E2AIX", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 300, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + }, + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + }, + { + "Code": 0, + "Tr": { + "ManageDataResult": { + "Code": 0 + }, + "Type": 10 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/multisig/payment.expected.json b/test/snapshots/multisig/payment.expected.json new file mode 100644 index 0000000..5171be3 --- /dev/null +++ b/test/snapshots/multisig/payment.expected.json @@ -0,0 +1,159 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + }, + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 10000000, + "Asset": { + "Type": 0 + }, + "Destination": "GCBRSVC52QKIYRCRXOPKNGX7HHY2P756VI3KPDMUULLYDMIVPEUNHTWA" + }, + "Type": 1 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDVNHAIMLUAM2TF3J3HKXFM7QC4PNF7WH6IICGIHB7E5QMLDL6G7M2HJ" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 10000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDVNHAIMLUAM2TF3J3HKXFM7QC4PNF7WH6IICGIHB7E5QMLDL6G7M2HJ", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCBRSVC52QKIYRCRXOPKNGX7HHY2P756VI3KPDMUULLYDMIVPEUNHTWA", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDVNHAIMLUAM2TF3J3HKXFM7QC4PNF7WH6IICGIHB7E5QMLDL6G7M2HJ", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/multisig/setup.expected.json b/test/snapshots/multisig/setup.expected.json new file mode 100644 index 0000000..5bd88e3 --- /dev/null +++ b/test/snapshots/multisig/setup.expected.json @@ -0,0 +1,110 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "SetOptionsOp": { + "HighThreshold": 2, + "MasterWeight": 1, + "MedThreshold": 2, + "Signer": { + "Key": "GAI4PO3S5C2U3IBNZM5IOOORAKUZNPHF26SHRFVB5THSR2XZW3IE2TJB", + "Weight": 1 + } + }, + "Type": 5 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDVNHAIMLUAM2TF3J3HKXFM7QC4PNF7WH6IICGIHB7E5QMLDL6G7M2HJ" + } + } + }, + "events": { + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDVNHAIMLUAM2TF3J3HKXFM7QC4PNF7WH6IICGIHB7E5QMLDL6G7M2HJ", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "SetOptionsResult": { + "Code": 0 + }, + "Type": 5 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/offer/issue_to_seller.expected.json b/test/snapshots/offer/issue_to_seller.expected.json new file mode 100644 index 0000000..16af35a --- /dev/null +++ b/test/snapshots/offer/issue_to_seller.expected.json @@ -0,0 +1,157 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 1000000000, + "Asset": { + "AlphaNum4": { + "AssetCode": [ + 79, + 70, + 70, + 82 + ], + "Issuer": "GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A" + }, + "Type": 1 + }, + "Destination": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS" + }, + "Type": 1 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 1000000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "mint", + "Type": 15 + }, + { + "Address": { + "AccountId": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "OFFR:GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A", + "Type": 14 + } + ] + } + }, + "ContractId": "CD6DKX6X5KYRTAOQILDZWKJAOOTK4EK4NEVUGGDTYLE2X3G3MGK55ZMM", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/offer/manage_sell.expected.json b/test/snapshots/offer/manage_sell.expected.json new file mode 100644 index 0000000..7475c63 --- /dev/null +++ b/test/snapshots/offer/manage_sell.expected.json @@ -0,0 +1,157 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "ManageSellOfferOp": { + "Amount": 100000000, + "Buying": { + "Type": 0 + }, + "OfferId": "$offerId", + "Price": { + "D": 2, + "N": 1 + }, + "Selling": { + "AlphaNum4": { + "AssetCode": [ + 79, + 70, + 70, + 82 + ], + "Issuer": "GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A" + }, + "Type": 1 + } + }, + "Type": 3 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS" + } + } + }, + "events": { + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "ManageSellOfferResult": { + "Code": 0, + "Success": { + "Offer": { + "Effect": 0, + "Offer": { + "Amount": 100000000, + "Buying": { + "Type": 0 + }, + "Ext": { + "V": 0 + }, + "Flags": 0, + "OfferId": "$offerId", + "Price": { + "D": 2, + "N": 1 + }, + "SellerId": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS", + "Selling": { + "AlphaNum4": { + "AssetCode": [ + 79, + 70, + 70, + 82 + ], + "Issuer": "GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A" + }, + "Type": 1 + } + } + } + } + }, + "Type": 3 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/offer/setup_trustline.expected.json b/test/snapshots/offer/setup_trustline.expected.json new file mode 100644 index 0000000..fbfa6a7 --- /dev/null +++ b/test/snapshots/offer/setup_trustline.expected.json @@ -0,0 +1,116 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "ChangeTrustOp": { + "Limit": 10000000000, + "Line": { + "AlphaNum4": { + "AssetCode": [ + 79, + 70, + 70, + 82 + ], + "Issuer": "GDJEBBEX7WFGALUI3LGK4QFBY3X7DJ2CERQ2ZDV5OYQR3ONBGAAZNV6A" + }, + "Type": 1 + } + }, + "Type": 6 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS" + } + } + }, + "events": { + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GAIZBKNNBCAICBLODMWMUQGBB3EC7W7JQVBMIPE22DKTDAFRHJRNBOHS", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "ChangeTrustResult": { + "Code": 0 + }, + "Type": 6 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/payment/double.expected.json b/test/snapshots/payment/double.expected.json new file mode 100644 index 0000000..5787421 --- /dev/null +++ b/test/snapshots/payment/double.expected.json @@ -0,0 +1,221 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 200, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 50000000, + "Asset": { + "Type": 0 + }, + "Destination": "GCUGCEXALUAFIL7WWX4MAYMH35NTWKRJA35S7XOZRMDQXSHVNAG262L6" + }, + "Type": 1 + } + }, + { + "Body": { + "PaymentOp": { + "Amount": 70000000, + "Asset": { + "Type": 0 + }, + "Destination": "GCUGCEXALUAFIL7WWX4MAYMH35NTWKRJA35S7XOZRMDQXSHVNAG262L6" + }, + "Type": 1 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDZYAQD2EJYDRBPDQ7QM66RC6LU3VDSYW2G3QEQEIRRIYT2SISWSRSFB" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 50000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDZYAQD2EJYDRBPDQ7QM66RC6LU3VDSYW2G3QEQEIRRIYT2SISWSRSFB", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCUGCEXALUAFIL7WWX4MAYMH35NTWKRJA35S7XOZRMDQXSHVNAG262L6", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ], + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 70000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDZYAQD2EJYDRBPDQ7QM66RC6LU3VDSYW2G3QEQEIRRIYT2SISWSRSFB", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCUGCEXALUAFIL7WWX4MAYMH35NTWKRJA35S7XOZRMDQXSHVNAG262L6", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 200 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDZYAQD2EJYDRBPDQ7QM66RC6LU3VDSYW2G3QEQEIRRIYT2SISWSRSFB", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 200, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + }, + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/payment/native.expected.json b/test/snapshots/payment/native.expected.json new file mode 100644 index 0000000..e690434 --- /dev/null +++ b/test/snapshots/payment/native.expected.json @@ -0,0 +1,155 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "PaymentOp": { + "Amount": 100000000, + "Asset": { + "Type": 0 + }, + "Destination": "GCNZXVGI7TBEFJLJLI6OXIMKJMQ65IJA4BA73QEPXCEJKHRYEE3QGQYM" + }, + "Type": 1 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDVVNK26QA3LZ5IVDWLT5XZDJMEKL4YBKZXI4QZQ2URCK2MQFDZ5WGOG" + } + } + }, + "events": { + "contract": [ + [ + { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100000000 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "transfer", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDVVNK26QA3LZ5IVDWLT5XZDJMEKL4YBKZXI4QZQ2URCK2MQFDZ5WGOG", + "Type": 0 + }, + "Type": 18 + }, + { + "Address": { + "AccountId": "GCNZXVGI7TBEFJLJLI6OXIMKJMQ65IJA4BA73QEPXCEJKHRYEE3QGQYM", + "Type": 0 + }, + "Type": 18 + }, + { + "Str": "native", + "Type": 14 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + } + ] + ], + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDVVNK26QA3LZ5IVDWLT5XZDJMEKL4YBKZXI4QZQ2URCK2MQFDZ5WGOG", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "PaymentResult": { + "Code": 0 + }, + "Type": 1 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/test/snapshots/seq/bump.expected.json b/test/snapshots/seq/bump.expected.json new file mode 100644 index 0000000..2ec2be8 --- /dev/null +++ b/test/snapshots/seq/bump.expected.json @@ -0,0 +1,104 @@ +{ + "applicationOrder": 1, + "createdAt": "$createdAt", + "envelope": { + "Type": 2, + "V1": { + "Signatures": [ + { + "Hint": "$hint", + "Signature": "$signature" + } + ], + "Tx": { + "Cond": { + "TimeBounds": { + "MaxTime": 0, + "MinTime": 0 + }, + "Type": 1 + }, + "Ext": { + "V": 0 + }, + "Fee": 100, + "Memo": { + "Type": 0 + }, + "Operations": [ + { + "Body": { + "BumpSequenceOp": { + "BumpTo": "$bumpTo" + }, + "Type": 11 + } + } + ], + "SeqNum": "$seqNum", + "SourceAccount": "GDWHN4UNM4HZFA25SLLMLXF7ES2IMTT66DQDIYKKUR2GUPEGIXARLT2J" + } + } + }, + "events": { + "transaction": [ + { + "Event": { + "Body": { + "V": 0, + "V0": { + "Data": { + "I128": { + "Hi": 0, + "Lo": 100 + }, + "Type": 10 + }, + "Topics": [ + { + "Sym": "fee", + "Type": 15 + }, + { + "Address": { + "AccountId": "GDWHN4UNM4HZFA25SLLMLXF7ES2IMTT66DQDIYKKUR2GUPEGIXARLT2J", + "Type": 0 + }, + "Type": 18 + } + ] + } + }, + "ContractId": "CDMLFMKMMD7MWZP3FKUBZPVHTUEDLSX4BYGYKH4GCESXYHS3IHQ4EIG4", + "Ext": { + "V": 0 + }, + "Type": 1 + }, + "Stage": 0 + } + ] + }, + "hash": "$hash", + "result": { + "Ext": { + "V": 0 + }, + "FeeCharged": 100, + "Result": { + "Code": 0, + "Results": [ + { + "Code": 0, + "Tr": { + "BumpSeqResult": { + "Code": 0 + }, + "Type": 11 + } + } + ] + } + }, + "status": "SUCCESS" +} diff --git a/utils/events.go b/utils/events.go new file mode 100644 index 0000000..3db535f --- /dev/null +++ b/utils/events.go @@ -0,0 +1,44 @@ +package utils + +import ( + "github.com/stellar/go-stellar-sdk/xdr" +) + +// IsNonDeterministicDiagnosticEvent reports whether a Soroban +// diagnostic event is wall-clock dependent and therefore varies across +// fetchers (captive-core replay vs RPC live observation). +// +// Currently only matches core_metrics.invoke_time_nsecs, which the +// host emits on every Soroban invocation with the measured wall-clock +// duration of the call. Other core_metrics (cpu_insns, mem_bytes, +// ledger_*_count) are deterministic functions of execution and match +// across fetchers. +// +// Stripping these before persisting blocks makes outputs byte-identical +// across fetchers. +func IsNonDeterministicDiagnosticEvent(ev xdr.DiagnosticEvent) bool { + body := ev.Event.Body.V0 + if body == nil || len(body.Topics) < 2 { + return false + } + t0, t1 := body.Topics[0], body.Topics[1] + if t0.Type != xdr.ScValTypeScvSymbol || t1.Type != xdr.ScValTypeScvSymbol { + return false + } + if t0.Sym == nil || t1.Sym == nil { + return false + } + return string(*t0.Sym) == "core_metrics" && string(*t1.Sym) == "invoke_time_nsecs" +} + +// IsNonDeterministicDiagnosticEventBytes is the raw-bytes variant: it +// decodes the XDR blob first, then defers to +// IsNonDeterministicDiagnosticEvent. Returns false if the bytes do not +// decode as a DiagnosticEvent. +func IsNonDeterministicDiagnosticEventBytes(raw []byte) bool { + var ev xdr.DiagnosticEvent + if err := ev.UnmarshalBinary(raw); err != nil { + return false + } + return IsNonDeterministicDiagnosticEvent(ev) +}