Add Captive Core block fetcher

.github/workflows/docker.yml

@@ -26,7 +26,7 @@ jobs:
 
           - platform: linux/arm64
             platform_suffix: arm64
-            runner: ubuntu-24.04
+            runner: ubuntu-24.04-arm
 
     runs-on: ${{ matrix.runner }}
 

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.24.x]
+        go-version: [1.26.x]
         os: [ubuntu-latest]
     steps:
       - name: Set up Go

.gitignore

@@ -7,3 +7,12 @@ devel/data
 /firestellar
 /build
 /dist
+
+# Runtime data dirs produced by devel/poller.sh and devel/captive-core.sh
+/data
+/data-cc
+/data-cc-mainnet
+/captive-core
+
+# Tests
+test/.data

Dockerfile

@@ -1,16 +1,13 @@
 ARG FIRECORE_VERSION=v1.14.1
 
-FROM golang:1.25-bookworm AS build
+FROM golang:1.26-bookworm AS build
 WORKDIR /app
 
-# Copy go mod files first for better caching
 COPY go.mod go.sum ./
 RUN go mod download
 
-# Copy source code
 COPY . ./
 
-# Build the binary with version information
 ARG VERSION="edge"
 ARG BINARY_NAME=firestellar
 
@@ -20,8 +17,43 @@ FROM ghcr.io/streamingfast/firehose-core:${FIRECORE_VERSION}
 
 ARG BINARY_NAME=firestellar
 
-# Copy the binary to the firehose-core image
+# Install stellar-core from SDF apt repo so the captive-core fetcher works
+# standalone (default --stellar-core-bin is /usr/bin/stellar-core).
+# SDF only publishes amd64 packages; arm64 images ship without stellar-core
+# and require mounting a binary at /usr/bin/stellar-core or overriding
+# --stellar-core-bin. The RPC fetcher works on arm64 without stellar-core.
+#
+# Security: stellar-core 26.1.0-3210.427aa3978 fixes a critical network
+# vulnerability (SDF advisory, May 2026). The build pulls from SDF's
+# `stable` apt channel which now ships the patched version; rebuilds
+# after that publish date pick it up automatically. STELLAR_CORE_MIN_VERSION
+# is asserted post-install to fail the build loudly if the apt index is
+# pinned/cached to a pre-fix package somehow.
+ARG TARGETARCH
+ARG STELLAR_CORE_MIN_VERSION=26.1.0
+RUN set -eux; \
+    if [ "${TARGETARCH}" = "amd64" ]; then \
+        apt-get update; \
+        apt-get install -y --no-install-recommends ca-certificates curl gnupg dpkg; \
+        install -m 0755 -d /etc/apt/keyrings; \
+        curl -sSL https://apt.stellar.org/SDF.asc | gpg --dearmor -o /etc/apt/keyrings/SDF.gpg; \
+        chmod a+r /etc/apt/keyrings/SDF.gpg; \
+        . /etc/os-release; \
+        echo "deb [signed-by=/etc/apt/keyrings/SDF.gpg] https://apt.stellar.org ${VERSION_CODENAME} stable" \
+            > /etc/apt/sources.list.d/SDF.list; \
+        apt-get update; \
+        apt-get install -y --no-install-recommends stellar-core; \
+        rm -rf /var/lib/apt/lists/*; \
+        stellar-core version; \
+        INSTALLED=$(dpkg-query -W -f='${Version}' stellar-core); \
+        if ! dpkg --compare-versions "${INSTALLED}" ge "${STELLAR_CORE_MIN_VERSION}"; then \
+            echo "stellar-core ${INSTALLED} is older than required ${STELLAR_CORE_MIN_VERSION}; refusing to build (see SDF May 2026 advisory)." >&2; \
+            exit 1; \
+        fi; \
+    else \
+        echo "Skipping stellar-core install on ${TARGETARCH} (SDF amd64-only). Mount /usr/bin/stellar-core or use --stellar-core-bin."; \
+    fi
+
 COPY --from=build "/app/${BINARY_NAME}" "/app/${BINARY_NAME}"
 
-# We use firecore entrypoint since it's the main application that people should run to setup Firehose stack
 ENTRYPOINT ["/app/firecore"]

Dockerfile-rpc-fetcher

@@ -0,0 +1,27 @@
+ARG FIRECORE_VERSION=v1.14.1
+
+FROM golang:1.26-bookworm AS build
+WORKDIR /app
+
+# Copy go mod files first for better caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY . ./
+
+# Build the binary with version information
+ARG VERSION="edge"
+ARG BINARY_NAME=firestellar
+
+RUN go build -v -ldflags "-X main.version=${VERSION}" -o "${BINARY_NAME}" "./cmd/${BINARY_NAME}"
+
+FROM ghcr.io/streamingfast/firehose-core:${FIRECORE_VERSION}
+
+ARG BINARY_NAME=firestellar
+
+# Copy the binary to the firehose-core image
+COPY --from=build "/app/${BINARY_NAME}" "/app/${BINARY_NAME}"
+
+# We use firecore entrypoint since it's the main application that people should run to setup Firehose stack
+ENTRYPOINT ["/app/firecore"]

README.md

@@ -11,14 +11,38 @@ Quick start with Firehose for Stellar can be found in the official Firehose docs
   - [Data Storage](https://firehose.streamingfast.io/concepts-and-architeceture/data-storage)
   - [Design Principles](https://firehose.streamingfast.io/concepts-and-architeceture/design-principles)
 
-## Running the Firehose poller
+## Running the Firehose fetcher
 
-The below command with start streaming Firehose Stellar blocks, check `proto/sf/stellar/type/v1/block.proto` for more information.
+Two fetcher backends are available. Both emit the same `pbbstream.Block` shape; check `proto/sf/stellar/type/v1/block.proto` for the payload schema.
+
+### RPC backend
+
+Streams ledgers from a Stellar RPC endpoint.
 
 ```bash
 firestellar fetch rpc {FIRST_STREAMABLE_BLOCK} --endpoints {STELLAR_RPC_ENDPOINT} --state-dir {STATE_DIR}
 ```
 
+### Captive-core backend
+
+Spawns a `stellar-core` subprocess and streams ledgers from it.
+
+```bash
+firestellar fetch captive-core {FIRST_STREAMABLE_BLOCK} \
+  --stellar-core-bin /usr/bin/stellar-core \
+  --stellar-core-network mainnet \
+  --state-dir {STATE_DIR}
+```
+
+### Resume behavior (`--state-dir` / `--ignore-cursor`)
+
+Both backends persist the last fired block to `{STATE_DIR}/cursor.json` after each successful emission. On restart, the fetcher resumes at `last_fired_block + 1` instead of replaying from `{FIRST_STREAMABLE_BLOCK}`.
+
+- `--state-dir` — directory holding `cursor.json`. Defaults: `/data/poller` (rpc), `/data/captive-core` (captive-core). Pass an empty string to disable persistence.
+- `--ignore-cursor` — ignore any persisted `cursor.json` and start fresh from `{FIRST_STREAMABLE_BLOCK}`. Use this when running under a supervisor (e.g. `firecore reader-node`) that already tracks downstream state and passes the correct start block on restart.
+
+The cursor schema is shared between the two backends, so a single state directory can be reused if you switch backends.
+
 ## Contributing
 
 For more information, please read the [CONTRIBUTING.md](CONTRIBUTING.md) file.