diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa400f12..af68eaaa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,6 @@ permissions: # If there's a prerelease-style suffix to the version, then the release(s) # will be marked as a prerelease. on: - pull_request: push: tags: - '**[0-9]+.[0-9]+.[0-9]+*' @@ -66,7 +65,7 @@ jobs: # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.29.0/cargo-dist-installer.sh | sh" + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.3/cargo-dist-installer.sh | sh" - name: Cache dist uses: actions/upload-artifact@v4 with: @@ -223,8 +222,8 @@ jobs: - plan - build-local-artifacts - build-global-artifacts - # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) - if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} runs-on: "ubuntu-22.04" diff --git a/dist-workspace.toml b/dist-workspace.toml index 2e8e3931..57274503 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -4,7 +4,7 @@ members = ["cargo:."] # Config for 'dist' [dist] # The preferred dist version to use in CI (Cargo.toml SemVer syntax) -cargo-dist-version = "0.29.0" +cargo-dist-version = "0.30.3" # CI backends to support ci = "github" # The installers to generate for each app @@ -16,7 +16,10 @@ install-path = "CARGO_HOME" # The preferred Rust toolchain to use in CI (rustup toolchain syntax) rust-toolchain-version = "1.89" # Whether to install an updater program -install-updater = true +install-updater = false +# Don't run the release dry-run on pull requests; only fire on version tags. +# (PR validation lives in ci.yml.) +pr-run-mode = "skip" # A GitHub repo to push Homebrew formulas to tap = "txpipe/homebrew-tap" # Publish jobs to run in CI diff --git a/examples/aws_lambda/README.md b/examples/aws_lambda/README.md index a4a5c73c..b9351f0b 100644 --- a/examples/aws_lambda/README.md +++ b/examples/aws_lambda/README.md @@ -13,20 +13,44 @@ flowchart LR - **Filters** - `SplitBlock`: breaks each block into individual transactions. - `ParseCbor`: decodes the raw transaction CBOR into structured records. -- **Sink** — `AwsLambda`: invokes `function_name` in `region` with each event as the payload. +- **Sink** — `AwsLambda`: invokes `function_name` in `region` with each event as the + JSON payload. ## Prerequisites - Built with the `aws` feature. -- AWS credentials available to the process (env vars, profile, or instance role) with - permission to invoke the function. -- Edit `region` and `function_name` in `daemon.toml` to match your function. +- A running Docker engine — LocalStack runs each Lambda invocation in its own container + (the compose file mounts the Docker socket for this). -## Run +## Run standalone (LocalStack) + +The included `docker-compose.yml` starts [LocalStack](https://www.localstack.cloud/) and +deploys `my-lambda` (a tiny function in `localstack/handler.py` that logs each event), so +the example runs without a real AWS account: ```sh cd examples/aws_lambda +docker compose up -d +``` + +Point the AWS SDK at LocalStack with dummy credentials, then run Oura: + +```sh +export AWS_ENDPOINT_URL=http://localhost.localstack.cloud:4566 +export AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_REGION=us-east-1 cargo run --features aws --bin oura -- daemon --config daemon.toml ``` (or `oura daemon --config daemon.toml` with a binary built with the `aws` feature.) + +Watch the function get invoked — one `lambda.Invoke => 200` per event: + +```sh +docker compose logs -f localstack | grep "lambda.Invoke" +``` + +## Run against real AWS + +Skip the compose step and the `AWS_ENDPOINT_URL` export. Provide real credentials (env +vars, profile, or instance role) with permission to invoke the function, and set `region` +and `function_name` in `daemon.toml` to match it. diff --git a/examples/aws_lambda/docker-compose.yml b/examples/aws_lambda/docker-compose.yml new file mode 100644 index 00000000..e6510c79 --- /dev/null +++ b/examples/aws_lambda/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3" +services: + # LocalStack emulates AWS locally, so the example runs without a real AWS account. + # The init script under ./localstack deploys a tiny function named `my-lambda` + # that logs each event it receives. + localstack: + image: localstack/localstack:3 + container_name: localstack-lambda + ports: + - "4566:4566" + environment: + - SERVICES=lambda + volumes: + - ./localstack:/etc/localstack/init/ready.d + # LocalStack runs each Lambda invocation in its own container. + - /var/run/docker.sock:/var/run/docker.sock diff --git a/examples/aws_lambda/localstack/handler.py b/examples/aws_lambda/localstack/handler.py new file mode 100644 index 00000000..d8fb960c --- /dev/null +++ b/examples/aws_lambda/localstack/handler.py @@ -0,0 +1,4 @@ +def handler(event, context): + # Oura invokes the function once per chain event; the payload is the JSON record. + print("oura event:", event) + return {"statusCode": 200} diff --git a/examples/aws_lambda/localstack/init.sh b/examples/aws_lambda/localstack/init.sh new file mode 100755 index 00000000..fa960f1f --- /dev/null +++ b/examples/aws_lambda/localstack/init.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Runs inside the LocalStack container once it is ready. +# Packages handler.py and deploys it as the function daemon.toml invokes. +cd /etc/localstack/init/ready.d +python3 -c "import zipfile; zipfile.ZipFile('/tmp/fn.zip','w').write('handler.py')" +awslocal lambda create-function \ + --function-name my-lambda \ + --runtime python3.12 \ + --handler handler.handler \ + --role arn:aws:iam::000000000000:role/lambda-role \ + --zip-file fileb:///tmp/fn.zip diff --git a/examples/aws_s3/README.md b/examples/aws_s3/README.md index 727fc598..296dc783 100644 --- a/examples/aws_s3/README.md +++ b/examples/aws_s3/README.md @@ -1,30 +1,52 @@ # AWS S3 sink -Decode transactions and write each one as an object into an S3 bucket. +Write each block's raw CBOR as an object into an S3 bucket. ## Pipeline ```mermaid flowchart LR - src[N2N source] --> f1[ParseCbor] --> sink[AwsS3 sink] + src[N2N source] --> sink[AwsS3 sink] ``` - **Source** — `N2N`: mainnet relay, starting from the chain tip. -- **Filters** — `ParseCbor`: decodes the raw transaction CBOR into structured records. -- **Sink** — `AwsS3`: writes objects under `prefix` in `bucket` (`region`). +- **Sink** — `AwsS3`: writes one object per block under `prefix` in `bucket` (`region`), + keyed by `slot.hash`. The `AwsS3` sink stores the block CBOR as-is, so the example runs + no filters — a `ParseCbor` / `SplitBlock` filter would change the record type and the + sink would reject it. ## Prerequisites - Built with the `aws` feature. -- AWS credentials available to the process (env vars, profile, or instance role) with - permission to write to the bucket. -- Edit `region`, `bucket`, and `prefix` in `daemon.toml` to match your bucket. -## Run +## Run standalone (LocalStack) + +The included `docker-compose.yml` starts [LocalStack](https://www.localstack.cloud/) and +provisions the `my-bucket` bucket, so the example runs without a real AWS account: ```sh cd examples/aws_s3 +docker compose up -d +``` + +Point the AWS SDK at LocalStack with dummy credentials, then run Oura: + +```sh +export AWS_ENDPOINT_URL=http://s3.localhost.localstack.cloud:4566 +export AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_REGION=us-east-1 cargo run --features aws --bin oura -- daemon --config daemon.toml ``` (or `oura daemon --config daemon.toml` with a binary built with the `aws` feature.) + +Inspect the objects Oura wrote: + +```sh +docker exec localstack-s3 awslocal s3 ls s3://my-bucket/mainnet/ +``` + +## Run against real AWS + +Skip the compose step and the `AWS_ENDPOINT_URL` export. Provide real credentials (env +vars, profile, or instance role) with permission to write to the bucket, and edit +`region`, `bucket`, and `prefix` in `daemon.toml` to match it. diff --git a/examples/aws_s3/daemon.toml b/examples/aws_s3/daemon.toml index 7a6d3b53..f0ff7211 100644 --- a/examples/aws_s3/daemon.toml +++ b/examples/aws_s3/daemon.toml @@ -8,8 +8,8 @@ type = "mainnet" [intersect] type = "Tip" -[[filters]] -type = "ParseCbor" +# The AwsS3 sink stores the raw block CBOR, so the block must reach it unparsed +# (no ParseCbor / SplitBlock filter). [sink] type = "AwsS3" diff --git a/examples/aws_s3/docker-compose.yml b/examples/aws_s3/docker-compose.yml new file mode 100644 index 00000000..60f02b00 --- /dev/null +++ b/examples/aws_s3/docker-compose.yml @@ -0,0 +1,13 @@ +version: "3" +services: + # LocalStack emulates AWS locally, so the example runs without a real AWS account. + # The init script under ./localstack provisions the bucket Oura writes to. + localstack: + image: localstack/localstack:3 + container_name: localstack-s3 + ports: + - "4566:4566" + environment: + - SERVICES=s3 + volumes: + - ./localstack:/etc/localstack/init/ready.d diff --git a/examples/aws_s3/localstack/init.sh b/examples/aws_s3/localstack/init.sh new file mode 100755 index 00000000..7fbb6d5b --- /dev/null +++ b/examples/aws_s3/localstack/init.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Runs inside the LocalStack container once it is ready. +# Creates the bucket that daemon.toml writes to. +awslocal s3 mb s3://my-bucket diff --git a/examples/aws_sqs/README.md b/examples/aws_sqs/README.md index 26c136b2..a01db93b 100644 --- a/examples/aws_sqs/README.md +++ b/examples/aws_sqs/README.md @@ -13,21 +13,43 @@ flowchart LR - **Filters** - `SplitBlock`: breaks each block into individual transactions. - `ParseCbor`: decodes the raw transaction CBOR into structured records. -- **Sink** — `AwsSqs`: sends messages to `queue_url` (`region`) with the configured - `group_id` (FIFO queues). +- **Sink** — `AwsSqs`: sends each event as a JSON message to `queue_url` (`region`). For a + FIFO queue (`queue_url` ending in `.fifo`) the configured `group_id` is used. ## Prerequisites - Built with the `aws` feature. -- AWS credentials available to the process (env vars, profile, or instance role) with - permission to send to the queue. -- Edit `region`, `queue_url`, and `group_id` in `daemon.toml` to match your queue. -## Run +## Run standalone (LocalStack) + +The included `docker-compose.yml` starts [LocalStack](https://www.localstack.cloud/) and +provisions the `my-queue` queue that `daemon.toml` points at, so the example runs without a +real AWS account: ```sh cd examples/aws_sqs +docker compose up -d +``` + +Point the AWS SDK at LocalStack with dummy credentials, then run Oura: + +```sh +export AWS_ENDPOINT_URL=http://localhost.localstack.cloud:4566 +export AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_REGION=us-east-1 cargo run --features aws --bin oura -- daemon --config daemon.toml ``` (or `oura daemon --config daemon.toml` with a binary built with the `aws` feature.) + +Read the messages Oura enqueued: + +```sh +docker exec localstack-sqs awslocal sqs receive-message \ + --queue-url http://localhost:4566/000000000000/my-queue +``` + +## Run against real AWS + +Skip the compose step and the `AWS_ENDPOINT_URL` export. Provide real credentials (env +vars, profile, or instance role) with permission to send to the queue, and set `region`, +`queue_url`, and `group_id` in `daemon.toml` to match your queue. diff --git a/examples/aws_sqs/daemon.toml b/examples/aws_sqs/daemon.toml index 635dfa95..e13e1c3e 100644 --- a/examples/aws_sqs/daemon.toml +++ b/examples/aws_sqs/daemon.toml @@ -17,5 +17,7 @@ type = "ParseCbor" [sink] type = "AwsSqs" region = "us-east-1" -queue_url = "https://sqs.us-east-1.amazonaws.com/*****/my-queue" +# Points at the queue created by the bundled LocalStack (see README). For a real +# AWS queue, use its URL: https://sqs..amazonaws.com//my-queue +queue_url = "http://localhost.localstack.cloud:4566/queue/us-east-1/000000000000/my-queue" group_id = "my_group" diff --git a/examples/aws_sqs/docker-compose.yml b/examples/aws_sqs/docker-compose.yml new file mode 100644 index 00000000..962c1aef --- /dev/null +++ b/examples/aws_sqs/docker-compose.yml @@ -0,0 +1,15 @@ +version: "3" +services: + # LocalStack emulates AWS locally, so the example runs without a real AWS account. + # The init script under ./localstack provisions the queue Oura writes to. + localstack: + image: localstack/localstack:3 + container_name: localstack-sqs + ports: + - "4566:4566" + environment: + - SERVICES=sqs + # Path-style queue URLs, so the queue_url in daemon.toml is deterministic. + - SQS_ENDPOINT_STRATEGY=path + volumes: + - ./localstack:/etc/localstack/init/ready.d diff --git a/examples/aws_sqs/localstack/init.sh b/examples/aws_sqs/localstack/init.sh new file mode 100755 index 00000000..2e7579ce --- /dev/null +++ b/examples/aws_sqs/localstack/init.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Runs inside the LocalStack container once it is ready. +# Creates the queue that daemon.toml writes to. +awslocal sqs create-queue --queue-name my-queue diff --git a/examples/gcp_cloudfunction/README.md b/examples/gcp_cloudfunction/README.md index c4ed194e..b7f7b294 100644 --- a/examples/gcp_cloudfunction/README.md +++ b/examples/gcp_cloudfunction/README.md @@ -1,6 +1,6 @@ # GCP Cloud Function sink -Decode transactions and call a Google Cloud Function once per event. +Decode transactions and call a Google Cloud Function (or any HTTP endpoint) once per event. ## Pipeline @@ -13,21 +13,36 @@ flowchart LR - **Filters** - `SplitBlock`: breaks each block into individual transactions. - `ParseCbor`: decodes the raw transaction CBOR into structured records. -- **Sink** — `GcpCloudFunction`: invokes the function at `url` - (`authentication = true` attaches a Google-issued identity token). +- **Sink** — `GcpCloudFunction`: POSTs each event as JSON to `url`. With + `authentication = true` it attaches a Google-issued identity token (requires real GCP + credentials); with `authentication = false` it is a plain HTTP POST. ## Prerequisites - Built with the `gcp` feature. -- GCP credentials available to the process (e.g. `GOOGLE_APPLICATION_CREDENTIALS`) with - permission to invoke the function. -- Edit `url` in `daemon.toml` to point at your deployed function. -## Run +## Run standalone (local endpoint) + +A Cloud Function is just an HTTP endpoint, so the included `docker-compose.yml` starts an +echo server on `:8080` that logs every request. `daemon.toml` posts to it with +`authentication = false`, so the example runs without a GCP project: ```sh cd examples/gcp_cloudfunction +docker compose up -d cargo run --features gcp --bin oura -- daemon --config daemon.toml ``` (or `oura daemon --config daemon.toml` with a binary built with the `gcp` feature.) + +Watch the events arrive at the endpoint: + +```sh +docker compose logs -f function +``` + +## Run against a real Cloud Function + +Set `url` in `daemon.toml` to the function's trigger URL and `authentication = true`, then +provide credentials via `GOOGLE_APPLICATION_CREDENTIALS` with permission to invoke it. The +identity-token path can't be exercised against the local echo server. diff --git a/examples/gcp_cloudfunction/daemon.toml b/examples/gcp_cloudfunction/daemon.toml index b8913166..3be8646b 100644 --- a/examples/gcp_cloudfunction/daemon.toml +++ b/examples/gcp_cloudfunction/daemon.toml @@ -13,5 +13,8 @@ type = "ParseCbor" [sink] type = "GcpCloudFunction" -url = "https://REGION-PROJECT_ID.cloudfunctions.net/FUNCTION_NAME" -authentication = true +# Posts events to the bundled echo server (see README). For a real Cloud Function, use its +# trigger URL (https://REGION-PROJECT_ID.cloudfunctions.net/FUNCTION_NAME) and set +# authentication = true so Oura attaches a Google-issued identity token. +url = "http://localhost:8080" +authentication = false diff --git a/examples/gcp_cloudfunction/docker-compose.yml b/examples/gcp_cloudfunction/docker-compose.yml new file mode 100644 index 00000000..aa0f7c29 --- /dev/null +++ b/examples/gcp_cloudfunction/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3" +services: + # A Cloud Function is just an HTTP endpoint, so this echo server stands in for one, + # logging every request Oura posts. It lets the example run without a GCP project. + function: + image: mendhak/http-https-echo:31 + container_name: cloudfunction-echo + environment: + - HTTP_PORT=8080 + ports: + - "8080:8080" diff --git a/examples/gcp_pubsub/README.md b/examples/gcp_pubsub/README.md index 931b38b6..419c7326 100644 --- a/examples/gcp_pubsub/README.md +++ b/examples/gcp_pubsub/README.md @@ -13,20 +13,45 @@ flowchart LR - **Filters** - `SplitBlock`: breaks each block into individual transactions. - `ParseCbor`: decodes the raw transaction CBOR into structured records. -- **Sink** — `GcpPubSub`: publishes events to `topic`. +- **Sink** — `GcpPubSub`: publishes each event as a message to `topic`. ## Prerequisites - Built with the `gcp` feature. -- GCP credentials available to the process (e.g. `GOOGLE_APPLICATION_CREDENTIALS`) with - permission to publish to the topic. -- Edit `topic` in `daemon.toml` to match your topic. -## Run +## Run standalone (Pub/Sub emulator) + +The included `docker-compose.yml` starts Google's Pub/Sub emulator and provisions the +`my-topic` topic (plus a `my-sub` subscription so you can read messages back), so the +example runs without a real GCP project: ```sh cd examples/gcp_pubsub +docker compose up -d +``` + +Point the client at the emulator, then run Oura: + +```sh +export PUBSUB_EMULATOR_HOST=localhost:8085 cargo run --features gcp --bin oura -- daemon --config daemon.toml ``` (or `oura daemon --config daemon.toml` with a binary built with the `gcp` feature.) + +> In emulator mode the Pub/Sub client always uses the project id `local-project`, so the +> compose file provisions the topic under that project. + +Pull the messages Oura published: + +```sh +curl -s -X POST \ + http://localhost:8085/v1/projects/local-project/subscriptions/my-sub:pull \ + -H 'Content-Type: application/json' -d '{"maxMessages":5}' +``` + +## Run against real GCP + +Skip the compose step and the `PUBSUB_EMULATOR_HOST` export. Provide credentials via +`GOOGLE_APPLICATION_CREDENTIALS` with permission to publish, and set `topic` in +`daemon.toml` to match your topic. diff --git a/examples/gcp_pubsub/docker-compose.yml b/examples/gcp_pubsub/docker-compose.yml new file mode 100644 index 00000000..09da7a37 --- /dev/null +++ b/examples/gcp_pubsub/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3" +services: + # Google's Pub/Sub emulator, so the example runs without a real GCP project. + # Oura targets it via the PUBSUB_EMULATOR_HOST env var (see README). + pubsub: + image: google/cloud-sdk:emulators + container_name: pubsub-emulator + command: gcloud beta emulators pubsub start --host-port=0.0.0.0:8085 --project=local-project + ports: + - "8085:8085" + + # Creates the topic Oura publishes to (plus a subscription so you can read the + # messages back), then exits. + init: + image: curlimages/curl + depends_on: + - pubsub + entrypoint: + - sh + - -c + - | + until curl -s -o /dev/null http://pubsub:8085; do sleep 1; done + curl -s -X PUT http://pubsub:8085/v1/projects/local-project/topics/my-topic + curl -s -X PUT http://pubsub:8085/v1/projects/local-project/subscriptions/my-sub \ + -H 'Content-Type: application/json' \ + -d '{"topic":"projects/local-project/topics/my-topic"}' diff --git a/examples/webhook_basics/README.md b/examples/webhook_basics/README.md index 1d41aba9..7b608aa7 100644 --- a/examples/webhook_basics/README.md +++ b/examples/webhook_basics/README.md @@ -20,7 +20,17 @@ See the [WebHook sink docs](../../docs/v2/sinks/webhook.mdx). ## Prerequisites -- An HTTP endpoint to receive the requests. Edit `url` in `daemon.toml` to point at it. +- An HTTP endpoint to receive the requests at the `url` in `daemon.toml` + (default `http://localhost:8080`). + +The included `docker-compose.yml` starts a small echo server on `:8080` that logs every +request it receives, so the example runs standalone: + +```sh +docker compose up -d +``` + +To post to your own endpoint instead, skip the compose step and edit `url` in `daemon.toml`. ## Run @@ -28,3 +38,9 @@ See the [WebHook sink docs](../../docs/v2/sinks/webhook.mdx). cd examples/webhook_basics oura daemon --config daemon.toml ``` + +Watch the events arrive at the receiver: + +```sh +docker compose logs -f webhook +``` diff --git a/examples/webhook_basics/docker-compose.yml b/examples/webhook_basics/docker-compose.yml new file mode 100644 index 00000000..32ba70b2 --- /dev/null +++ b/examples/webhook_basics/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3" +services: + webhook: + # Echoes every received request (method, headers, body) to the container logs, + # so you can watch the events Oura posts with `docker compose logs -f`. + image: mendhak/http-https-echo:31 + container_name: webhook-echo + environment: + - HTTP_PORT=8080 + ports: + - "8080:8080"