From ef30b5daafe0fcd720699b5cf511520290e54cb1 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 20 Feb 2026 17:10:51 +0000 Subject: [PATCH 001/115] Basic CCC store and message --- CLAUDE.md | 34 +++- lib/sourced/ccc.rb | 9 + lib/sourced/ccc/message.rb | 92 +++++++++ lib/sourced/ccc/store.rb | 193 +++++++++++++++++++ spec/sourced/ccc/message_spec.rb | 137 ++++++++++++++ spec/sourced/ccc/store_spec.rb | 315 +++++++++++++++++++++++++++++++ 6 files changed, 779 insertions(+), 1 deletion(-) create mode 100644 lib/sourced/ccc.rb create mode 100644 lib/sourced/ccc/message.rb create mode 100644 lib/sourced/ccc/store.rb create mode 100644 spec/sourced/ccc/message_spec.rb create mode 100644 spec/sourced/ccc/store_spec.rb diff --git a/CLAUDE.md b/CLAUDE.md index eb56bbc0..2855e28d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -135,4 +135,36 @@ end - Default error strategy logs exceptions and stops consumer groups - Configurable retry/backoff strategies available -- Consumer groups can be stopped/started programmatically via backend API \ No newline at end of file +- Consumer groups can be stopped/started programmatically via backend API + +## CCC Module — Stream-less Event Sourcing (Experimental) + +`Sourced::CCC` is a prototype for aggregateless, stream-less event sourcing inspired by "Context-driven Consistency Checks". Events go into a flat, globally-ordered log. Consistency context is assembled dynamically by querying relevant facts via normalized key-value pairs extracted from event payloads. + +### Files +- `lib/sourced/ccc.rb` — module entrypoint +- `lib/sourced/ccc/message.rb` — `CCC::Message` base class, `QueryCondition` +- `lib/sourced/ccc/store.rb` — `CCC::Store` (SQLite), `PositionedMessage` wrapper +- `spec/sourced/ccc/` — specs (40 examples) + +### CCC::Message +- Extends `Types::Data` like `Sourced::Message`, but without `stream_id`, `seq`, `causation_id`, `correlation_id` +- Own `Registry`, separate from `Sourced::Message.registry` +- `.define(type_str, &block)` — creates subclass with typed payload (same DSL as `Sourced::Message`) +- `.from(hash)` — instantiate correct subclass from type string +- `#extracted_keys` — auto-extracts `[[name, value], ...]` from all top-level payload attributes (skips nils) + +### CCC::Store +- Accepts a `Sequel::SQLite::Database` connection +- 3 tables: `ccc_messages` (append-only log with auto-increment position), `ccc_key_pairs` (deduplicated name/value pairs), `ccc_message_key_pairs` (join table) +- `append(messages)` — writes messages + auto-indexes all payload keys, returns last position +- `read(conditions, from_position:, limit:)` — queries by `QueryCondition` array (message_type + key_name + key_value), OR semantics +- `messages_since(conditions, position)` — conflict detection (messages matching conditions after position) +- `PositionedMessage` — `SimpleDelegator` wrapper adding `#position` to frozen `Types::Data` instances + +### CCC::QueryCondition +- `Data.define(:message_type, :key_name, :key_value)` — used to query the store + +### Design Reference +- Article: `plans/ccc/ccc.md` +- TypeScript reference: [Boundless SQLite storage](https://github.com/SBortz/boundless) \ No newline at end of file diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb new file mode 100644 index 00000000..4bcd5f5e --- /dev/null +++ b/lib/sourced/ccc.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Sourced + module CCC + end +end + +require 'sourced/ccc/message' +require 'sourced/ccc/store' diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb new file mode 100644 index 00000000..55b4e47c --- /dev/null +++ b/lib/sourced/ccc/message.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'sourced/types' + +module Sourced + module CCC + QueryCondition = Data.define(:message_type, :key_name, :key_value) + + class Message < Types::Data + attribute :id, Types::AutoUUID + attribute :type, Types::String.present + attribute :created_at, Types::Forms::Time.default { Time.now } + attribute :metadata, Types::Hash.default(Plumb::BLANK_HASH) + attribute :payload, Types::Static[nil] + + class Registry + def initialize(message_class) + @message_class = message_class + @lookup = {} + end + + def keys = @lookup.keys + def subclasses = message_class.subclasses + + def []=(key, klass) + @lookup[key] = klass + end + + def [](key) + klass = lookup[key] + return klass if klass + + subclasses.each do |c| + klass = c.registry[key] + return klass if klass + end + nil + end + + private + + attr_reader :lookup, :message_class + end + + def self.registry + @registry ||= Registry.new(self) + end + + class Payload < Types::Data + def [](key) = attributes[key] + def fetch(...) = to_h.fetch(...) + end + + def self.define(type_str, &payload_block) + type_str.freeze unless type_str.frozen? + + registry[type_str] = Class.new(self) do + def self.node_name = :data + define_singleton_method(:type) { type_str } + + attribute :type, Types::Static[type_str] + if block_given? + const_set(:Payload, Class.new(Payload, &payload_block)) + attribute :payload, self::Payload + end + end + end + + def self.from(attrs) + klass = registry[attrs[:type]] + raise Sourced::UnknownMessageError, "Unknown message type: #{attrs[:type]}" unless klass + + klass.new(attrs) + end + + def initialize(attrs = {}) + attrs = attrs.merge(payload: {}) unless attrs[:payload] + super(attrs) + end + + # Auto-extract key-value pairs from all top-level payload attributes. + # Skips nil values. Returns array of [name, value] pairs. + def extracted_keys + return [] unless payload + + payload.to_h.filter_map { |k, v| + [k.to_s, v.to_s] unless v.nil? + } + end + end + end +end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb new file mode 100644 index 00000000..6e73dc75 --- /dev/null +++ b/lib/sourced/ccc/store.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +require 'json' + +module Sourced + module CCC + # Wraps a Message with a storage position. Delegates all message methods. + class PositionedMessage < SimpleDelegator + attr_reader :position + + def initialize(message, position) + super(message) + @position = position + end + + def class = __getobj__.class + def is_a?(klass) = __getobj__.is_a?(klass) || super + def kind_of?(klass) = is_a?(klass) + def instance_of?(klass) = __getobj__.instance_of?(klass) + end + + class Store + attr_reader :db + + def initialize(db) + @db = db + @db.run('PRAGMA foreign_keys = ON') + @db.run('PRAGMA journal_mode = WAL') + @db.run('PRAGMA busy_timeout = 5000') + end + + def installed? + db.table_exists?(:ccc_messages) && + db.table_exists?(:ccc_key_pairs) && + db.table_exists?(:ccc_message_key_pairs) + end + + def install! + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_messages ( + position INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT NOT NULL UNIQUE, + message_type TEXT NOT NULL, + payload TEXT NOT NULL, + metadata TEXT, + created_at TEXT NOT NULL + ) + SQL + db.run('CREATE INDEX IF NOT EXISTS idx_ccc_message_type ON ccc_messages(message_type)') + + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_key_pairs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + value TEXT NOT NULL, + UNIQUE(name, value) + ) + SQL + db.run('CREATE INDEX IF NOT EXISTS idx_ccc_key_pair_nv ON ccc_key_pairs(name, value)') + + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_message_key_pairs ( + message_position INTEGER NOT NULL REFERENCES ccc_messages(position), + key_pair_id INTEGER NOT NULL REFERENCES ccc_key_pairs(id), + PRIMARY KEY (message_position, key_pair_id) + ) + SQL + db.run('CREATE INDEX IF NOT EXISTS idx_ccc_mkp_key ON ccc_message_key_pairs(key_pair_id, message_position)') + end + + # Append messages to the store. Extracts keys and indexes them. + # Returns the last assigned position. + def append(messages) + messages = Array(messages) + return latest_position if messages.empty? + + last_position = nil + + db.transaction do + messages.each do |msg| + payload_json = msg.payload ? JSON.dump(msg.payload.to_h) : '{}' + metadata_json = msg.metadata.empty? ? nil : JSON.dump(msg.metadata) + + db[:ccc_messages].insert( + message_id: msg.id, + message_type: msg.type, + payload: payload_json, + metadata: metadata_json, + created_at: msg.created_at.iso8601 + ) + + last_position = db[:ccc_messages].where(message_id: msg.id).get(:position) + + # Extract and index key pairs + msg.extracted_keys.each do |name, value| + db.run("INSERT OR IGNORE INTO ccc_key_pairs (name, value) VALUES (#{db.literal(name)}, #{db.literal(value)})") + key_pair_id = db[:ccc_key_pairs].where(name: name, value: value).get(:id) + + db[:ccc_message_key_pairs].insert( + message_position: last_position, + key_pair_id: key_pair_id + ) + end + end + end + + last_position + end + + # Query messages by conditions (array of QueryCondition). + # Each condition matches (message_type AND key_name/key_value). + # Conditions are OR'd together. + def read(conditions, from_position: nil, limit: nil) + conditions = Array(conditions) + return [] if conditions.empty? + + # Step 1: resolve key_pair IDs + key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq + or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } + key_rows = db.fetch("SELECT id, name, value FROM ccc_key_pairs WHERE #{or_clauses.join(' OR ')}").all + + key_pair_index = {} + key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } + + # Build condition clauses using resolved key_pair IDs + where_parts = conditions.filter_map do |c| + kp_id = key_pair_index[[c.key_name, c.key_value]] + next unless kp_id # key pair not in DB means no matches for this condition + + "(m.message_type = #{db.literal(c.message_type)} AND mkp.key_pair_id = #{db.literal(kp_id)})" + end + + return [] if where_parts.empty? + + sql = <<~SQL + SELECT DISTINCT m.position, m.message_id, m.message_type, m.payload, m.metadata, m.created_at + FROM ccc_messages m + JOIN ccc_message_key_pairs mkp ON m.position = mkp.message_position + WHERE (#{where_parts.join(' OR ')}) + SQL + + sql += " AND m.position > #{db.literal(from_position)}" if from_position + sql += ' ORDER BY m.position' + sql += " LIMIT #{db.literal(limit)}" if limit + + db.fetch(sql).map { |row| deserialize(row) } + end + + # Conflict detection: returns messages matching conditions that appeared + # after the given position. Empty array means no conflicts. + def messages_since(conditions, position) + read(conditions, from_position: position) + end + + # Current max position, or 0 if the store is empty. + def latest_position + db[:ccc_messages].max(:position) || 0 + end + + # Clear all tables. For testing only. + def clear! + db[:ccc_message_key_pairs].delete + db[:ccc_key_pairs].delete + db[:ccc_messages].delete + db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) + end + + private + + def deserialize(row) + payload = JSON.parse(row[:payload], symbolize_names: true) + metadata = row[:metadata] ? JSON.parse(row[:metadata], symbolize_names: true) : {} + + klass = Message.registry[row[:message_type]] + attrs = { + id: row[:message_id], + type: row[:message_type], + created_at: row[:created_at], + metadata: metadata, + payload: payload + } + + msg = if klass + klass.new(attrs) + else + Message.new(attrs) + end + + PositionedMessage.new(msg, row[:position]) + end + end + end +end diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb new file mode 100644 index 00000000..fd0cd9b0 --- /dev/null +++ b/spec/sourced/ccc/message_spec.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +module CCCTestMessages + DeviceRegistered = Sourced::CCC::Message.define('device.registered') do + attribute :device_id, String + attribute :name, String + end + + AssetRegistered = Sourced::CCC::Message.define('asset.registered') do + attribute :asset_id, String + attribute :label, String + end + + SystemUpdated = Sourced::CCC::Message.define('system.updated') do + attribute :version, String + end + + OptionalFields = Sourced::CCC::Message.define('test.optional_fields') do + attribute? :required_field, String + attribute? :optional_field, String + end +end + +RSpec.describe Sourced::CCC::Message do + describe '.define' do + it 'creates a subclass with a type string' do + expect(CCCTestMessages::DeviceRegistered.type).to eq('device.registered') + end + + it 'creates a typed payload' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.payload.device_id).to eq('dev-1') + expect(msg.payload.name).to eq('Sensor A') + end + + it 'auto-generates an id' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.id).not_to be_nil + expect(msg.id).to match(/\A[0-9a-f-]{36}\z/) + end + + it 'sets created_at automatically' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.created_at).to be_a(Time) + end + + it 'sets type on the instance' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.type).to eq('device.registered') + end + + it 'defaults metadata to empty hash' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.metadata).to eq({}) + end + + it 'accepts metadata' do + msg = CCCTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + expect(msg.metadata[:user_id]).to eq(42) + end + end + + describe '.from' do + it 'instantiates the correct subclass from a hash' do + msg = Sourced::CCC::Message.from(type: 'device.registered', payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg).to be_a(CCCTestMessages::DeviceRegistered) + expect(msg.payload.device_id).to eq('dev-1') + end + + it 'raises UnknownMessageError for unknown types' do + expect { + Sourced::CCC::Message.from(type: 'unknown.type', payload: {}) + }.to raise_error(Sourced::UnknownMessageError, /Unknown message type: unknown.type/) + end + end + + describe '.registry' do + it 'stores defined message types' do + expect(Sourced::CCC::Message.registry['device.registered']).to eq(CCCTestMessages::DeviceRegistered) + expect(Sourced::CCC::Message.registry['asset.registered']).to eq(CCCTestMessages::AssetRegistered) + end + + it 'is separate from Sourced::Message registry' do + expect(Sourced::CCC::Message.registry['device.registered']).not_to be_nil + expect(Sourced::Message.registry['device.registered']).to be_nil + end + end + + describe '#extracted_keys' do + it 'extracts all top-level payload attributes as string pairs' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + keys = msg.extracted_keys + expect(keys).to contain_exactly( + ['device_id', 'dev-1'], + ['name', 'Sensor A'] + ) + end + + it 'skips nil values' do + msg = CCCTestMessages::OptionalFields.new(payload: { required_field: 'present', optional_field: nil }) + keys = msg.extracted_keys + expect(keys).to eq([['required_field', 'present']]) + end + + it 'converts values to strings' do + msg = CCCTestMessages::SystemUpdated.new(payload: { version: 'v2.0.5' }) + keys = msg.extracted_keys + expect(keys).to eq([['version', 'v2.0.5']]) + end + + it 'returns empty array for messages without payload attributes' do + # Message base class with no payload definition + bare = Sourced::CCC::Message.define('test.bare') + msg = bare.new + expect(msg.extracted_keys).to eq([]) + end + end + + describe Sourced::CCC::QueryCondition do + it 'is a Data struct with message_type, key_name, key_value' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + expect(cond.message_type).to eq('device.registered') + expect(cond.key_name).to eq('device_id') + expect(cond.key_value).to eq('dev-1') + end + end +end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb new file mode 100644 index 00000000..33233f08 --- /dev/null +++ b/spec/sourced/ccc/store_spec.rb @@ -0,0 +1,315 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +# Define test messages for store specs (namespaced to avoid collisions) +module CCCStoreTestMessages + DeviceRegistered = Sourced::CCC::Message.define('store_test.device.registered') do + attribute :device_id, String + attribute :name, String + end + + AssetRegistered = Sourced::CCC::Message.define('store_test.asset.registered') do + attribute :asset_id, String + attribute :label, String + end + + DeviceBound = Sourced::CCC::Message.define('store_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end +end + +RSpec.describe Sourced::CCC::Store do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + + before do + store.install! + end + + describe '#installed?' do + it 'returns true after install!' do + expect(store.installed?).to be true + end + + it 'returns false before install!' do + fresh_db = Sequel.sqlite + fresh_store = Sourced::CCC::Store.new(fresh_db) + expect(fresh_store.installed?).to be false + end + end + + describe '#install!' do + it 'creates the three tables' do + expect(db.table_exists?(:ccc_messages)).to be true + expect(db.table_exists?(:ccc_key_pairs)).to be true + expect(db.table_exists?(:ccc_message_key_pairs)).to be true + end + + it 'is idempotent' do + expect { store.install! }.not_to raise_error + end + end + + describe '#append' do + it 'appends a single message and returns position' do + msg = CCCStoreTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' } + ) + pos = store.append(msg) + expect(pos).to eq(1) + end + + it 'appends multiple messages and returns last position' do + msgs = [ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }) + ] + pos = store.append(msgs) + expect(pos).to eq(2) + end + + it 'extracts and indexes key pairs' do + msg = CCCStoreTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' } + ) + store.append(msg) + + key_pairs = db[:ccc_key_pairs].all + expect(key_pairs.map { |r| [r[:name], r[:value]] }).to contain_exactly( + ['device_id', 'dev-1'], + ['name', 'Sensor A'] + ) + + join_rows = db[:ccc_message_key_pairs].all + expect(join_rows.size).to eq(2) + end + + it 'deduplicates key pairs across messages' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor B' }) + store.append([msg1, msg2]) + + # 'device_id'/'dev-1' should exist once in key_pairs + count = db[:ccc_key_pairs].where(name: 'device_id', value: 'dev-1').count + expect(count).to eq(1) + + # But both messages reference it via the join table + kp_id = db[:ccc_key_pairs].where(name: 'device_id', value: 'dev-1').get(:id) + join_count = db[:ccc_message_key_pairs].where(key_pair_id: kp_id).count + expect(join_count).to eq(2) + end + + it 'stores metadata as JSON' do + msg = CCCStoreTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + store.append(msg) + + row = db[:ccc_messages].first + meta = JSON.parse(row[:metadata], symbolize_names: true) + expect(meta[:user_id]).to eq(42) + end + + it 'returns latest_position for empty array' do + pos = store.append([]) + expect(pos).to eq(0) + end + end + + describe '#read' do + before do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'Sensor B' }) + ]) + end + + it 'queries by message_type and key condition' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + results = store.read([cond]) + expect(results.size).to eq(1) + expect(results.first.type).to eq('store_test.device.registered') + expect(results.first.payload.device_id).to eq('dev-1') + end + + it 'returns messages with position' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + results = store.read([cond]) + expect(results.first.position).to eq(1) + end + + it 'queries with multiple OR conditions' do + conditions = [ + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ), + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + key_name: 'device_id', + key_value: 'dev-1' + ) + ] + results = store.read(conditions) + expect(results.size).to eq(2) + expect(results.map(&:type)).to contain_exactly( + 'store_test.device.registered', + 'store_test.device.bound' + ) + end + + it 'filters with from_position' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + # dev-1 registered is at position 1, so from_position: 1 should return nothing + results = store.read([cond], from_position: 1) + expect(results).to be_empty + + # from_position: 0 should return it + results = store.read([cond], from_position: 0) + expect(results.size).to eq(1) + end + + it 'applies limit' do + conditions = [ + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ), + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + key_name: 'device_id', + key_value: 'dev-1' + ) + ] + results = store.read(conditions, limit: 1) + expect(results.size).to eq(1) + expect(results.first.position).to eq(1) # first by position order + end + + it 'returns empty for non-matching conditions' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'nonexistent' + ) + results = store.read([cond]) + expect(results).to be_empty + end + + it 'returns empty for empty conditions' do + expect(store.read([])).to be_empty + end + + it 'deserializes into correct message subclasses' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + results = store.read([cond]) + msg = results.first + expect(msg).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(msg.id).to match(/\A[0-9a-f-]{36}\z/) + expect(msg.created_at).to be_a(Time) + end + end + + describe '#messages_since' do + it 'returns messages after the given position' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg1) + pos = store.latest_position + + msg2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A Updated' }) + store.append(msg2) + + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + + conflicts = store.messages_since([cond], pos) + expect(conflicts.size).to eq(1) + expect(conflicts.first.payload.name).to eq('Sensor A Updated') + end + + it 'returns empty when no new messages' do + msg = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg) + pos = store.latest_position + + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + + conflicts = store.messages_since([cond], pos) + expect(conflicts).to be_empty + end + end + + describe '#latest_position' do + it 'returns 0 for empty store' do + expect(store.latest_position).to eq(0) + end + + it 'returns max position after appends' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + expect(store.latest_position).to eq(2) + end + end + + describe '#clear!' do + it 'deletes all data and resets position' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + expect(store.latest_position).to eq(1) + + store.clear! + + expect(store.latest_position).to eq(0) + expect(db[:ccc_messages].count).to eq(0) + expect(db[:ccc_key_pairs].count).to eq(0) + expect(db[:ccc_message_key_pairs].count).to eq(0) + end + + it 'resets autoincrement so next position starts from 1' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + store.clear! + + pos = store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ) + expect(pos).to eq(1) + end + end +end From 6758e111059a1ae69fc523e3269b042a4f4c3c2c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 20 Feb 2026 17:30:25 +0000 Subject: [PATCH 002/115] Read and append with consistency guard --- lib/sourced/ccc/message.rb | 1 + lib/sourced/ccc/store.rb | 67 ++++++++++----- spec/sourced/ccc/store_spec.rb | 146 ++++++++++++++++++++++++++++++--- 3 files changed, 183 insertions(+), 31 deletions(-) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 55b4e47c..327101dd 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -5,6 +5,7 @@ module Sourced module CCC QueryCondition = Data.define(:message_type, :key_name, :key_value) + ConsistencyGuard = Data.define(:conditions, :last_position) class Message < Types::Data attribute :id, Types::AutoUUID diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 6e73dc75..f2094584 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -69,14 +69,22 @@ def install! end # Append messages to the store. Extracts keys and indexes them. + # When a ConsistencyGuard is provided via `guard:`, checks for conflicts + # before inserting. Raises Sourced::ConcurrentAppendError if conflicting + # messages have been appended since the guard's position. # Returns the last assigned position. - def append(messages) + def append(messages, guard: nil) messages = Array(messages) return latest_position if messages.empty? last_position = nil db.transaction do + if guard + conflicts = check_conflicts(guard.conditions, guard.last_position) + raise Sourced::ConcurrentAppendError, "Conflicting messages found after position #{guard.last_position}" if conflicts.any? + end + messages.each do |msg| payload_json = msg.payload ? JSON.dump(msg.payload.to_h) : '{}' metadata_json = msg.metadata.empty? ? nil : JSON.dump(msg.metadata) @@ -112,8 +120,41 @@ def append(messages) # Conditions are OR'd together. def read(conditions, from_position: nil, limit: nil) conditions = Array(conditions) - return [] if conditions.empty? + if conditions.empty? + guard = ConsistencyGuard.new(conditions: conditions, last_position: from_position || latest_position) + return [[], guard] + end + + messages = query_messages(conditions, from_position: from_position, limit: limit) + last_pos = messages.any? ? messages.last.position : (from_position || latest_position) + guard = ConsistencyGuard.new(conditions: conditions, last_position: last_pos) + [messages, guard] + end + + # Conflict detection: returns messages matching conditions that appeared + # after the given position. Empty array means no conflicts. + # Returns [messages, guard] like #read. + def messages_since(conditions, position) + read(conditions, from_position: position) + end + + # Current max position, or 0 if the store is empty. + def latest_position + db[:ccc_messages].max(:position) || 0 + end + + # Clear all tables. For testing only. + def clear! + db[:ccc_message_key_pairs].delete + db[:ccc_key_pairs].delete + db[:ccc_messages].delete + db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) + end + + private + # Core query logic shared by #read and #check_conflicts. + def query_messages(conditions, from_position: nil, limit: nil) # Step 1: resolve key_pair IDs key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } @@ -146,27 +187,13 @@ def read(conditions, from_position: nil, limit: nil) db.fetch(sql).map { |row| deserialize(row) } end - # Conflict detection: returns messages matching conditions that appeared - # after the given position. Empty array means no conflicts. - def messages_since(conditions, position) - read(conditions, from_position: position) - end - - # Current max position, or 0 if the store is empty. - def latest_position - db[:ccc_messages].max(:position) || 0 - end + # Check for conflicting messages after a given position. + def check_conflicts(conditions, after_position) + return [] if conditions.empty? - # Clear all tables. For testing only. - def clear! - db[:ccc_message_key_pairs].delete - db[:ccc_key_pairs].delete - db[:ccc_messages].delete - db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) + query_messages(conditions, from_position: after_position) end - private - def deserialize(row) payload = JSON.parse(row[:payload], symbolize_names: true) metadata = row[:metadata] ? JSON.parse(row[:metadata], symbolize_names: true) : {} diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 33233f08..dea68df3 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -121,6 +121,85 @@ module CCCStoreTestMessages end end + describe '#append with guard (conditional append)' do + let(:cond) do + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + end + + it 'succeeds when no conflicts exist' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg1) + + _events, guard = store.read([cond]) + + msg2 = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + pos = store.append(msg2, guard: guard) + expect(pos).to eq(2) + end + + it 'raises ConcurrentAppendError when conflicts exist' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg1) + + _events, guard = store.read([cond]) + + # Concurrent write by another process + conflicting = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) + store.append(conflicting) + + new_msg = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + expect { + store.append(new_msg, guard: guard) + }.to raise_error(Sourced::ConcurrentAppendError) + end + + it 'is atomic — failed append does not change store state' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg1) + + _events, guard = store.read([cond]) + + # Concurrent write + conflicting = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) + store.append(conflicting) + + position_before = store.latest_position + count_before = db[:ccc_messages].count + + new_msg = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + expect { + store.append(new_msg, guard: guard) + }.to raise_error(Sourced::ConcurrentAppendError) + + expect(store.latest_position).to eq(position_before) + expect(db[:ccc_messages].count).to eq(count_before) + end + + it 'works with a manually constructed guard' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg1) + + guard = Sourced::CCC::ConsistencyGuard.new(conditions: [cond], last_position: store.latest_position) + + msg2 = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + pos = store.append(msg2, guard: guard) + expect(pos).to eq(2) + end + + it 'append without guard still works unconditionally' do + msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + store.append(msg1) + + msg2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) + pos = store.append(msg2) + expect(pos).to eq(2) + end + end + describe '#read' do before do store.append([ @@ -137,7 +216,7 @@ module CCCStoreTestMessages key_name: 'device_id', key_value: 'dev-1' ) - results = store.read([cond]) + results, guard = store.read([cond]) expect(results.size).to eq(1) expect(results.first.type).to eq('store_test.device.registered') expect(results.first.payload.device_id).to eq('dev-1') @@ -149,7 +228,7 @@ module CCCStoreTestMessages key_name: 'device_id', key_value: 'dev-1' ) - results = store.read([cond]) + results, _guard = store.read([cond]) expect(results.first.position).to eq(1) end @@ -166,7 +245,7 @@ module CCCStoreTestMessages key_value: 'dev-1' ) ] - results = store.read(conditions) + results, _guard = store.read(conditions) expect(results.size).to eq(2) expect(results.map(&:type)).to contain_exactly( 'store_test.device.registered', @@ -181,11 +260,11 @@ module CCCStoreTestMessages key_value: 'dev-1' ) # dev-1 registered is at position 1, so from_position: 1 should return nothing - results = store.read([cond], from_position: 1) + results, _guard = store.read([cond], from_position: 1) expect(results).to be_empty # from_position: 0 should return it - results = store.read([cond], from_position: 0) + results, _guard = store.read([cond], from_position: 0) expect(results.size).to eq(1) end @@ -202,7 +281,7 @@ module CCCStoreTestMessages key_value: 'dev-1' ) ] - results = store.read(conditions, limit: 1) + results, _guard = store.read(conditions, limit: 1) expect(results.size).to eq(1) expect(results.first.position).to eq(1) # first by position order end @@ -213,12 +292,13 @@ module CCCStoreTestMessages key_name: 'device_id', key_value: 'nonexistent' ) - results = store.read([cond]) + results, _guard = store.read([cond]) expect(results).to be_empty end it 'returns empty for empty conditions' do - expect(store.read([])).to be_empty + results, _guard = store.read([]) + expect(results).to be_empty end it 'deserializes into correct message subclasses' do @@ -227,12 +307,55 @@ module CCCStoreTestMessages key_name: 'device_id', key_value: 'dev-1' ) - results = store.read([cond]) + results, _guard = store.read([cond]) msg = results.first expect(msg).to be_a(CCCStoreTestMessages::DeviceRegistered) expect(msg.id).to match(/\A[0-9a-f-]{36}\z/) expect(msg.created_at).to be_a(Time) end + + it 'returns a ConsistencyGuard with correct conditions' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-1' + ) + _results, guard = store.read([cond]) + expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(guard.conditions).to eq([cond]) + end + + it 'guard last_position reflects the last result position' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'dev-2' + ) + results, guard = store.read([cond]) + # dev-2 is at position 4 + expect(results.size).to eq(1) + expect(guard.last_position).to eq(4) + end + + it 'guard last_position falls back to latest_position when no results' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'nonexistent' + ) + _results, guard = store.read([cond]) + expect(guard.last_position).to eq(store.latest_position) + end + + it 'guard last_position falls back to from_position when no results and from_position given' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + key_name: 'device_id', + key_value: 'nonexistent' + ) + _results, guard = store.read([cond], from_position: 2) + expect(guard.last_position).to eq(2) + end end describe '#messages_since' do @@ -250,9 +373,10 @@ module CCCStoreTestMessages key_value: 'dev-1' ) - conflicts = store.messages_since([cond], pos) + conflicts, guard = store.messages_since([cond], pos) expect(conflicts.size).to eq(1) expect(conflicts.first.payload.name).to eq('Sensor A Updated') + expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) end it 'returns empty when no new messages' do @@ -266,7 +390,7 @@ module CCCStoreTestMessages key_value: 'dev-1' ) - conflicts = store.messages_since([cond], pos) + conflicts, _guard = store.messages_since([cond], pos) expect(conflicts).to be_empty end end From 87376611f825cd0dd47cddc931020234f0dbb631 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 20 Feb 2026 22:59:58 +0000 Subject: [PATCH 003/115] Add consumer group support with partition-based offset tracking Implements store-level primitives for parallel background processing of the CCC message log. Partitions are discovered via AND semantics (messages must have all partition attributes) and fetched via conditional AND (each message matches all partition attributes it has). Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 268 +++++++++++++++++++- spec/sourced/ccc/store_spec.rb | 431 +++++++++++++++++++++++++++++++++ 2 files changed, 698 insertions(+), 1 deletion(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index f2094584..5f24e166 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -20,6 +20,9 @@ def instance_of?(klass) = __getobj__.instance_of?(klass) end class Store + ACTIVE = 'active' + STOPPED = 'stopped' + attr_reader :db def initialize(db) @@ -32,7 +35,10 @@ def initialize(db) def installed? db.table_exists?(:ccc_messages) && db.table_exists?(:ccc_key_pairs) && - db.table_exists?(:ccc_message_key_pairs) + db.table_exists?(:ccc_message_key_pairs) && + db.table_exists?(:ccc_consumer_groups) && + db.table_exists?(:ccc_offsets) && + db.table_exists?(:ccc_offset_key_pairs) end def install! @@ -66,6 +72,39 @@ def install! ) SQL db.run('CREATE INDEX IF NOT EXISTS idx_ccc_mkp_key ON ccc_message_key_pairs(key_pair_id, message_position)') + + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_consumer_groups ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + group_id TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT '#{ACTIVE}', + error_context TEXT, + retry_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + SQL + + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_offsets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + consumer_group_id INTEGER NOT NULL REFERENCES ccc_consumer_groups(id) ON DELETE CASCADE, + partition_key TEXT NOT NULL, + last_position INTEGER NOT NULL DEFAULT 0, + claimed INTEGER NOT NULL DEFAULT 0, + claimed_at TEXT, + claimed_by TEXT, + UNIQUE(consumer_group_id, partition_key) + ) + SQL + + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_offset_key_pairs ( + offset_id INTEGER NOT NULL REFERENCES ccc_offsets(id) ON DELETE CASCADE, + key_pair_id INTEGER NOT NULL REFERENCES ccc_key_pairs(id), + PRIMARY KEY (offset_id, key_pair_id) + ) + SQL end # Append messages to the store. Extracts keys and indexes them. @@ -138,6 +177,104 @@ def messages_since(conditions, position) read(conditions, from_position: position) end + # Register a consumer group. Idempotent. + def register_consumer_group(group_id) + now = Time.now.iso8601 + db.run(<<~SQL) + INSERT OR IGNORE INTO ccc_consumer_groups (group_id, status, created_at, updated_at) + VALUES (#{db.literal(group_id)}, '#{ACTIVE}', #{db.literal(now)}, #{db.literal(now)}) + SQL + end + + def consumer_group_active?(group_id) + row = db[:ccc_consumer_groups].where(group_id: group_id).select(:status).first + return false unless row + + row[:status] == ACTIVE + end + + def stop_consumer_group(group_id) + db[:ccc_consumer_groups].where(group_id: group_id).update(status: STOPPED, updated_at: Time.now.iso8601) + end + + def start_consumer_group(group_id) + db[:ccc_consumer_groups].where(group_id: group_id).update(status: ACTIVE, updated_at: Time.now.iso8601) + end + + def reset_consumer_group(group_id) + cg = db[:ccc_consumer_groups].where(group_id: group_id).first + return unless cg + + db[:ccc_offsets].where(consumer_group_id: cg[:id]).delete + end + + # Claim the next available partition for processing. + # partition_by: String or Array of attribute names + # handled_types: Array of message type strings + # worker_id: String identifying the claiming worker + # Returns Hash { offset_id:, key_pair_ids:, partition_key:, partition_value:, messages: } or nil + def claim_next(group_id, partition_by:, handled_types:, worker_id:) + partition_by = Array(partition_by).sort + cg = db[:ccc_consumer_groups].where(group_id: group_id, status: ACTIVE).first + return nil unless cg + + bootstrap_offsets(cg[:id], partition_by) + + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + return nil unless claimed + + key_pair_ids = db[:ccc_offset_key_pairs] + .where(offset_id: claimed[:offset_id]) + .select_map(:key_pair_id) + + messages = fetch_partition_messages(key_pair_ids, claimed[:last_position], handled_types) + + # If no messages pass the conditional AND filter, release and return nil + if messages.empty? + release(group_id, offset_id: claimed[:offset_id]) + return nil + end + + # Build partition_value hash from key_pairs + partition_value = {} + db[:ccc_key_pairs].where(id: key_pair_ids).each do |kp| + partition_value[kp[:name]] = kp[:value] + end + + { + offset_id: claimed[:offset_id], + key_pair_ids: key_pair_ids, + partition_key: claimed[:partition_key], + partition_value: partition_value, + messages: messages + } + end + + # Acknowledge processing: advance offset and release claim. + def ack(group_id, offset_id:, position:) + cg = db[:ccc_consumer_groups].where(group_id: group_id).first + return unless cg + + db[:ccc_offsets].where(id: offset_id, consumer_group_id: cg[:id]).update( + last_position: position, + claimed: 0, + claimed_at: nil, + claimed_by: nil + ) + end + + # Release claim without advancing offset (for error recovery). + def release(group_id, offset_id:) + cg = db[:ccc_consumer_groups].where(group_id: group_id).first + return unless cg + + db[:ccc_offsets].where(id: offset_id, consumer_group_id: cg[:id]).update( + claimed: 0, + claimed_at: nil, + claimed_by: nil + ) + end + # Current max position, or 0 if the store is empty. def latest_position db[:ccc_messages].max(:position) || 0 @@ -145,6 +282,9 @@ def latest_position # Clear all tables. For testing only. def clear! + db[:ccc_offset_key_pairs].delete + db[:ccc_offsets].delete + db[:ccc_consumer_groups].delete db[:ccc_message_key_pairs].delete db[:ccc_key_pairs].delete db[:ccc_messages].delete @@ -153,6 +293,132 @@ def clear! private + # Build canonical partition key string from attribute names and values. + # Sorted by attribute name for consistency. + def build_partition_key(partition_by, values) + partition_by.sort.map { |attr| "#{attr}:#{values[attr]}" }.join('|') + end + + # Discover partition tuples via AND self-joins and create offset + key_pair rows. + # Only messages with ALL partition attributes create partition tuples. + def bootstrap_offsets(cg_id, partition_by) + # Build AND self-join query to find all unique tuples + joins = [] + selects = [] + partition_by.each_with_index do |attr, i| + joins << "JOIN ccc_message_key_pairs mkp#{i} ON m.position = mkp#{i}.message_position" + joins << "JOIN ccc_key_pairs kp#{i} ON mkp#{i}.key_pair_id = kp#{i}.id AND kp#{i}.name = #{db.literal(attr)}" + selects << "kp#{i}.id AS kp_id_#{i}, kp#{i}.value AS val_#{i}" + end + + group_by = partition_by.each_index.map { |i| "kp#{i}.id" }.join(', ') + + sql = <<~SQL + SELECT #{selects.join(', ')} + FROM ccc_messages m + #{joins.join("\n")} + GROUP BY #{group_by} + SQL + + db.fetch(sql).each do |row| + # Build the values hash and collect key_pair_ids + values = {} + kp_ids = [] + partition_by.each_with_index do |attr, i| + values[attr] = row[:"val_#{i}"] + kp_ids << row[:"kp_id_#{i}"] + end + + partition_key = build_partition_key(partition_by, values) + + # INSERT OR IGNORE the offset row + db.run(<<~SQL) + INSERT OR IGNORE INTO ccc_offsets (consumer_group_id, partition_key, last_position, claimed) + VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) + SQL + + offset_id = db[:ccc_offsets].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + + # INSERT OR IGNORE the offset_key_pairs join rows + kp_ids.each do |kp_id| + db.run(<<~SQL) + INSERT OR IGNORE INTO ccc_offset_key_pairs (offset_id, key_pair_id) + VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) + SQL + end + end + end + + # Find the next unclaimed partition with pending messages and claim it. + # Uses OR semantics for detection (any matching key_pair); exact filtering at fetch time. + def find_and_claim_partition(cg_id, handled_types, worker_id) + types_list = handled_types.map { |t| db.literal(t) }.join(', ') + + sql = <<~SQL + SELECT o.id AS offset_id, o.partition_key, o.last_position, + MIN(m.position) AS next_position + FROM ccc_offsets o + JOIN ccc_offset_key_pairs okp ON o.id = okp.offset_id + JOIN ccc_message_key_pairs mkp ON okp.key_pair_id = mkp.key_pair_id + JOIN ccc_messages m ON mkp.message_position = m.position + WHERE o.consumer_group_id = #{db.literal(cg_id)} + AND o.claimed = 0 + AND m.position > o.last_position + AND m.message_type IN (#{types_list}) + GROUP BY o.id + ORDER BY next_position ASC + LIMIT 1 + SQL + + row = db.fetch(sql).first + return nil unless row + + now = Time.now.iso8601 + updated = db[:ccc_offsets] + .where(id: row[:offset_id], claimed: 0) + .update(claimed: 1, claimed_at: now, claimed_by: worker_id) + + return nil if updated == 0 + + { offset_id: row[:offset_id], partition_key: row[:partition_key], last_position: row[:last_position] } + end + + # Fetch messages for a partition using conditional AND semantics. + # For each message: match against ALL of the partition's attributes that the message has. + def fetch_partition_messages(key_pair_ids, last_position, handled_types) + return [] if key_pair_ids.empty? + + kp_ids_list = key_pair_ids.map { |id| db.literal(id) }.join(', ') + types_list = handled_types.map { |t| db.literal(t) }.join(', ') + + sql = <<~SQL + SELECT DISTINCT m.position, m.message_id, m.message_type, m.payload, m.metadata, m.created_at + FROM ccc_messages m + WHERE m.position > #{db.literal(last_position)} + AND m.message_type IN (#{types_list}) + AND EXISTS ( + SELECT 1 FROM ccc_message_key_pairs mkp + WHERE mkp.message_position = m.position + AND mkp.key_pair_id IN (#{kp_ids_list}) + ) + AND ( + SELECT COUNT(*) FROM ccc_message_key_pairs mkp + WHERE mkp.message_position = m.position + AND mkp.key_pair_id IN (#{kp_ids_list}) + ) = ( + SELECT COUNT(DISTINCT kp_part.name) + FROM ccc_message_key_pairs mkp2 + JOIN ccc_key_pairs kp_msg ON mkp2.key_pair_id = kp_msg.id + JOIN ccc_key_pairs kp_part ON kp_part.id IN (#{kp_ids_list}) + AND kp_part.name = kp_msg.name + WHERE mkp2.message_position = m.position + ) + ORDER BY m.position ASC + SQL + + db.fetch(sql).map { |row| deserialize(row) } + end + # Core query logic shared by #read and #check_conflicts. def query_messages(conditions, from_position: nil, limit: nil) # Step 1: resolve key_pair IDs diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index dea68df3..8a298d45 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -20,6 +20,25 @@ module CCCStoreTestMessages attribute :device_id, String attribute :asset_id, String end + + # Course/user messages for composite partition tests + CourseCreated = Sourced::CCC::Message.define('store_test.course.created') do + attribute :course_name, String + end + + UserRegistered = Sourced::CCC::Message.define('store_test.user.registered') do + attribute :user_id, String + attribute :name, String + end + + UserJoinedCourse = Sourced::CCC::Message.define('store_test.user.joined_course') do + attribute :course_name, String + attribute :user_id, String + end + + CourseClosed = Sourced::CCC::Message.define('store_test.course.closed') do + attribute :course_name, String + end end RSpec.describe Sourced::CCC::Store do @@ -435,5 +454,417 @@ module CCCStoreTestMessages ) expect(pos).to eq(1) end + + it 'clears consumer groups, offsets, and offset_key_pairs' do + store.register_consumer_group('test-group') + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + store.claim_next('test-group', + partition_by: 'device_id', + handled_types: ['store_test.device.registered'], + worker_id: 'w-1') + + store.clear! + + expect(db[:ccc_consumer_groups].count).to eq(0) + expect(db[:ccc_offsets].count).to eq(0) + expect(db[:ccc_offset_key_pairs].count).to eq(0) + end + end + + describe '#register_consumer_group' do + it 'creates row with active status' do + store.register_consumer_group('my-group') + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + expect(row).not_to be_nil + expect(row[:status]).to eq('active') + expect(row[:created_at]).not_to be_nil + expect(row[:updated_at]).not_to be_nil + end + + it 'is idempotent' do + store.register_consumer_group('my-group') + expect { store.register_consumer_group('my-group') }.not_to raise_error + expect(db[:ccc_consumer_groups].where(group_id: 'my-group').count).to eq(1) + end + end + + describe '#consumer_group_active?' do + it 'returns true for active group' do + store.register_consumer_group('my-group') + expect(store.consumer_group_active?('my-group')).to be true + end + + it 'returns false for stopped group' do + store.register_consumer_group('my-group') + store.stop_consumer_group('my-group') + expect(store.consumer_group_active?('my-group')).to be false + end + + it 'returns false for nonexistent group' do + expect(store.consumer_group_active?('nope')).to be false + end + end + + describe '#stop/start_consumer_group' do + it 'toggles status' do + store.register_consumer_group('my-group') + store.stop_consumer_group('my-group') + expect(store.consumer_group_active?('my-group')).to be false + + store.start_consumer_group('my-group') + expect(store.consumer_group_active?('my-group')).to be true + end + end + + describe '#reset_consumer_group' do + it 'deletes all offsets' do + store.register_consumer_group('my-group') + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + store.claim_next('my-group', + partition_by: 'device_id', + handled_types: ['store_test.device.registered'], + worker_id: 'w-1') + + expect(db[:ccc_offsets].count).to be > 0 + store.reset_consumer_group('my-group') + expect(db[:ccc_offsets].count).to eq(0) + end + end + + describe '#claim_next (single attribute partition)' do + let(:group_id) { 'single-test' } + let(:handled_types) { ['store_test.device.registered'] } + + before do + store.register_consumer_group(group_id) + end + + it 'bootstraps offsets for new partitions' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(db[:ccc_offsets].count).to be >= 1 + end + + it 'returns nil when no pending messages' do + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result).to be_nil + end + + it 'returns messages for next unclaimed partition with correct shape' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result).not_to be_nil + expect(result[:offset_id]).to be_a(Integer) + expect(result[:key_pair_ids]).to be_a(Array) + expect(result[:partition_key]).to eq('device_id:dev-1') + expect(result[:partition_value]).to eq({ 'device_id' => 'dev-1' }) + expect(result[:messages]).to be_a(Array) + expect(result[:messages].size).to eq(1) + expect(result[:messages].first).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(result[:messages].first.position).to eq(1) + end + + it 'returns multiple pending messages for same partition' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + ]) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result[:messages].size).to eq(2) + end + + it 'skips claimed partitions — second worker gets different partition' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-2') + + expect(r1[:partition_key]).not_to eq(r2[:partition_key]) + end + + it 'returns nil when all partitions claimed' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-2') + expect(result).to be_nil + end + + it 'respects handled_types filter' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.asset.registered'], worker_id: 'w-1') + expect(result).to be_nil + end + + it 'only returns messages after last_position' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + + # Append another message for same partition + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A updated' }) + ) + + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2[:messages].size).to eq(1) + expect(r2[:messages].first.payload.name).to eq('A updated') + end + + it 'returns nil for stopped consumer group' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + store.stop_consumer_group(group_id) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result).to be_nil + end + + it 'prioritizes partition with earliest pending message' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ) + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + # dev-2 was appended first (position 1), so it should be prioritized + expect(result[:partition_value]).to eq({ 'device_id' => 'dev-2' }) + end + + it 'bootstraps newly appeared partitions on subsequent calls' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + + # New partition appears + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + ) + + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2).not_to be_nil + expect(r2[:partition_value]).to eq({ 'device_id' => 'dev-3' }) + end + end + + describe '#claim_next (composite partition — conditional AND fetch)' do + let(:group_id) { 'composite-test' } + let(:handled_types) do + [ + 'store_test.course.created', + 'store_test.user.registered', + 'store_test.user.joined_course', + 'store_test.course.closed' + ] + end + + before do + store.register_consumer_group(group_id) + end + + it 'bootstraps composite partitions (only messages with ALL attributes create partitions)' do + # CourseCreated only has course_name — not enough for a (course_name, user_id) partition + store.append( + CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }) + ) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + expect(result).to be_nil + + # UserJoinedCourse has both — NOW a partition is created + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + expect(result).not_to be_nil + expect(result[:partition_value]).to eq({ 'course_name' => 'Algebra', 'user_id' => 'joe' }) + end + + it 'fetches messages with single partition attribute matching' do + store.append([ + CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + CCCStoreTestMessages::UserRegistered.new(payload: { user_id: 'joe', name: 'Joe' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ]) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + types = result[:messages].map(&:type) + expect(types).to contain_exactly( + 'store_test.course.created', + 'store_test.user.registered', + 'store_test.user.joined_course' + ) + end + + it 'different composite partitions can be claimed in parallel' do + store.append([ + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Physics', user_id: 'jake' }) + ]) + + r1 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + r2 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-2') + + expect(r1).not_to be_nil + expect(r2).not_to be_nil + expect(r1[:partition_key]).not_to eq(r2[:partition_key]) + end + + it 'excludes messages with ALL partition attributes that do not match ALL values' do + store.append([ + CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) + ]) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + + # The first partition claimed should be one of the two — let's check its messages + if result[:partition_value]['user_id'] == 'joe' + # Should include CourseCreated (1 attr, matches) and joe's join (2 attrs, both match) + # Should NOT include jake's join (2 attrs, user_id doesn't match) + user_ids = result[:messages] + .select { |m| m.type == 'store_test.user.joined_course' } + .map { |m| m.payload.user_id } + expect(user_ids).to eq(['joe']) + else + user_ids = result[:messages] + .select { |m| m.type == 'store_test.user.joined_course' } + .map { |m| m.payload.user_id } + expect(user_ids).to eq(['jake']) + end + end + + it 'messages with ALL partition attributes matching are not duplicated' do + store.append([ + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ]) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + # The join message matches both key_pairs but should appear only once + expect(result[:messages].size).to eq(1) + end + + it 'excludes messages with partial attributes matching wrong value' do + store.append([ + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'History', user_id: 'joe' }) + ]) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + + # Whichever partition we get, the other course's join should be excluded + courses = result[:messages].map { |m| m.payload.course_name } + expect(courses.uniq.size).to eq(1) + end + end + + describe '#ack' do + let(:group_id) { 'ack-test' } + + before do + store.register_consumer_group(group_id) + end + + it 'advances offset and releases claim' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + + store.ack(group_id, offset_id: result[:offset_id], position: result[:messages].last.position) + + offset = db[:ccc_offsets].where(id: result[:offset_id]).first + expect(offset[:last_position]).to eq(result[:messages].last.position) + expect(offset[:claimed]).to eq(0) + expect(offset[:claimed_at]).to be_nil + expect(offset[:claimed_by]).to be_nil + end + + it 'after ack, subsequent claim skips processed messages' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + ]) + + r1 = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + + # No new messages — should return nil + r2 = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + expect(r2).to be_nil + end + end + + describe '#release' do + let(:group_id) { 'release-test' } + + before do + store.register_consumer_group(group_id) + end + + it 'releases claim without advancing' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + + store.release(group_id, offset_id: result[:offset_id]) + + offset = db[:ccc_offsets].where(id: result[:offset_id]).first + expect(offset[:last_position]).to eq(0) # not advanced + expect(offset[:claimed]).to eq(0) + end + + it 'after release, same partition re-claimed with same messages' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + r1 = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + store.release(group_id, offset_id: r1[:offset_id]) + + r2 = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-2') + + expect(r2[:offset_id]).to eq(r1[:offset_id]) + expect(r2[:messages].map(&:position)).to eq(r1[:messages].map(&:position)) + end end end From 8772eb6fc3e844e99a5817bbab87e9eae40689c7 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 20 Feb 2026 23:27:44 +0000 Subject: [PATCH 004/115] Return ConsistencyGuard from claim_next for optimistic concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit claim_next now builds guard conditions from each handled_type × partition key_pair combination, enabling deciders to detect concurrent writes at append time via store.append(events, guard: result[:guard]). Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 16 +++++- spec/sourced/ccc/store_spec.rb | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 5f24e166..2a761030 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -235,18 +235,30 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) return nil end - # Build partition_value hash from key_pairs + # Build partition_value hash and guard conditions from key_pairs partition_value = {} + guard_conditions = [] db[:ccc_key_pairs].where(id: key_pair_ids).each do |kp| partition_value[kp[:name]] = kp[:value] + handled_types.each do |type| + guard_conditions << QueryCondition.new( + message_type: type, + key_name: kp[:name], + key_value: kp[:value] + ) + end end + last_pos = messages.last.position + guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) + { offset_id: claimed[:offset_id], key_pair_ids: key_pair_ids, partition_key: claimed[:partition_key], partition_value: partition_value, - messages: messages + messages: messages, + guard: guard } end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 8a298d45..7c4db610 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -572,6 +572,7 @@ module CCCStoreTestMessages expect(result[:messages].size).to eq(1) expect(result[:messages].first).to be_a(CCCStoreTestMessages::DeviceRegistered) expect(result[:messages].first.position).to eq(1) + expect(result[:guard]).to be_a(Sourced::CCC::ConsistencyGuard) end it 'returns multiple pending messages for same partition' do @@ -673,6 +674,57 @@ module CCCStoreTestMessages expect(r2).not_to be_nil expect(r2[:partition_value]).to eq({ 'device_id' => 'dev-3' }) end + + it 'returns a guard with conditions for each handled_type × partition key_pair' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + guard = result[:guard] + + expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(guard.last_position).to eq(result[:messages].last.position) + + # 1 key_pair (device_id=dev-1) × 1 handled_type = 1 condition + expect(guard.conditions.size).to eq(1) + cond = guard.conditions.first + expect(cond.message_type).to eq('store_test.device.registered') + expect(cond.key_name).to eq('device_id') + expect(cond.key_value).to eq('dev-1') + end + + it 'guard can be used for optimistic concurrency on append' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + + # No concurrent writes — append with guard succeeds + new_event = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + expect { store.append(new_event, guard: result[:guard]) }.not_to raise_error + + store.ack(group_id, offset_id: result[:offset_id], position: result[:messages].last.position) + end + + it 'guard detects concurrent conflicting writes' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + + # Simulate concurrent write after claim + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Concurrent' }) + ) + + new_event = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + expect { + store.append(new_event, guard: result[:guard]) + }.to raise_error(Sourced::ConcurrentAppendError) + end end describe '#claim_next (composite partition — conditional AND fetch)' do @@ -786,6 +838,48 @@ module CCCStoreTestMessages courses = result[:messages].map { |m| m.payload.course_name } expect(courses.uniq.size).to eq(1) end + + it 'returns guard with conditions for each handled_type × partition key_pair' do + store.append([ + CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ]) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + guard = result[:guard] + + expect(guard.last_position).to eq(result[:messages].last.position) + + # 2 key_pairs × 4 handled_types = 8 conditions + expect(guard.conditions.size).to eq(8) + + # Check that conditions cover both key_pairs + key_names = guard.conditions.map(&:key_name).uniq.sort + expect(key_names).to eq(['course_name', 'user_id']) + + # Check that conditions cover all handled_types + types = guard.conditions.map(&:message_type).uniq.sort + expect(types).to eq(handled_types.sort) + end + + it 'guard detects concurrent writes in composite partition' do + store.append([ + CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ]) + + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + + # Concurrent write: course closed while decider was processing + store.append( + CCCStoreTestMessages::CourseClosed.new(payload: { course_name: 'Algebra' }) + ) + + new_event = CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + expect { + store.append(new_event, guard: result[:guard]) + }.to raise_error(Sourced::ConcurrentAppendError) + end end describe '#ack' do From ac0b674069281700cbe90fefc1d87503ba93bb75 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 20 Feb 2026 23:38:36 +0000 Subject: [PATCH 005/115] Use Message.to_conditions for type-aware guard conditions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guard conditions are now derived from each message class's declared payload attributes via Message.to_conditions(**partition_attrs). This avoids nonsensical conditions (e.g. CourseCreated × user_id) while still covering all handled_types for conflict detection. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/message.rb | 26 +++++++++++++++++++++++ lib/sourced/ccc/store.rb | 20 ++++++++++-------- spec/sourced/ccc/message_spec.rb | 32 ++++++++++++++++++++++++++++ spec/sourced/ccc/store_spec.rb | 36 +++++++++++++++++++++----------- 4 files changed, 93 insertions(+), 21 deletions(-) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 327101dd..1197867f 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -79,6 +79,32 @@ def initialize(attrs = {}) super(attrs) end + # Returns the payload attribute names for this message class. + def self.payload_attribute_names + return [] unless const_defined?(:Payload) + + self::Payload._schema.to_h.keys.map(&:to_sym) + end + + # Build QueryConditions for the intersection of this message's attributes + # and the given key-value pairs. + # Example: + # CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') + # # => [QueryCondition('course.created', 'course_name', 'Algebra')] + # # user_id is ignored because CourseCreated doesn't have it + def self.to_conditions(**attrs) + supported = payload_attribute_names + attrs.filter_map do |key, value| + next unless supported.include?(key) + + QueryCondition.new( + message_type: type, + key_name: key.to_s, + key_value: value.to_s + ) + end + end + # Auto-extract key-value pairs from all top-level payload attributes. # Skips nil values. Returns array of [name, value] pairs. def extracted_keys diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 2a761030..574fc589 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -235,20 +235,22 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) return nil end - # Build partition_value hash and guard conditions from key_pairs + # Build partition_value hash from key_pairs partition_value = {} - guard_conditions = [] db[:ccc_key_pairs].where(id: key_pair_ids).each do |kp| partition_value[kp[:name]] = kp[:value] - handled_types.each do |type| - guard_conditions << QueryCondition.new( - message_type: type, - key_name: kp[:name], - key_value: kp[:value] - ) - end end + # Build guard conditions from handled_types. + # Each class's to_conditions only generates conditions for attributes it actually has. + # We use handled_types (not just fetched messages) so the guard also covers + # message types that haven't appeared yet but would be conflicts. + partition_attrs = partition_value.transform_keys(&:to_sym) + guard_conditions = handled_types.filter_map do |type| + klass = Message.registry[type] + klass&.to_conditions(**partition_attrs) + end.flatten + last_pos = messages.last.position guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index fd0cd9b0..2ab5e32f 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -122,6 +122,38 @@ module CCCTestMessages end end + describe '.payload_attribute_names' do + it 'returns attribute names for a defined message class' do + expect(CCCTestMessages::DeviceRegistered.payload_attribute_names).to eq([:device_id, :name]) + end + + it 'returns empty array for a bare message class' do + bare = Sourced::CCC::Message.define('test.payload_attrs.bare') + expect(bare.payload_attribute_names).to eq([]) + end + end + + describe '.to_conditions' do + it 'returns conditions only for attributes the message class has' do + conditions = CCCTestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', asset_id: 'asset-1') + expect(conditions.size).to eq(1) + expect(conditions.first.message_type).to eq('device.registered') + expect(conditions.first.key_name).to eq('device_id') + expect(conditions.first.key_value).to eq('dev-1') + end + + it 'returns conditions for all matching attributes' do + conditions = CCCTestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', name: 'Sensor A') + expect(conditions.size).to eq(2) + expect(conditions.map(&:key_name).sort).to eq(['device_id', 'name']) + end + + it 'returns empty array when no attributes match' do + conditions = CCCTestMessages::DeviceRegistered.to_conditions(course_name: 'Algebra') + expect(conditions).to eq([]) + end + end + describe Sourced::CCC::QueryCondition do it 'is a Data struct with message_type, key_name, key_value' do cond = Sourced::CCC::QueryCondition.new( diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 7c4db610..0c9f55b7 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -675,7 +675,7 @@ module CCCStoreTestMessages expect(r2[:partition_value]).to eq({ 'device_id' => 'dev-3' }) end - it 'returns a guard with conditions for each handled_type × partition key_pair' do + it 'returns a guard with conditions only for key_names each type actually has' do store.append( CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) @@ -839,7 +839,7 @@ module CCCStoreTestMessages expect(courses.uniq.size).to eq(1) end - it 'returns guard with conditions for each handled_type × partition key_pair' do + it 'returns guard with conditions only for key_names each message type actually has' do store.append([ CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) @@ -850,16 +850,28 @@ module CCCStoreTestMessages expect(guard.last_position).to eq(result[:messages].last.position) - # 2 key_pairs × 4 handled_types = 8 conditions - expect(guard.conditions.size).to eq(8) - - # Check that conditions cover both key_pairs - key_names = guard.conditions.map(&:key_name).uniq.sort - expect(key_names).to eq(['course_name', 'user_id']) - - # Check that conditions cover all handled_types - types = guard.conditions.map(&:message_type).uniq.sort - expect(types).to eq(handled_types.sort) + # Expected conditions (derived from message class definitions, not store data): + # CourseCreated has course_name only → 1 condition + # UserRegistered has user_id (+ name, not a partition attr) → 1 condition + # CourseClosed has course_name only → 1 condition + # UserJoinedCourse has course_name + user_id → 2 conditions + # Total: 5 conditions + expect(guard.conditions.size).to eq(5) + + # CourseCreated should NOT have a user_id condition + course_created_conditions = guard.conditions.select { |c| c.message_type == 'store_test.course.created' } + expect(course_created_conditions.size).to eq(1) + expect(course_created_conditions.first.key_name).to eq('course_name') + + # UserRegistered should NOT have a course_name condition + user_registered_conditions = guard.conditions.select { |c| c.message_type == 'store_test.user.registered' } + expect(user_registered_conditions.size).to eq(1) + expect(user_registered_conditions.first.key_name).to eq('user_id') + + # UserJoinedCourse has both + joined_conditions = guard.conditions.select { |c| c.message_type == 'store_test.user.joined_course' } + expect(joined_conditions.size).to eq(2) + expect(joined_conditions.map(&:key_name).sort).to eq(['course_name', 'user_id']) end it 'guard detects concurrent writes in composite partition' do From a77102b50e3a3a2541b61405b906a0a6394f28c1 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 20 Feb 2026 23:44:33 +0000 Subject: [PATCH 006/115] Cache payload_attribute_names at define time Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/message.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 1197867f..035a7dc6 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -8,6 +8,8 @@ module CCC ConsistencyGuard = Data.define(:conditions, :last_position) class Message < Types::Data + EMPTY_ARRAY = [].freeze + attribute :id, Types::AutoUUID attribute :type, Types::String.present attribute :created_at, Types::Forms::Time.default { Time.now } @@ -61,8 +63,11 @@ def self.node_name = :data attribute :type, Types::Static[type_str] if block_given? - const_set(:Payload, Class.new(Payload, &payload_block)) - attribute :payload, self::Payload + payload_class = Class.new(Payload, &payload_block) + const_set(:Payload, payload_class) + attribute :payload, payload_class + names = payload_class._schema.to_h.keys.map(&:to_sym).freeze + define_singleton_method(:payload_attribute_names) { names } end end end @@ -80,11 +85,8 @@ def initialize(attrs = {}) end # Returns the payload attribute names for this message class. - def self.payload_attribute_names - return [] unless const_defined?(:Payload) - - self::Payload._schema.to_h.keys.map(&:to_sym) - end + # Subclasses created via .define override this with a cached frozen array. + def self.payload_attribute_names = EMPTY_ARRAY # Build QueryConditions for the intersection of this message's attributes # and the given key-value pairs. From 6581cbcbd310968ac7259368c6fa1dedab797e75 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 00:04:55 +0000 Subject: [PATCH 007/115] Update partition plan with implemented semantics Documents conditional AND fetch, ConsistencyGuard from claim_next, Message.to_conditions, cached payload_attribute_names, and the SQLite DISTINCT requirement. Co-Authored-By: Claude Opus 4.6 --- plans/ccc/partitions.md | 379 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 plans/ccc/partitions.md diff --git a/plans/ccc/partitions.md b/plans/ccc/partitions.md new file mode 100644 index 00000000..66d99b60 --- /dev/null +++ b/plans/ccc/partitions.md @@ -0,0 +1,379 @@ +# CCC: Consumer Group Support (Store-Level Primitives) + +## Context + +CCC has a flat, globally-ordered message log with automatic key-pair indexing. To process messages in parallel in the background (like Sourced's workers/reactors), we need per-consumer-group, per-partition offset tracking. + +A **partition** is defined by one or more attribute names (e.g., `[:course_name, :user_id]`). Each unique combination of those attribute values forms a partition instance. The key CCC insight: + +- **Partition identity** (AND): a partition `(course_name=Algebra, user_id=joe)` is discovered from messages that have ALL those attributes (e.g. `UserJoinedCourse`). Messages with fewer attributes (e.g. `CourseCreated` with only `course_name`) do not create partitions — they are included when *fetching* for a partition. +- **Message fetch** (conditional AND): for a given partition, each message is matched against the *intersection* of the partition's attributes and the message's own declared payload attributes. See "Fetch Semantics" below. +- **ConsistencyGuard**: `claim_next` returns a guard for optimistic concurrency, enabling deciders to detect concurrent writes at append time. + +Store-level primitives only — consumer DSL and worker/dispatcher come later. + +## Fetch Semantics: Conditional AND + +Pure OR fetching ("return messages where `course_name=Algebra` OR `user_id=joe`") is **incorrect** because it pulls in unrelated messages. + +**Example of the problem with pure OR:** + +Given partition `(course_name=Algebra, user_id=joe)`: +- `CourseCreated(course_name: Algebra)` — only has `course_name` → match on `course_name=Algebra` → **correct** +- `UserJoined(user_id: joe, course_name: Algebra)` — has both → must match BOTH → **correct** +- `UserJoined(user_id: jake, course_name: Algebra)` — has both → must match BOTH → `user_id=jake ≠ joe` → **excluded** (but pure OR would include it!) +- `UserJoined(user_id: joe, course_name: History)` — has both → must match BOTH → `course_name=History ≠ Algebra` → **excluded** (but pure OR would include it!) + +**The correct rule:** + +> For each message, match against ALL of the partition's attributes that the message has. If a message has 1 of 2 partition attributes, match on that 1. If it has 2 of 2, match on both. + +In SQL terms: for each message, count how many of the partition's key_pairs it matches, count how many of the partition's attribute *names* appear in the message's key_pairs (regardless of value), and include the message only if those counts are equal. + +**Implementation approach:** + +Each partition has N key_pairs (e.g., `course_name=Algebra` and `user_id=joe`). Each key_pair has an attribute name. For a candidate message: + +1. `matched_count` = how many of the partition's key_pairs this message is joined to (via `ccc_message_key_pairs`) +2. `relevant_count` = how many of the partition's attribute *names* this message has ANY value for + +Include the message iff `matched_count = relevant_count` (i.e., for every partition attribute the message knows about, it has the right value). + +**SQL for fetch:** + +```sql +SELECT DISTINCT m.position, m.message_id, m.message_type, m.payload, m.metadata, m.created_at +FROM ccc_messages m +WHERE m.position > :last_position + AND m.message_type IN (:handled_types) + AND EXISTS ( + SELECT 1 FROM ccc_message_key_pairs mkp + WHERE mkp.message_position = m.position + AND mkp.key_pair_id IN (:partition_key_pair_ids) + ) + AND ( + SELECT COUNT(*) FROM ccc_message_key_pairs mkp + WHERE mkp.message_position = m.position + AND mkp.key_pair_id IN (:partition_key_pair_ids) + ) = ( + SELECT COUNT(DISTINCT kp_part.name) + FROM ccc_message_key_pairs mkp2 + JOIN ccc_key_pairs kp_msg ON mkp2.key_pair_id = kp_msg.id + JOIN ccc_key_pairs kp_part ON kp_part.id IN (:partition_key_pair_ids) + AND kp_part.name = kp_msg.name + WHERE mkp2.message_position = m.position + ) +ORDER BY m.position ASC +``` + +Note: DISTINCT is required due to a SQLite optimizer behavior where EXISTS with `IN (...)` can produce duplicate rows via index semi-join optimization. + +**Walk-through:** + +| Message | Partition attrs present | Matched key_pairs | Left=Right? | Included? | +|---------|------------------------|-------------------|-------------|-----------| +| `CourseCreated(course_name=Algebra)` | course_name (1) | course_name=Algebra (1) | 1=1 | Yes | +| `UserJoined(user_id=joe, course_name=Algebra)` | course_name, user_id (2) | both (2) | 2=2 | Yes | +| `UserJoined(user_id=jake, course_name=Algebra)` | course_name, user_id (2) | course_name=Algebra only (1) | 1!=2 | No | +| `CourseClosed(course_name=Algebra)` | course_name (1) | course_name=Algebra (1) | 1=1 | Yes | +| `UserRegistered(user_id=joe)` | user_id (1) | user_id=joe (1) | 1=1 | Yes | +| `UserJoined(user_id=joe, course_name=History)` | course_name, user_id (2) | user_id=joe only (1) | 1!=2 | No | + +## ConsistencyGuard from claim_next + +`claim_next` returns a `ConsistencyGuard` alongside the messages. This enables deciders to do optimistic concurrency checks at append time: "fail if any new messages matching these conditions appeared since I read." + +**Guard conditions are built from `Message.to_conditions`** — each message class knows its own declared payload attributes and only generates conditions for attributes it actually has. This avoids nonsensical conditions (e.g. `CourseCreated × user_id`) while still covering all `handled_types` for conflict detection. + +```ruby +# Message class API +CourseCreated.payload_attribute_names # => [:course_name] +CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') +# => [QueryCondition('course.created', 'course_name', 'Algebra')] +# user_id is ignored — CourseCreated doesn't have it + +UserJoinedCourse.to_conditions(course_name: 'Algebra', user_id: 'joe') +# => [QueryCondition('user.joined_course', 'course_name', 'Algebra'), +# QueryCondition('user.joined_course', 'user_id', 'joe')] +``` + +`payload_attribute_names` is cached at define time (frozen array stored on the class). + +Guard conditions are built from `handled_types` (not just the fetched messages) so the guard covers message types that haven't appeared yet but would be conflicts (e.g. `CourseClosed`). + +## Schema + +Add three tables to `install!`: + +```sql +CREATE TABLE IF NOT EXISTS ccc_consumer_groups ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + group_id TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'active', + error_context TEXT, + retry_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS ccc_offsets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + consumer_group_id INTEGER NOT NULL REFERENCES ccc_consumer_groups(id) ON DELETE CASCADE, + partition_key TEXT NOT NULL, + last_position INTEGER NOT NULL DEFAULT 0, + claimed INTEGER NOT NULL DEFAULT 0, + claimed_at TEXT, + claimed_by TEXT, + UNIQUE(consumer_group_id, partition_key) +); + +-- Maps each offset to its constituent key_pairs (supports composite partitions) +CREATE TABLE IF NOT EXISTS ccc_offset_key_pairs ( + offset_id INTEGER NOT NULL REFERENCES ccc_offsets(id) ON DELETE CASCADE, + key_pair_id INTEGER NOT NULL REFERENCES ccc_key_pairs(id), + PRIMARY KEY (offset_id, key_pair_id) +); +``` + +- `partition_key` TEXT: canonical serialized string for uniqueness (e.g. `"course_name:Algebra|user_id:joe"`, sorted by attribute name) +- `ccc_offset_key_pairs`: join table mapping offsets to their key_pairs — enables message queries via SQL joins +- Single-attribute partitions: one row per offset in join table. Composite: N rows. + +## API + +```ruby +# Lifecycle +store.register_consumer_group(group_id) +store.consumer_group_active?(group_id) +store.stop_consumer_group(group_id) +store.start_consumer_group(group_id) +store.reset_consumer_group(group_id) # deletes all offsets + +# Core processing +store.claim_next(group_id, partition_by:, handled_types:, worker_id:) +# partition_by: String or Array — e.g. 'device_id' or ['course_name', 'user_id'] +# → { offset_id:, key_pair_ids:, partition_key:, partition_value:, messages: [...], guard: } or nil + +store.ack(group_id, offset_id:, position:) +store.release(group_id, offset_id:) +``` + +`partition_by` and `handled_types` are passed each call (not persisted), matching Sourced's pattern. + +`partition_value` is a Hash: `{ 'course_name' => 'Algebra', 'user_id' => 'joe' }`. + +`guard` is a `ConsistencyGuard` ready to pass to `store.append(events, guard:)`. + +## Claim Flow + +### 1. Bootstrap offsets + +Discover unique partition tuples via AND self-joins (one per partition attribute) — messages must have ALL attributes to define a partition: + +```sql +-- For partition_by: ['course_name', 'user_id'] +-- Finds tuples like (Algebra, joe), (Physics, jake), etc. +SELECT kp0.id AS kp_id_0, kp0.value AS val_0, + kp1.id AS kp_id_1, kp1.value AS val_1 +FROM ccc_messages m +JOIN ccc_message_key_pairs mkp0 ON m.position = mkp0.message_position +JOIN ccc_key_pairs kp0 ON mkp0.key_pair_id = kp0.id AND kp0.name = 'course_name' +JOIN ccc_message_key_pairs mkp1 ON m.position = mkp1.message_position +JOIN ccc_key_pairs kp1 ON mkp1.key_pair_id = kp1.id AND kp1.name = 'user_id' +GROUP BY kp0.id, kp1.id +``` + +No type filter — partitions are discovered from any message with all the attributes. For each tuple: INSERT OR IGNORE into `ccc_offsets` + `ccc_offset_key_pairs`. + +### 2. Find and claim (inside brief transaction) + +Find the next unclaimed partition with pending messages. Uses OR semantics for detection (any matching key_pair has messages beyond `last_position`). False positives are harmless — the partition is claimed, fetched, gets no results after conditional AND filtering, and is released. + +```sql +SELECT o.id AS offset_id, o.partition_key, o.last_position, + MIN(m.position) AS next_position +FROM ccc_offsets o +JOIN ccc_offset_key_pairs okp ON o.id = okp.offset_id +JOIN ccc_message_key_pairs mkp ON okp.key_pair_id = mkp.key_pair_id +JOIN ccc_messages m ON mkp.message_position = m.position +WHERE o.consumer_group_id = :cg_id + AND o.claimed = 0 + AND m.position > o.last_position + AND m.message_type IN (:handled_types) +GROUP BY o.id +ORDER BY next_position ASC +LIMIT 1 +``` + +Then claim: `UPDATE ccc_offsets SET claimed=1, claimed_at=now, claimed_by=worker_id WHERE id=? AND claimed=0` + +### 3. Fetch messages (outside transaction, read-only) + +**Conditional AND semantics** — see "Fetch Semantics" section above for the full SQL and walk-through. + +### 4. Build ConsistencyGuard + +For each `handled_type`, look up the message class via `Message.registry` and call `klass.to_conditions(**partition_attrs)`. This only creates conditions for attributes the class actually declares. `last_position` is the position of the last fetched message. + +### 5. Ack / Release + +```sql +-- ack: advance offset + release claim +UPDATE ccc_offsets SET last_position=?, claimed=0, claimed_at=NULL, claimed_by=NULL + WHERE consumer_group_id=? AND id=? + +-- release: release claim without advancing (for error recovery) +UPDATE ccc_offsets SET claimed=0, claimed_at=NULL, claimed_by=NULL + WHERE consumer_group_id=? AND id=? +``` + +## Message Class API + +```ruby +# Defined on CCC::Message subclasses: + +# Returns frozen array of payload attribute names (cached at define time) +CourseCreated.payload_attribute_names # => [:course_name] + +# Builds QueryConditions for attributes this class actually has +CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') +# => [QueryCondition('course.created', 'course_name', 'Algebra')] +``` + +## Private Helper Methods + +| Method | Purpose | +|--------|---------| +| `bootstrap_offsets(cg_id, partition_by)` | AND self-joins to discover tuples, create offset + key_pair rows | +| `find_and_claim_partition(cg_id, handled_types, worker_id)` | OR-join to find unclaimed offset with pending messages, claim it | +| `fetch_partition_messages(key_pair_ids, last_position, handled_types)` | Conditional AND query, return PositionedMessages | +| `build_partition_key(partition_by, values)` | Build canonical string: `"attr1:v1\|attr2:v2"` (sorted) | + +## Files Modified + +| File | Change | +|------|--------| +| `lib/sourced/ccc/message.rb` | Add `EMPTY_ARRAY`, `payload_attribute_names` (cached at define time), `to_conditions` | +| `lib/sourced/ccc/store.rb` | Add 3 tables to `install!`, update `installed?` + `clear!`, add `ACTIVE`/`STOPPED` constants, add all lifecycle + claim/ack/release methods with guard | +| `spec/sourced/ccc/message_spec.rb` | Tests for `payload_attribute_names` and `to_conditions` | +| `spec/sourced/ccc/store_spec.rb` | Tests for all new methods | + +## Tests + +### Consumer Group Lifecycle +1. `register_consumer_group` creates row with active status +2. `register_consumer_group` is idempotent +3. `consumer_group_active?` returns true/false for active/stopped/nonexistent +4. `stop/start_consumer_group` toggle status +5. `reset_consumer_group` deletes all offsets + +### claim_next (single attribute partition) +6. Bootstraps offsets for new partitions +7. Returns nil when no pending messages +8. Returns messages for next unclaimed partition with correct shape (including guard) +9. Returns multiple pending messages for same partition +10. Skips claimed partitions — second worker gets different partition +11. Returns nil when all partitions claimed +12. Respects handled_types filter +13. Only returns messages after last_position +14. Returns nil for stopped consumer group +15. Prioritizes partition with earliest pending message +16. Bootstraps newly appeared partitions on subsequent calls +17. Returns guard with conditions only for key_names each type actually has +18. Guard can be used for optimistic concurrency on append +19. Guard detects concurrent conflicting writes + +### claim_next (composite partition — conditional AND fetch semantics) +20. Bootstraps composite partitions (only messages with ALL attributes create partitions) +21. Fetch returns messages with single partition attribute (e.g., `CourseCreated(course_name)` for partition `(course_name, user_id)`) +22. Different composite partitions can be claimed in parallel +23. Messages with ALL partition attributes must match ALL values — `UserJoined(user_id=jake, course_name=Algebra)` excluded from partition `(course_name=Algebra, user_id=joe)` +24. Messages with ALL partition attributes matching are not duplicated in results +25. Messages with partial attributes matching wrong value are excluded — `UserJoined(user_id=joe, course_name=History)` excluded from partition `(course_name=Algebra, user_id=joe)` +26. Guard conditions only for key_names each message type actually has +27. Guard detects concurrent writes in composite partition + +### ack / release +28. ack advances offset and releases claim +29. After ack, subsequent claim skips processed messages +30. release releases claim without advancing +31. After release, same partition re-claimed with same messages + +### clear! +32. Clears consumer groups, offsets, and offset_key_pairs + +### Message class methods +33. `payload_attribute_names` returns attribute names for defined message class +34. `payload_attribute_names` returns empty array for bare message class +35. `to_conditions` returns conditions only for attributes the message class has +36. `to_conditions` returns conditions for all matching attributes +37. `to_conditions` returns empty array when no attributes match + +## Usage Example + +```ruby +# Message definitions +CourseCreated = CCC::Message.define('course.created') do + attribute :course_name, String +end + +UserRegistered = CCC::Message.define('user.registered') do + attribute :user_id, String + attribute :name, String +end + +UserJoinedCourse = CCC::Message.define('user.joined_course') do + attribute :course_name, String + attribute :user_id, String +end + +CourseClosed = CCC::Message.define('course.closed') do + attribute :course_name, String +end + +# Register consumer group +store.register_consumer_group('enrollment-decider') + +# Append events +store.append(CourseCreated.new(payload: { course_name: 'Algebra' })) +store.append(UserRegistered.new(payload: { user_id: 'joe', name: 'Joe' })) +store.append(UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' })) +store.append(UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' })) + +# Claim partition (Algebra, joe) — discovered from UserJoinedCourse +result = store.claim_next('enrollment-decider', + partition_by: ['course_name', 'user_id'], + handled_types: ['course.created', 'user.registered', 'user.joined_course', 'course.closed'], + worker_id: 'w-1') + +result[:messages] +# => [CourseCreated(course_name: Algebra), <- 1 attr, matches +# UserRegistered(user_id: joe), <- 1 attr, matches +# UserJoinedCourse(course_name: Algebra, user_id: joe)] <- 2 attrs, both match +# NOT: UserJoinedCourse(course_name: Algebra, user_id: jake) <- 2 attrs, user_id != joe + +result[:guard].conditions +# => [QueryCondition('course.created', 'course_name', 'Algebra'), <- CourseCreated has course_name +# QueryCondition('user.registered', 'user_id', 'joe'), <- UserRegistered has user_id (not course_name) +# QueryCondition('user.joined_course', 'course_name', 'Algebra'), <- UserJoinedCourse has both +# QueryCondition('user.joined_course', 'user_id', 'joe'), +# QueryCondition('course.closed', 'course_name', 'Algebra')] <- CourseClosed has course_name + +# Decider processes messages, produces events +events = decider.decide(result[:messages]) + +# Append with guard — raises ConcurrentAppendError if conflicts +store.append(events, guard: result[:guard]) + +# Ack on success +store.ack('enrollment-decider', + offset_id: result[:offset_id], + position: result[:messages].last.position) +``` + +## Verification + +```bash +bundle exec rspec spec/sourced/ccc/ +# 88 examples, 0 failures +``` From 0342f2c53acd2dc0f8bf97267c50213132f1c552 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 00:11:28 +0000 Subject: [PATCH 008/115] Add YARD comments to CCC::Store methods Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 142 +++++++++++++++++++++++++++++++++------ 1 file changed, 121 insertions(+), 21 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 574fc589..c2965c7f 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -19,12 +19,18 @@ def kind_of?(klass) = is_a?(klass) def instance_of?(klass) = __getobj__.instance_of?(klass) end + # SQLite-backed store for CCC's flat, globally-ordered message log. + # Provides message storage with automatic key-pair indexing, + # consumer group management, and partition-based offset tracking + # for parallel background processing. class Store ACTIVE = 'active' STOPPED = 'stopped' + # @return [Sequel::SQLite::Database] attr_reader :db + # @param db [Sequel::SQLite::Database] a Sequel SQLite connection def initialize(db) @db = db @db.run('PRAGMA foreign_keys = ON') @@ -32,6 +38,8 @@ def initialize(db) @db.run('PRAGMA busy_timeout = 5000') end + # Whether all required tables exist. + # @return [Boolean] def installed? db.table_exists?(:ccc_messages) && db.table_exists?(:ccc_key_pairs) && @@ -41,6 +49,8 @@ def installed? db.table_exists?(:ccc_offset_key_pairs) end + # Create all required tables and indexes. Idempotent. + # @return [void] def install! db.run(<<~SQL) CREATE TABLE IF NOT EXISTS ccc_messages ( @@ -107,11 +117,16 @@ def install! SQL end - # Append messages to the store. Extracts keys and indexes them. - # When a ConsistencyGuard is provided via `guard:`, checks for conflicts - # before inserting. Raises Sourced::ConcurrentAppendError if conflicting - # messages have been appended since the guard's position. - # Returns the last assigned position. + # Append messages to the store. Extracts and indexes key-value pairs + # from each message's payload automatically. + # + # When a {ConsistencyGuard} is provided, checks for conflicting messages + # before inserting (optimistic concurrency). + # + # @param messages [CCC::Message, Array] one or more messages to append + # @param guard [ConsistencyGuard, nil] optional guard for conflict detection + # @return [Integer] the last assigned position + # @raise [Sourced::ConcurrentAppendError] if conflicting messages found after guard position def append(messages, guard: nil) messages = Array(messages) return latest_position if messages.empty? @@ -154,9 +169,13 @@ def append(messages, guard: nil) last_position end - # Query messages by conditions (array of QueryCondition). - # Each condition matches (message_type AND key_name/key_value). - # Conditions are OR'd together. + # Query messages by conditions. Each condition matches on + # (message_type AND key_name AND key_value). Multiple conditions are OR'd. + # + # @param conditions [QueryCondition, Array] query conditions + # @param from_position [Integer, nil] only return messages after this position + # @param limit [Integer, nil] max number of messages to return + # @return [Array(Array, ConsistencyGuard)] messages and a guard def read(conditions, from_position: nil, limit: nil) conditions = Array(conditions) if conditions.empty? @@ -172,12 +191,18 @@ def read(conditions, from_position: nil, limit: nil) # Conflict detection: returns messages matching conditions that appeared # after the given position. Empty array means no conflicts. - # Returns [messages, guard] like #read. + # + # @param conditions [Array] conditions to check + # @param position [Integer] check for messages after this position + # @return [Array(Array, ConsistencyGuard)] def messages_since(conditions, position) read(conditions, from_position: position) end # Register a consumer group. Idempotent. + # + # @param group_id [String] unique identifier for the consumer group + # @return [void] def register_consumer_group(group_id) now = Time.now.iso8601 db.run(<<~SQL) @@ -186,6 +211,10 @@ def register_consumer_group(group_id) SQL end + # Whether the consumer group exists and is active. + # + # @param group_id [String] + # @return [Boolean] def consumer_group_active?(group_id) row = db[:ccc_consumer_groups].where(group_id: group_id).select(:status).first return false unless row @@ -193,14 +222,26 @@ def consumer_group_active?(group_id) row[:status] == ACTIVE end + # Stop a consumer group. Stopped groups are skipped by {#claim_next}. + # + # @param group_id [String] + # @return [void] def stop_consumer_group(group_id) db[:ccc_consumer_groups].where(group_id: group_id).update(status: STOPPED, updated_at: Time.now.iso8601) end + # Re-activate a stopped consumer group. + # + # @param group_id [String] + # @return [void] def start_consumer_group(group_id) db[:ccc_consumer_groups].where(group_id: group_id).update(status: ACTIVE, updated_at: Time.now.iso8601) end + # Delete all offsets for a consumer group, resetting it to process from the beginning. + # + # @param group_id [String] + # @return [void] def reset_consumer_group(group_id) cg = db[:ccc_consumer_groups].where(group_id: group_id).first return unless cg @@ -209,10 +250,19 @@ def reset_consumer_group(group_id) end # Claim the next available partition for processing. - # partition_by: String or Array of attribute names - # handled_types: Array of message type strings - # worker_id: String identifying the claiming worker - # Returns Hash { offset_id:, key_pair_ids:, partition_key:, partition_value:, messages: } or nil + # + # Bootstraps partition offsets (discovering new partitions from messages with + # ALL +partition_by+ attributes), finds the unclaimed partition with the earliest + # pending message, claims it, and fetches messages using conditional AND semantics. + # + # Returns a {ConsistencyGuard} alongside the messages, built from each handled + # message class's declared payload attributes via {Message.to_conditions}. + # + # @param group_id [String] consumer group identifier + # @param partition_by [String, Array] attribute name(s) defining partitions + # @param handled_types [Array] message type strings this consumer handles + # @param worker_id [String] identifier for the claiming worker + # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, guard: }+ or nil def claim_next(group_id, partition_by:, handled_types:, worker_id:) partition_by = Array(partition_by).sort cg = db[:ccc_consumer_groups].where(group_id: group_id, status: ACTIVE).first @@ -264,7 +314,12 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) } end - # Acknowledge processing: advance offset and release claim. + # Acknowledge processing: advance the offset to +position+ and release the claim. + # + # @param group_id [String] consumer group identifier + # @param offset_id [Integer] offset ID from the claim result + # @param position [Integer] position of the last processed message + # @return [void] def ack(group_id, offset_id:, position:) cg = db[:ccc_consumer_groups].where(group_id: group_id).first return unless cg @@ -277,7 +332,12 @@ def ack(group_id, offset_id:, position:) ) end - # Release claim without advancing offset (for error recovery). + # Release a claim without advancing the offset. Use for error recovery + # so the partition can be re-claimed and retried. + # + # @param group_id [String] consumer group identifier + # @param offset_id [Integer] offset ID from the claim result + # @return [void] def release(group_id, offset_id:) cg = db[:ccc_consumer_groups].where(group_id: group_id).first return unless cg @@ -289,12 +349,16 @@ def release(group_id, offset_id:) ) end - # Current max position, or 0 if the store is empty. + # Current max position in the message log. + # + # @return [Integer] max position, or 0 if the store is empty def latest_position db[:ccc_messages].max(:position) || 0 end - # Clear all tables. For testing only. + # Delete all data from all tables and reset autoincrement. For testing only. + # + # @return [void] def clear! db[:ccc_offset_key_pairs].delete db[:ccc_offsets].delete @@ -308,13 +372,21 @@ def clear! private # Build canonical partition key string from attribute names and values. - # Sorted by attribute name for consistency. + # Sorted by attribute name for deterministic uniqueness. + # + # @param partition_by [Array] attribute names + # @param values [Hash{String => String}] attribute values keyed by name + # @return [String] e.g. "course_name:Algebra|user_id:joe" def build_partition_key(partition_by, values) partition_by.sort.map { |attr| "#{attr}:#{values[attr]}" }.join('|') end # Discover partition tuples via AND self-joins and create offset + key_pair rows. # Only messages with ALL partition attributes create partition tuples. + # + # @param cg_id [Integer] consumer group internal ID + # @param partition_by [Array] sorted attribute names + # @return [void] def bootstrap_offsets(cg_id, partition_by) # Build AND self-join query to find all unique tuples joins = [] @@ -364,7 +436,13 @@ def bootstrap_offsets(cg_id, partition_by) end # Find the next unclaimed partition with pending messages and claim it. - # Uses OR semantics for detection (any matching key_pair); exact filtering at fetch time. + # Uses OR semantics for detection (any matching key_pair is sufficient); + # exact conditional AND filtering happens at fetch time. + # + # @param cg_id [Integer] consumer group internal ID + # @param handled_types [Array] message type strings + # @param worker_id [String] claiming worker identifier + # @return [Hash, nil] +{ offset_id:, partition_key:, last_position: }+ or nil def find_and_claim_partition(cg_id, handled_types, worker_id) types_list = handled_types.map { |t| db.literal(t) }.join(', ') @@ -398,7 +476,14 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) end # Fetch messages for a partition using conditional AND semantics. - # For each message: match against ALL of the partition's attributes that the message has. + # For each candidate message, it must match ALL of the partition's attributes + # that the message itself has. Messages with a single partition attribute match + # on that one; messages with multiple must match all of them. + # + # @param key_pair_ids [Array] partition key_pair IDs + # @param last_position [Integer] fetch messages after this position + # @param handled_types [Array] message type strings + # @return [Array] def fetch_partition_messages(key_pair_ids, last_position, handled_types) return [] if key_pair_ids.empty? @@ -433,7 +518,13 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types) db.fetch(sql).map { |row| deserialize(row) } end - # Core query logic shared by #read and #check_conflicts. + # Core query logic shared by {#read} and {#check_conflicts}. + # Resolves key_pair IDs from conditions, then queries messages via OR'd clauses. + # + # @param conditions [Array] + # @param from_position [Integer, nil] + # @param limit [Integer, nil] + # @return [Array] def query_messages(conditions, from_position: nil, limit: nil) # Step 1: resolve key_pair IDs key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq @@ -468,12 +559,21 @@ def query_messages(conditions, from_position: nil, limit: nil) end # Check for conflicting messages after a given position. + # + # @param conditions [Array] + # @param after_position [Integer] + # @return [Array] def check_conflicts(conditions, after_position) return [] if conditions.empty? query_messages(conditions, from_position: after_position) end + # Deserialize a database row into a {PositionedMessage}. + # Looks up the message class from the registry; falls back to base {Message}. + # + # @param row [Hash] database row with :position, :message_id, :message_type, :payload, :metadata, :created_at + # @return [PositionedMessage] def deserialize(row) payload = JSON.parse(row[:payload], symbolize_names: true) metadata = row[:metadata] ? JSON.parse(row[:metadata], symbolize_names: true) : {} From bd3be9fc6b60d5e5b6193412f52a5e31049cd5d1 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 00:12:42 +0000 Subject: [PATCH 009/115] Add YARD comments to CCC::Message, QueryCondition, ConsistencyGuard Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/message.rb | 77 ++++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 035a7dc6..61642ed4 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -4,9 +4,25 @@ module Sourced module CCC + # A query condition for reading messages from the store. + # Matches on (message_type AND key_name AND key_value). + # Multiple conditions are OR'd when passed to {Store#read}. QueryCondition = Data.define(:message_type, :key_name, :key_value) + + # Returned by {Store#read} and {Store#claim_next} for optimistic concurrency. + # Pass to {Store#append} via +guard:+ to detect conflicting writes. ConsistencyGuard = Data.define(:conditions, :last_position) + # Base message class for CCC's stream-less event sourcing. + # Unlike {Sourced::Message}, CCC messages have no stream_id, seq, + # or causation_id — they go into a flat, globally-ordered log. + # + # Define message types via {.define}: + # + # CourseCreated = CCC::Message.define('course.created') do + # attribute :course_name, String + # end + # class Message < Types::Data EMPTY_ARRAY = [].freeze @@ -16,19 +32,34 @@ class Message < Types::Data attribute :metadata, Types::Hash.default(Plumb::BLANK_HASH) attribute :payload, Types::Static[nil] + # Lookup table mapping type strings to message subclasses. + # Separate from {Sourced::Message}'s registry. class Registry + # @param message_class [Class] the root message class for this registry def initialize(message_class) @message_class = message_class @lookup = {} end + # @return [Array] registered type strings def keys = @lookup.keys + + # @return [Array] direct subclasses of the root message class def subclasses = message_class.subclasses + # Register a message class under a type string. + # + # @param key [String] message type string + # @param klass [Class] message subclass def []=(key, klass) @lookup[key] = klass end + # Look up a message class by type string. + # Searches this registry first, then recurses into subclass registries. + # + # @param key [String] message type string + # @return [Class, nil] def [](key) klass = lookup[key] return klass if klass @@ -45,15 +76,33 @@ def [](key) attr_reader :lookup, :message_class end + # @return [Registry] the message type registry for this class def self.registry @registry ||= Registry.new(self) end + # Base class for typed message payloads. class Payload < Types::Data + # @param key [Symbol] attribute name + # @return [Object] attribute value def [](key) = attributes[key] + + # @see Hash#fetch def fetch(...) = to_h.fetch(...) end + # Define a new message type. Registers it in the {.registry} and + # optionally defines a typed payload. + # + # @param type_str [String] unique message type identifier (e.g. 'course.created') + # @yield optional block to define payload attributes via +attribute+ DSL + # @return [Class] the new message subclass + # + # @example + # UserJoined = CCC::Message.define('user.joined') do + # attribute :course_name, String + # attribute :user_id, String + # end def self.define(type_str, &payload_block) type_str.freeze unless type_str.frozen? @@ -72,6 +121,11 @@ def self.node_name = :data end end + # Instantiate the correct message subclass from a hash with a +:type+ key. + # + # @param attrs [Hash] must include +:type+ matching a registered type string + # @return [Message] instance of the appropriate subclass + # @raise [Sourced::UnknownMessageError] if the type string is not registered def self.from(attrs) klass = registry[attrs[:type]] raise Sourced::UnknownMessageError, "Unknown message type: #{attrs[:type]}" unless klass @@ -84,16 +138,23 @@ def initialize(attrs = {}) super(attrs) end - # Returns the payload attribute names for this message class. - # Subclasses created via .define override this with a cached frozen array. + # Returns the declared payload attribute names for this message class. + # Subclasses created via {.define} override this with a cached frozen array. + # + # @return [Array] attribute names (e.g. +[:course_name, :user_id]+) def self.payload_attribute_names = EMPTY_ARRAY - # Build QueryConditions for the intersection of this message's attributes - # and the given key-value pairs. - # Example: + # Build {QueryCondition}s for the intersection of this message's declared + # attributes and the given key-value pairs. Attributes not declared on this + # message class are silently ignored. + # + # @param attrs [Hash{Symbol => String}] partition attribute values + # @return [Array] + # + # @example # CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') # # => [QueryCondition('course.created', 'course_name', 'Algebra')] - # # user_id is ignored because CourseCreated doesn't have it + # # user_id ignored — CourseCreated doesn't declare it def self.to_conditions(**attrs) supported = payload_attribute_names attrs.filter_map do |key, value| @@ -108,7 +169,9 @@ def self.to_conditions(**attrs) end # Auto-extract key-value pairs from all top-level payload attributes. - # Skips nil values. Returns array of [name, value] pairs. + # Used by {Store#append} to index messages for querying. + # + # @return [Array] pairs of [name, value], skipping nils def extracted_keys return [] unless payload From 0444e2aa266c057bd2d55a3281e9ba181c5cf6ca Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 00:21:20 +0000 Subject: [PATCH 010/115] Add causation_id, correlation_id, and #correlate to CCC::Message Enable causal chain tracing across CCC messages, matching the pattern from Sourced::Message. Both IDs default to the message's own id via Plumb's prepare_attributes hook. Store schema and serialization updated to persist and round-trip the new fields. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/message.rb | 38 +++++++++++++++++- lib/sourced/ccc/store.rb | 12 ++++-- spec/sourced/ccc/message_spec.rb | 67 ++++++++++++++++++++++++++++++++ spec/sourced/ccc/store_spec.rb | 22 +++++++++++ 4 files changed, 134 insertions(+), 5 deletions(-) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 61642ed4..489693ad 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -14,8 +14,11 @@ module CCC ConsistencyGuard = Data.define(:conditions, :last_position) # Base message class for CCC's stream-less event sourcing. - # Unlike {Sourced::Message}, CCC messages have no stream_id, seq, - # or causation_id — they go into a flat, globally-ordered log. + # Unlike {Sourced::Message}, CCC messages have no stream_id or seq + # — they go into a flat, globally-ordered log. + # + # Supports +causation_id+ and +correlation_id+ for tracing causal chains + # across messages, similar to {Sourced::Message}. # # Define message types via {.define}: # @@ -28,6 +31,8 @@ class Message < Types::Data attribute :id, Types::AutoUUID attribute :type, Types::String.present + attribute? :causation_id, Types::UUID::V4 + attribute? :correlation_id, Types::UUID::V4 attribute :created_at, Types::Forms::Time.default { Time.now } attribute :metadata, Types::Hash.default(Plumb::BLANK_HASH) attribute :payload, Types::Static[nil] @@ -138,6 +143,25 @@ def initialize(attrs = {}) super(attrs) end + # Set causation and correlation IDs on another message, establishing + # a causal link from this message to +message+. Merges metadata. + # + # @param message [Message] the message to correlate + # @return [Message] a copy of +message+ with causation/correlation set + # + # @example + # caused = source_event.correlate(SomeCommand.new(payload: { ... })) + # caused.causation_id # => source_event.id + # caused.correlation_id # => source_event.correlation_id + def correlate(message) + attrs = { + causation_id: id, + correlation_id: correlation_id, + metadata: metadata.merge(message.metadata || Plumb::BLANK_HASH) + } + message.with(attrs) + end + # Returns the declared payload attribute names for this message class. # Subclasses created via {.define} override this with a cached frozen array. # @@ -179,6 +203,16 @@ def extracted_keys [k.to_s, v.to_s] unless v.nil? } end + + private + + # Hook called by Plumb after schema parsing, when +:id+ has been resolved. + # Defaults +causation_id+ and +correlation_id+ to the message's own +id+. + def prepare_attributes(attrs) + attrs[:correlation_id] = attrs[:id] unless attrs[:correlation_id] + attrs[:causation_id] = attrs[:id] unless attrs[:causation_id] + attrs + end end end end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index c2965c7f..c572771f 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -57,6 +57,8 @@ def install! position INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL UNIQUE, message_type TEXT NOT NULL, + causation_id TEXT, + correlation_id TEXT, payload TEXT NOT NULL, metadata TEXT, created_at TEXT NOT NULL @@ -146,6 +148,8 @@ def append(messages, guard: nil) db[:ccc_messages].insert( message_id: msg.id, message_type: msg.type, + causation_id: msg.causation_id, + correlation_id: msg.correlation_id, payload: payload_json, metadata: metadata_json, created_at: msg.created_at.iso8601 @@ -491,7 +495,7 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types) types_list = handled_types.map { |t| db.literal(t) }.join(', ') sql = <<~SQL - SELECT DISTINCT m.position, m.message_id, m.message_type, m.payload, m.metadata, m.created_at + SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at FROM ccc_messages m WHERE m.position > #{db.literal(last_position)} AND m.message_type IN (#{types_list}) @@ -545,7 +549,7 @@ def query_messages(conditions, from_position: nil, limit: nil) return [] if where_parts.empty? sql = <<~SQL - SELECT DISTINCT m.position, m.message_id, m.message_type, m.payload, m.metadata, m.created_at + SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at FROM ccc_messages m JOIN ccc_message_key_pairs mkp ON m.position = mkp.message_position WHERE (#{where_parts.join(' OR ')}) @@ -572,7 +576,7 @@ def check_conflicts(conditions, after_position) # Deserialize a database row into a {PositionedMessage}. # Looks up the message class from the registry; falls back to base {Message}. # - # @param row [Hash] database row with :position, :message_id, :message_type, :payload, :metadata, :created_at + # @param row [Hash] database row with :position, :message_id, :message_type, :causation_id, :correlation_id, :payload, :metadata, :created_at # @return [PositionedMessage] def deserialize(row) payload = JSON.parse(row[:payload], symbolize_names: true) @@ -582,6 +586,8 @@ def deserialize(row) attrs = { id: row[:message_id], type: row[:message_type], + causation_id: row[:causation_id], + correlation_id: row[:correlation_id], created_at: row[:created_at], metadata: metadata, payload: payload diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index 2ab5e32f..237d08e1 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -64,6 +64,22 @@ module CCCTestMessages ) expect(msg.metadata[:user_id]).to eq(42) end + + it 'defaults causation_id and correlation_id to id' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.causation_id).to eq(msg.id) + expect(msg.correlation_id).to eq(msg.id) + end + + it 'accepts explicit causation_id and correlation_id' do + msg = CCCTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + causation_id: 'cause-1', + correlation_id: 'corr-1' + ) + expect(msg.causation_id).to eq('cause-1') + expect(msg.correlation_id).to eq('corr-1') + end end describe '.from' do @@ -154,6 +170,57 @@ module CCCTestMessages end end + describe '#correlate' do + it 'sets causation_id to source message id' do + source = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + target = CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) + + correlated = source.correlate(target) + expect(correlated.causation_id).to eq(source.id) + end + + it 'propagates correlation_id from source' do + source = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + target = CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) + + correlated = source.correlate(target) + expect(correlated.correlation_id).to eq(source.correlation_id) + end + + it 'preserves correlation_id through a chain' do + first = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + second = first.correlate(CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' })) + third = second.correlate(CCCTestMessages::SystemUpdated.new(payload: { version: 'v1' })) + + expect(third.causation_id).to eq(second.id) + expect(third.correlation_id).to eq(first.id) + end + + it 'merges metadata from both messages' do + source = CCCTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + target = CCCTestMessages::AssetRegistered.new( + payload: { asset_id: 'asset-1', label: 'Label' }, + metadata: { request_id: 'req-1' } + ) + + correlated = source.correlate(target) + expect(correlated.metadata).to eq({ user_id: 42, request_id: 'req-1' }) + end + + it 'returns a new instance without mutating the original' do + source = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + target = CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) + + correlated = source.correlate(target) + expect(correlated).not_to equal(target) + expect(correlated).to be_a(CCCTestMessages::AssetRegistered) + expect(target.causation_id).to eq(target.id) # original unchanged + end + end + describe Sourced::CCC::QueryCondition do it 'is a Data struct with message_type, key_name, key_value' do cond = Sourced::CCC::QueryCondition.new( diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 0c9f55b7..03baf116 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -134,6 +134,28 @@ module CCCStoreTestMessages expect(meta[:user_id]).to eq(42) end + it 'persists and round-trips causation_id and correlation_id' do + source = CCCStoreTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' } + ) + caused = source.correlate( + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }) + ) + store.append([source, caused]) + + cond1 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.device.registered', key_name: 'device_id', key_value: 'dev-1') + cond2 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.asset.registered', key_name: 'asset_id', key_value: 'asset-1') + messages, = store.read([cond1, cond2]) + + src = messages.find { |m| m.type == 'store_test.device.registered' } + csd = messages.find { |m| m.type == 'store_test.asset.registered' } + + expect(src.causation_id).to eq(src.id) + expect(src.correlation_id).to eq(src.id) + expect(csd.causation_id).to eq(source.id) + expect(csd.correlation_id).to eq(source.correlation_id) + end + it 'returns latest_position for empty array' do pos = store.append([]) expect(pos).to eq(0) From ff63b2aa9f23cf2d1991caddf76f75dbb66b84e2 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 00:53:33 +0000 Subject: [PATCH 011/115] Add replaying flag to claim_next based on highest acked position highest_position on the consumer group tracks the furthest position ever successfully acked (advanced in ack, never decreased). claim_next returns replaying: true when all returned messages are at or below this watermark, meaning they have been processed before (e.g. after an offset reset). First-time processing is never replaying. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 28 ++++++++++++-- spec/sourced/ccc/store_spec.rb | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index c572771f..5b7f0153 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -90,6 +90,7 @@ def install! id INTEGER PRIMARY KEY AUTOINCREMENT, group_id TEXT NOT NULL UNIQUE, status TEXT NOT NULL DEFAULT '#{ACTIVE}', + highest_position INTEGER NOT NULL DEFAULT 0, error_context TEXT, retry_at TEXT, created_at TEXT NOT NULL, @@ -210,8 +211,8 @@ def messages_since(conditions, position) def register_consumer_group(group_id) now = Time.now.iso8601 db.run(<<~SQL) - INSERT OR IGNORE INTO ccc_consumer_groups (group_id, status, created_at, updated_at) - VALUES (#{db.literal(group_id)}, '#{ACTIVE}', #{db.literal(now)}, #{db.literal(now)}) + INSERT OR IGNORE INTO ccc_consumer_groups (group_id, status, highest_position, created_at, updated_at) + VALUES (#{db.literal(group_id)}, '#{ACTIVE}', 0, #{db.literal(now)}, #{db.literal(now)}) SQL end @@ -262,11 +263,17 @@ def reset_consumer_group(group_id) # Returns a {ConsistencyGuard} alongside the messages, built from each handled # message class's declared payload attributes via {Message.to_conditions}. # + # The +replaying+ flag indicates whether the returned messages have been + # processed by this consumer group before. A message is replaying when its + # position is at or before the consumer group's +highest_position+ — the + # furthest position ever successfully acked. After a reset, re-claimed + # messages are correctly flagged as replaying. + # # @param group_id [String] consumer group identifier # @param partition_by [String, Array] attribute name(s) defining partitions # @param handled_types [Array] message type strings this consumer handles # @param worker_id [String] identifier for the claiming worker - # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, guard: }+ or nil + # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, replaying:, guard: }+ or nil def claim_next(group_id, partition_by:, handled_types:, worker_id:) partition_by = Array(partition_by).sort cg = db[:ccc_consumer_groups].where(group_id: group_id, status: ACTIVE).first @@ -308,17 +315,24 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) last_pos = messages.last.position guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) + # replaying: true when all messages are at or below the highest position + # ever acked by this consumer group (i.e. they've been processed before). + replaying = messages.last.position <= cg[:highest_position] + { offset_id: claimed[:offset_id], key_pair_ids: key_pair_ids, partition_key: claimed[:partition_key], partition_value: partition_value, messages: messages, + replaying: replaying, guard: guard } end # Acknowledge processing: advance the offset to +position+ and release the claim. + # Also advances the consumer group's +highest_position+ watermark (never decreases), + # which drives the {#claim_next} +replaying+ flag. # # @param group_id [String] consumer group identifier # @param offset_id [Integer] offset ID from the claim result @@ -334,6 +348,14 @@ def ack(group_id, offset_id:, position:) claimed_at: nil, claimed_by: nil ) + + # Advance the high watermark (never decrease) + if position > cg[:highest_position] + db[:ccc_consumer_groups].where(id: cg[:id]).update( + highest_position: position, + updated_at: Time.now.iso8601 + ) + end end # Release a claim without advancing the offset. Use for error recovery diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 03baf116..c2e6aa56 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -747,6 +747,74 @@ module CCCStoreTestMessages store.append(new_event, guard: result[:guard]) }.to raise_error(Sourced::ConcurrentAppendError) end + + it 'replaying is false when consumer group has never processed the partition' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result[:replaying]).to be false + end + + it 'replaying is false for new messages after ack' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + + # New message arrives after ack + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + ) + + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2[:replaying]).to be false + end + + it 'replaying is true when offset is reset and messages are re-claimed' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + ]) + + # Process and ack — highest_position advances to 2 + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + + # Reset offsets — highest_position stays at 2 + store.reset_consumer_group(group_id) + + # Re-claim same messages — replaying because positions <= highest_position + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2[:replaying]).to be true + expect(r2[:messages].size).to eq(2) + end + + it 'replaying transitions to false once consumer passes highest_position after reset' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + # Process and ack up to position 1 + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + + # Reset offsets + store.reset_consumer_group(group_id) + + # Add a new message at position 2 + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + ) + + # Re-claim: both messages, but last position (2) > highest_position (1) + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2[:replaying]).to be false + expect(r2[:messages].size).to eq(2) + end end describe '#claim_next (composite partition — conditional AND fetch)' do From 732fd2c9d729c90427b3272a182e6389af0ff4b6 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 00:58:00 +0000 Subject: [PATCH 012/115] Return ClaimResult Data struct from Store#claim_next Replace the plain Hash with an immutable ClaimResult value object (Data.define) for type safety and a cleaner API. Defined in store.rb alongside the Store class. Tests updated to use method access. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 7 ++- spec/sourced/ccc/store_spec.rb | 103 +++++++++++++++++---------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 5b7f0153..dc29fc9c 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -19,6 +19,9 @@ def kind_of?(klass) = is_a?(klass) def instance_of?(klass) = __getobj__.instance_of?(klass) end + # Returned by {Store#claim_next} with everything needed to process and ack a partition. + ClaimResult = Data.define(:offset_id, :key_pair_ids, :partition_key, :partition_value, :messages, :replaying, :guard) + # SQLite-backed store for CCC's flat, globally-ordered message log. # Provides message storage with automatic key-pair indexing, # consumer group management, and partition-based offset tracking @@ -319,7 +322,7 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) # ever acked by this consumer group (i.e. they've been processed before). replaying = messages.last.position <= cg[:highest_position] - { + ClaimResult.new( offset_id: claimed[:offset_id], key_pair_ids: key_pair_ids, partition_key: claimed[:partition_key], @@ -327,7 +330,7 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) messages: messages, replaying: replaying, guard: guard - } + ) end # Acknowledge processing: advance the offset to +position+ and release the claim. diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index c2e6aa56..82dbeecb 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -586,15 +586,16 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') expect(result).not_to be_nil - expect(result[:offset_id]).to be_a(Integer) - expect(result[:key_pair_ids]).to be_a(Array) - expect(result[:partition_key]).to eq('device_id:dev-1') - expect(result[:partition_value]).to eq({ 'device_id' => 'dev-1' }) - expect(result[:messages]).to be_a(Array) - expect(result[:messages].size).to eq(1) - expect(result[:messages].first).to be_a(CCCStoreTestMessages::DeviceRegistered) - expect(result[:messages].first.position).to eq(1) - expect(result[:guard]).to be_a(Sourced::CCC::ConsistencyGuard) + expect(result).to be_a(Sourced::CCC::ClaimResult) + expect(result.offset_id).to be_a(Integer) + expect(result.key_pair_ids).to be_a(Array) + expect(result.partition_key).to eq('device_id:dev-1') + expect(result.partition_value).to eq({ 'device_id' => 'dev-1' }) + expect(result.messages).to be_a(Array) + expect(result.messages.size).to eq(1) + expect(result.messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(result.messages.first.position).to eq(1) + expect(result.guard).to be_a(Sourced::CCC::ConsistencyGuard) end it 'returns multiple pending messages for same partition' do @@ -604,7 +605,7 @@ module CCCStoreTestMessages ]) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(result[:messages].size).to eq(2) + expect(result.messages.size).to eq(2) end it 'skips claimed partitions — second worker gets different partition' do @@ -616,7 +617,7 @@ module CCCStoreTestMessages r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-2') - expect(r1[:partition_key]).not_to eq(r2[:partition_key]) + expect(r1.partition_key).not_to eq(r2.partition_key) end it 'returns nil when all partitions claimed' do @@ -644,7 +645,7 @@ module CCCStoreTestMessages ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) # Append another message for same partition store.append( @@ -652,8 +653,8 @@ module CCCStoreTestMessages ) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(r2[:messages].size).to eq(1) - expect(r2[:messages].first.payload.name).to eq('A updated') + expect(r2.messages.size).to eq(1) + expect(r2.messages.first.payload.name).to eq('A updated') end it 'returns nil for stopped consumer group' do @@ -676,7 +677,7 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') # dev-2 was appended first (position 1), so it should be prioritized - expect(result[:partition_value]).to eq({ 'device_id' => 'dev-2' }) + expect(result.partition_value).to eq({ 'device_id' => 'dev-2' }) end it 'bootstraps newly appeared partitions on subsequent calls' do @@ -685,7 +686,7 @@ module CCCStoreTestMessages ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) # New partition appears store.append( @@ -694,7 +695,7 @@ module CCCStoreTestMessages r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') expect(r2).not_to be_nil - expect(r2[:partition_value]).to eq({ 'device_id' => 'dev-3' }) + expect(r2.partition_value).to eq({ 'device_id' => 'dev-3' }) end it 'returns a guard with conditions only for key_names each type actually has' do @@ -703,10 +704,10 @@ module CCCStoreTestMessages ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - guard = result[:guard] + guard = result.guard expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) - expect(guard.last_position).to eq(result[:messages].last.position) + expect(guard.last_position).to eq(result.messages.last.position) # 1 key_pair (device_id=dev-1) × 1 handled_type = 1 condition expect(guard.conditions.size).to eq(1) @@ -725,9 +726,9 @@ module CCCStoreTestMessages # No concurrent writes — append with guard succeeds new_event = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) - expect { store.append(new_event, guard: result[:guard]) }.not_to raise_error + expect { store.append(new_event, guard: result.guard) }.not_to raise_error - store.ack(group_id, offset_id: result[:offset_id], position: result[:messages].last.position) + store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) end it 'guard detects concurrent conflicting writes' do @@ -744,7 +745,7 @@ module CCCStoreTestMessages new_event = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) expect { - store.append(new_event, guard: result[:guard]) + store.append(new_event, guard: result.guard) }.to raise_error(Sourced::ConcurrentAppendError) end @@ -754,7 +755,7 @@ module CCCStoreTestMessages ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(result[:replaying]).to be false + expect(result.replaying).to be false end it 'replaying is false for new messages after ack' do @@ -763,7 +764,7 @@ module CCCStoreTestMessages ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) # New message arrives after ack store.append( @@ -771,7 +772,7 @@ module CCCStoreTestMessages ) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(r2[:replaying]).to be false + expect(r2.replaying).to be false end it 'replaying is true when offset is reset and messages are re-claimed' do @@ -782,15 +783,15 @@ module CCCStoreTestMessages # Process and ack — highest_position advances to 2 r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) # Reset offsets — highest_position stays at 2 store.reset_consumer_group(group_id) # Re-claim same messages — replaying because positions <= highest_position r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(r2[:replaying]).to be true - expect(r2[:messages].size).to eq(2) + expect(r2.replaying).to be true + expect(r2.messages.size).to eq(2) end it 'replaying transitions to false once consumer passes highest_position after reset' do @@ -800,7 +801,7 @@ module CCCStoreTestMessages # Process and ack up to position 1 r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) # Reset offsets store.reset_consumer_group(group_id) @@ -812,8 +813,8 @@ module CCCStoreTestMessages # Re-claim: both messages, but last position (2) > highest_position (1) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(r2[:replaying]).to be false - expect(r2[:messages].size).to eq(2) + expect(r2.replaying).to be false + expect(r2.messages.size).to eq(2) end end @@ -848,7 +849,7 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') expect(result).not_to be_nil - expect(result[:partition_value]).to eq({ 'course_name' => 'Algebra', 'user_id' => 'joe' }) + expect(result.partition_value).to eq({ 'course_name' => 'Algebra', 'user_id' => 'joe' }) end it 'fetches messages with single partition attribute matching' do @@ -859,7 +860,7 @@ module CCCStoreTestMessages ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') - types = result[:messages].map(&:type) + types = result.messages.map(&:type) expect(types).to contain_exactly( 'store_test.course.created', 'store_test.user.registered', @@ -878,7 +879,7 @@ module CCCStoreTestMessages expect(r1).not_to be_nil expect(r2).not_to be_nil - expect(r1[:partition_key]).not_to eq(r2[:partition_key]) + expect(r1.partition_key).not_to eq(r2.partition_key) end it 'excludes messages with ALL partition attributes that do not match ALL values' do @@ -891,15 +892,15 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') # The first partition claimed should be one of the two — let's check its messages - if result[:partition_value]['user_id'] == 'joe' + if result.partition_value['user_id'] == 'joe' # Should include CourseCreated (1 attr, matches) and joe's join (2 attrs, both match) # Should NOT include jake's join (2 attrs, user_id doesn't match) - user_ids = result[:messages] + user_ids = result.messages .select { |m| m.type == 'store_test.user.joined_course' } .map { |m| m.payload.user_id } expect(user_ids).to eq(['joe']) else - user_ids = result[:messages] + user_ids = result.messages .select { |m| m.type == 'store_test.user.joined_course' } .map { |m| m.payload.user_id } expect(user_ids).to eq(['jake']) @@ -913,7 +914,7 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') # The join message matches both key_pairs but should appear only once - expect(result[:messages].size).to eq(1) + expect(result.messages.size).to eq(1) end it 'excludes messages with partial attributes matching wrong value' do @@ -925,7 +926,7 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') # Whichever partition we get, the other course's join should be excluded - courses = result[:messages].map { |m| m.payload.course_name } + courses = result.messages.map { |m| m.payload.course_name } expect(courses.uniq.size).to eq(1) end @@ -936,9 +937,9 @@ module CCCStoreTestMessages ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') - guard = result[:guard] + guard = result.guard - expect(guard.last_position).to eq(result[:messages].last.position) + expect(guard.last_position).to eq(result.messages.last.position) # Expected conditions (derived from message class definitions, not store data): # CourseCreated has course_name only → 1 condition @@ -979,7 +980,7 @@ module CCCStoreTestMessages new_event = CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) expect { - store.append(new_event, guard: result[:guard]) + store.append(new_event, guard: result.guard) }.to raise_error(Sourced::ConcurrentAppendError) end end @@ -999,10 +1000,10 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-1') - store.ack(group_id, offset_id: result[:offset_id], position: result[:messages].last.position) + store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) - offset = db[:ccc_offsets].where(id: result[:offset_id]).first - expect(offset[:last_position]).to eq(result[:messages].last.position) + offset = db[:ccc_offsets].where(id: result.offset_id).first + expect(offset[:last_position]).to eq(result.messages.last.position) expect(offset[:claimed]).to eq(0) expect(offset[:claimed_at]).to be_nil expect(offset[:claimed_by]).to be_nil @@ -1016,7 +1017,7 @@ module CCCStoreTestMessages r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-1') - store.ack(group_id, offset_id: r1[:offset_id], position: r1[:messages].last.position) + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) # No new messages — should return nil r2 = store.claim_next(group_id, partition_by: 'device_id', @@ -1040,9 +1041,9 @@ module CCCStoreTestMessages result = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-1') - store.release(group_id, offset_id: result[:offset_id]) + store.release(group_id, offset_id: result.offset_id) - offset = db[:ccc_offsets].where(id: result[:offset_id]).first + offset = db[:ccc_offsets].where(id: result.offset_id).first expect(offset[:last_position]).to eq(0) # not advanced expect(offset[:claimed]).to eq(0) end @@ -1054,13 +1055,13 @@ module CCCStoreTestMessages r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-1') - store.release(group_id, offset_id: r1[:offset_id]) + store.release(group_id, offset_id: r1.offset_id) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-2') - expect(r2[:offset_id]).to eq(r1[:offset_id]) - expect(r2[:messages].map(&:position)).to eq(r1[:messages].map(&:position)) + expect(r2.offset_id).to eq(r1.offset_id) + expect(r2.messages.map(&:position)).to eq(r1.messages.map(&:position)) end end end From 8c17f0f19fc3145acc324050222eeb856f09169d Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 15:39:15 +0000 Subject: [PATCH 013/115] Add CCC reactor abstraction layer (Decider, Projector, Router) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the Decide/Evolve/React pattern for CCC's stream-less model. Deciders request history via context_for() conditions, Projectors evolve from claimed messages directly. Router orchestrates claim→handle→execute→ack with transactional action execution, partial batch ACK, and error recovery. New modules: Actions (Append/Sync), Consumer, Evolve, React, Sync. Store#read now returns ReadResult data struct. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 8 + lib/sourced/ccc/actions.rb | 44 +++++ lib/sourced/ccc/consumer.rb | 52 ++++++ lib/sourced/ccc/decider.rb | 88 ++++++++++ lib/sourced/ccc/evolve.rb | 63 ++++++++ lib/sourced/ccc/projector.rb | 48 ++++++ lib/sourced/ccc/react.rb | 52 ++++++ lib/sourced/ccc/router.rb | 95 +++++++++++ lib/sourced/ccc/store.rb | 15 +- lib/sourced/ccc/sync.rb | 38 +++++ spec/sourced/ccc/decider_spec.rb | 204 +++++++++++++++++++++++ spec/sourced/ccc/evolve_spec.rb | 99 ++++++++++++ spec/sourced/ccc/projector_spec.rb | 152 +++++++++++++++++ spec/sourced/ccc/react_spec.rb | 93 +++++++++++ spec/sourced/ccc/router_spec.rb | 251 +++++++++++++++++++++++++++++ 15 files changed, 1298 insertions(+), 4 deletions(-) create mode 100644 lib/sourced/ccc/actions.rb create mode 100644 lib/sourced/ccc/consumer.rb create mode 100644 lib/sourced/ccc/decider.rb create mode 100644 lib/sourced/ccc/evolve.rb create mode 100644 lib/sourced/ccc/projector.rb create mode 100644 lib/sourced/ccc/react.rb create mode 100644 lib/sourced/ccc/router.rb create mode 100644 lib/sourced/ccc/sync.rb create mode 100644 spec/sourced/ccc/decider_spec.rb create mode 100644 spec/sourced/ccc/evolve_spec.rb create mode 100644 spec/sourced/ccc/projector_spec.rb create mode 100644 spec/sourced/ccc/react_spec.rb create mode 100644 spec/sourced/ccc/router_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 4bcd5f5e..2f5bd464 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -7,3 +7,11 @@ module CCC require 'sourced/ccc/message' require 'sourced/ccc/store' +require 'sourced/ccc/actions' +require 'sourced/ccc/consumer' +require 'sourced/ccc/evolve' +require 'sourced/ccc/react' +require 'sourced/ccc/sync' +require 'sourced/ccc/decider' +require 'sourced/ccc/projector' +require 'sourced/ccc/router' diff --git a/lib/sourced/ccc/actions.rb b/lib/sourced/ccc/actions.rb new file mode 100644 index 00000000..b7e3d630 --- /dev/null +++ b/lib/sourced/ccc/actions.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Sourced + module CCC + module Actions + OK = :ok + RETRY = :retry + + # Append messages to the CCC store with optional consistency guard. + # Auto-correlates messages with the source message at execution time. + class Append + attr_reader :messages, :guard + + def initialize(messages, guard: nil) + @messages = Array(messages) + @guard = guard + end + + # @param store [CCC::Store] + # @param source_message [CCC::Message] message to correlate from + # @return [Array] correlated messages that were appended + def execute(store, source_message) + correlated = messages.map { |m| source_message.correlate(m) } + store.append(correlated, guard: guard) + correlated + end + end + + # Execute a synchronous side effect within the current transaction. + class Sync + def initialize(work) + @work = work + end + + def call = @work.call + + def execute(_store, _source_message) + call + nil + end + end + end + end +end diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb new file mode 100644 index 00000000..f9275bb6 --- /dev/null +++ b/lib/sourced/ccc/consumer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # Shared consumer configuration for CCC reactors. + # Extended (not included) onto reactor classes. + module Consumer + def partition_keys + @partition_keys ||= [] + end + + def partition_by(*keys) + @partition_keys = keys.flatten.map(&:to_sym) + end + + def group_id + @group_id ||= name + end + + def consumer_group(id) + @group_id = id + end + + # Build query conditions from partition attributes and handled evolve types. + # Override in reactor for custom per-command conditions. + def context_for(partition_attrs) + handled_messages_for_evolve.flat_map { |klass| + klass.to_conditions(**partition_attrs) + } + end + + def on_exception(exception, message, group) + Sourced.config.error_strategy.call(exception, message, group) + end + + # Iterate messages collecting [actions, message] pairs. + # On mid-batch failure, raises PartialBatchError with pairs collected so far. + # If the first message fails, re-raises the original error. + def each_with_partial_ack(messages) + results = [] + messages.each do |msg| + pair = yield(msg) + results << pair if pair + rescue StandardError => e + raise e if results.empty? + raise Sourced::PartialBatchError.new(results, msg, e) + end + results + end + end + end +end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb new file mode 100644 index 00000000..c1741a7f --- /dev/null +++ b/lib/sourced/ccc/decider.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Sourced + module CCC + class Decider + include CCC::Evolve + include CCC::React + include CCC::Sync + extend CCC::Consumer + + class << self + def handled_commands + @handled_commands ||= [] + end + + # Messages to claim: commands to decide on + events to react to. + # Evolve types are NOT included — they are only for context_for(). + def handled_messages + handled_commands + handled_messages_for_react + end + + # Register a command handler. + def command(message_class, &block) + handled_commands << message_class + define_method(Sourced.message_method_name('ccc_decide', message_class.to_s), &block) + end + + # Reactor interface — requests history: via signature. + def handle_batch(claim, history:) + values = partition_keys.map { |k| claim.partition_value[k.to_s] } + instance = new(values) + instance.evolve(history.messages) + + each_with_partial_ack(claim.messages) do |msg| + if handled_commands.include?(msg.class) + events = instance.decide(msg) + actions = [] + actions << Actions::Append.new(events, guard: history.guard) if events.any? + + events.each do |evt| + next unless instance.reacts_to?(evt) + reaction_msgs = Array(instance.react(evt)) + actions << Actions::Append.new(reaction_msgs) if reaction_msgs.any? + end + + actions += instance.sync_actions( + state: instance.state, messages: [msg], events: events + ) + + [actions, msg] + else + [Actions::OK, msg] + end + end + end + + def inherited(subclass) + super + handled_commands.each do |cmd_class| + subclass.handled_commands << cmd_class + end + end + end + + attr_reader :partition_values + + def initialize(partition_values = []) + @partition_values = partition_values + @uncommitted_events = [] + end + + def decide(command) + @uncommitted_events = [] + method_name = Sourced.message_method_name('ccc_decide', command.class.to_s) + send(method_name, state, command) if respond_to?(method_name) + @uncommitted_events.dup + end + + # Called from within command handlers to produce events. + def event(event_class, payload = {}) + evt = event_class.new(payload: payload) + @uncommitted_events << evt + evolve([evt]) + evt + end + end + end +end diff --git a/lib/sourced/ccc/evolve.rb b/lib/sourced/ccc/evolve.rb new file mode 100644 index 00000000..8d9c8fc6 --- /dev/null +++ b/lib/sourced/ccc/evolve.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # Evolve mixin for CCC reactors. + # Adapted from Sourced::Evolve for CCC::Message (no stream_id/seq). + # State block receives partition values array instead of stream id. + module Evolve + PREFIX = 'ccc_evolution' + + def self.included(base) + super + base.extend ClassMethods + end + + def init_state(_partition_values) + nil + end + + def state + @state ||= init_state(partition_values) + end + + def partition_values + @partition_values ||= [] + end + + # Apply messages to state via registered handlers. + # Skips messages without a registered handler. + def evolve(messages) + Array(messages).each do |msg| + method_name = Sourced.message_method_name(PREFIX, msg.class.to_s) + send(method_name, state, msg) if respond_to?(method_name) + end + state + end + + module ClassMethods + def inherited(subclass) + super + handled_messages_for_evolve.each do |klass| + subclass.handled_messages_for_evolve << klass + end + end + + def handled_messages_for_evolve + @handled_messages_for_evolve ||= [] + end + + # Define initial state factory. Block receives partition values array. + def state(&blk) + define_method(:init_state, &blk) + end + + # Register an evolve handler for a CCC::Message subclass. + def evolve(message_class, &block) + handled_messages_for_evolve << message_class + define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) + end + end + end + end +end diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb new file mode 100644 index 00000000..b2e61895 --- /dev/null +++ b/lib/sourced/ccc/projector.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Sourced + module CCC + class Projector + include CCC::Evolve + include CCC::React + include CCC::Sync + extend CCC::Consumer + + class << self + # Projectors claim events they evolve from + events they react to. + def handled_messages + (handled_messages_for_evolve + handled_messages_for_react).uniq + end + + # No history: — uses claim.messages directly. + def handle_batch(claim) + values = partition_keys.map { |k| claim.partition_value[k.to_s] } + instance = new(values) + instance.evolve(claim.messages) + + sync_actions = instance.sync_actions( + state: instance.state, messages: claim.messages, replaying: claim.replaying + ) + + reaction_pairs = if claim.replaying + [] + else + each_with_partial_ack(claim.messages) do |msg| + next unless instance.reacts_to?(msg) + reaction_msgs = Array(instance.react(msg)) + reaction_msgs.any? ? [Actions::Append.new(reaction_msgs), msg] : nil + end + end + + reaction_pairs + [[sync_actions, claim.messages.last]] + end + end + + attr_reader :partition_values + + def initialize(partition_values = []) + @partition_values = partition_values + end + end + end +end diff --git a/lib/sourced/ccc/react.rb b/lib/sourced/ccc/react.rb new file mode 100644 index 00000000..165951ed --- /dev/null +++ b/lib/sourced/ccc/react.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # React mixin for CCC reactors. + # Reactions return raw CCC::Message instances (not correlated). + # The runtime handles auto-correlation via Actions::Append#execute. + module React + PREFIX = 'ccc_reaction' + EMPTY_ARRAY = [].freeze + + def self.included(base) + super + base.extend ClassMethods + end + + # Run the reaction handler for a single message. + # Returns raw messages (not correlated). + def react(message) + method_name = Sourced.message_method_name(PREFIX, message.class.to_s) + if respond_to?(method_name) + Array(send(method_name, state, message)).compact + else + EMPTY_ARRAY + end + end + + def reacts_to?(message) + self.class.handled_messages_for_react.include?(message.class) + end + + module ClassMethods + def inherited(subclass) + super + handled_messages_for_react.each do |klass| + subclass.handled_messages_for_react << klass + end + end + + def handled_messages_for_react + @handled_messages_for_react ||= [] + end + + # Register a reaction handler for a CCC::Message subclass. + def reaction(message_class, &block) + handled_messages_for_react << message_class + define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) + end + end + end + end +end diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb new file mode 100644 index 00000000..c9be2d7c --- /dev/null +++ b/lib/sourced/ccc/router.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'sourced/injector' + +module Sourced + module CCC + class Router + attr_reader :store, :reactors + + def initialize(store:) + @store = store + @reactors = [] + @needs_history = {} + end + + def register(reactor_class) + @reactors << reactor_class + store.register_consumer_group(reactor_class.group_id) + @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_batch).include?(:history) + end + + def handle_next_for(reactor_class, worker_id: 'default') + handled_types = reactor_class.handled_messages.map(&:type).uniq + + claim = store.claim_next( + reactor_class.group_id, + partition_by: reactor_class.partition_keys.map(&:to_s), + handled_types: handled_types, + worker_id: worker_id + ) + return false unless claim + + begin + kwargs = {} + if @needs_history[reactor_class] + attrs = claim.partition_value.transform_keys(&:to_sym) + conditions = reactor_class.context_for(attrs) + kwargs[:history] = store.read(conditions) + end + + action_pairs = reactor_class.handle_batch(claim, **kwargs) + + if action_pairs == Actions::RETRY + store.release(reactor_class.group_id, offset_id: claim.offset_id) + return true + end + + execute_actions(action_pairs, claim, reactor_class.group_id) + true + + rescue Sourced::PartialBatchError => e + execute_actions(e.action_pairs, claim, reactor_class.group_id) + reactor_class.on_exception(e, e.failed_message, nil) + true + rescue Sourced::ConcurrentAppendError + store.release(reactor_class.group_id, offset_id: claim.offset_id) + true + rescue StandardError => e + store.release(reactor_class.group_id, offset_id: claim.offset_id) + reactor_class.on_exception(e, claim.messages.first, nil) + true + end + end + + def drain(limit = Float::INFINITY) + count = 0 + loop do + count += 1 + found_any = @reactors.any? { |r| handle_next_for(r) } + break unless found_any && count < limit + end + end + + private + + def execute_actions(action_pairs, claim, group_id) + store.db.transaction do + last_position = nil + Array(action_pairs).each do |(actions, source_message)| + Array(actions).each do |action| + action.execute(store, source_message) unless action == Actions::OK + end + last_position = source_message.position if source_message.respond_to?(:position) + end + + if last_position + store.ack(group_id, offset_id: claim.offset_id, position: last_position) + else + store.release(group_id, offset_id: claim.offset_id) + end + end + end + end + end +end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index dc29fc9c..f492eb76 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -22,6 +22,13 @@ def instance_of?(klass) = __getobj__.instance_of?(klass) # Returned by {Store#claim_next} with everything needed to process and ack a partition. ClaimResult = Data.define(:offset_id, :key_pair_ids, :partition_key, :partition_value, :messages, :replaying, :guard) + # Returned by {Store#read} with messages and a consistency guard. + # Supports array destructuring via #to_ary for backwards compatibility: + # messages, guard = store.read(conditions) + ReadResult = Data.define(:messages, :guard) do + def to_ary = [messages, guard] + end + # SQLite-backed store for CCC's flat, globally-ordered message log. # Provides message storage with automatic key-pair indexing, # consumer group management, and partition-based offset tracking @@ -183,18 +190,18 @@ def append(messages, guard: nil) # @param conditions [QueryCondition, Array] query conditions # @param from_position [Integer, nil] only return messages after this position # @param limit [Integer, nil] max number of messages to return - # @return [Array(Array, ConsistencyGuard)] messages and a guard + # @return [ReadResult] messages and a guard def read(conditions, from_position: nil, limit: nil) conditions = Array(conditions) if conditions.empty? guard = ConsistencyGuard.new(conditions: conditions, last_position: from_position || latest_position) - return [[], guard] + return ReadResult.new(messages: [], guard: guard) end messages = query_messages(conditions, from_position: from_position, limit: limit) last_pos = messages.any? ? messages.last.position : (from_position || latest_position) guard = ConsistencyGuard.new(conditions: conditions, last_position: last_pos) - [messages, guard] + ReadResult.new(messages: messages, guard: guard) end # Conflict detection: returns messages matching conditions that appeared @@ -202,7 +209,7 @@ def read(conditions, from_position: nil, limit: nil) # # @param conditions [Array] conditions to check # @param position [Integer] check for messages after this position - # @return [Array(Array, ConsistencyGuard)] + # @return [ReadResult] def messages_since(conditions, position) read(conditions, from_position: position) end diff --git a/lib/sourced/ccc/sync.rb b/lib/sourced/ccc/sync.rb new file mode 100644 index 00000000..8f17b3d4 --- /dev/null +++ b/lib/sourced/ccc/sync.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # Sync mixin for CCC reactors. + # Registers blocks that run within the store transaction. + module Sync + def self.included(base) + super + base.extend ClassMethods + end + + # Build Actions::Sync wrappers for all registered sync blocks. + def sync_actions(**args) + self.class.sync_blocks.map do |block| + Actions::Sync.new(proc { instance_exec(**args, &block) }) + end + end + + module ClassMethods + def inherited(subclass) + super + sync_blocks.each do |blk| + subclass.sync_blocks << blk + end + end + + def sync_blocks + @sync_blocks ||= [] + end + + def sync(&block) + sync_blocks << block + end + end + end + end +end diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/sourced/ccc/decider_spec.rb new file mode 100644 index 00000000..0229f8bc --- /dev/null +++ b/spec/sourced/ccc/decider_spec.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +module CCCDeciderTestMessages + DeviceRegistered = Sourced::CCC::Message.define('decider_test.device.registered') do + attribute :device_id, String + attribute :name, String + end + + DeviceBound = Sourced::CCC::Message.define('decider_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDevice = Sourced::CCC::Message.define('decider_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end + + NotifyBound = Sourced::CCC::Message.define('decider_test.notify_bound') do + attribute :device_id, String + end +end + +class TestDeviceDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'device-decider-test' + + state { |_| { exists: false, bound: false } } + + evolve CCCDeciderTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + evolve CCCDeciderTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command CCCDeciderTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event CCCDeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end + + reaction CCCDeciderTestMessages::DeviceBound do |_state, evt| + CCCDeciderTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) + end +end + +RSpec.describe Sourced::CCC::Decider do + describe '.command' do + it 'registers handler and #decide runs it' do + expect(TestDeviceDecider.handled_commands).to include(CCCDeciderTestMessages::BindDevice) + + instance = TestDeviceDecider.new + instance.instance_variable_set(:@state, { exists: true, bound: false }) + + events = instance.decide( + CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + expect(events.size).to eq(1) + expect(events.first).to be_a(CCCDeciderTestMessages::DeviceBound) + end + end + + describe '#event inside command handler' do + it 'adds to uncommitted events and evolves state immediately' do + instance = TestDeviceDecider.new + instance.instance_variable_set(:@state, { exists: true, bound: false }) + + events = instance.decide( + CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + expect(events.size).to eq(1) + expect(instance.state[:bound]).to be true + end + end + + describe '.handle_batch' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + + before do + store.install! + end + + it 'evolves from history, decides commands, returns action pairs' do + # Set up history + reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + history_msgs = [Sourced::CCC::PositionedMessage.new(reg, 1)] + guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 1) + history = Sourced::CCC::ReadResult.new(messages: history_msgs, guard: guard) + + cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + cmd_positioned = Sourced::CCC::PositionedMessage.new(cmd, 2) + + claim = Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [cmd_positioned], replaying: false, guard: guard + ) + + pairs = TestDeviceDecider.handle_batch(claim, history: history) + + # Should have action pairs from the command + expect(pairs).to be_a(Array) + expect(pairs.size).to eq(1) # one command processed + + actions, source_msg = pairs.first + expect(source_msg).to eq(cmd_positioned) + + # Actions: Append(events with guard), Append(reactions), possibly sync + append_actions = Array(actions).select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + expect(append_actions.size).to be >= 1 + + # First append has the events with guard + event_append = append_actions.first + expect(event_append.messages.first).to be_a(CCCDeciderTestMessages::DeviceBound) + expect(event_append.guard).to eq(guard) + + # Second append has the reactions (no guard) + if append_actions.size > 1 + reaction_append = append_actions[1] + expect(reaction_append.messages.first).to be_a(CCCDeciderTestMessages::NotifyBound) + expect(reaction_append.guard).to be_nil + end + end + + it 'returns [OK, msg] for non-command messages' do + reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + reg_positioned = Sourced::CCC::PositionedMessage.new(reg, 1) + + guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 0) + history = Sourced::CCC::ReadResult.new(messages: [], guard: guard) + + claim = Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [reg_positioned], replaying: false, guard: guard + ) + + pairs = TestDeviceDecider.handle_batch(claim, history: history) + + expect(pairs.size).to eq(1) + actions, source_msg = pairs.first + expect(actions).to eq(Sourced::CCC::Actions::OK) + expect(source_msg).to eq(reg_positioned) + end + + it 'invariant violation propagates as error' do + guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 0) + history = Sourced::CCC::ReadResult.new(messages: [], guard: guard) + + cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + cmd_positioned = Sourced::CCC::PositionedMessage.new(cmd, 1) + + claim = Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [cmd_positioned], replaying: false, guard: guard + ) + + # No history → state[:exists] is false → raises 'Not found' + expect { + TestDeviceDecider.handle_batch(claim, history: history) + }.to raise_error(RuntimeError, 'Not found') + end + end + + describe '.handled_messages' do + it 'includes commands and react types but not evolve types' do + msgs = TestDeviceDecider.handled_messages + expect(msgs).to include(CCCDeciderTestMessages::BindDevice) + expect(msgs).to include(CCCDeciderTestMessages::DeviceBound) # reaction + expect(msgs).not_to include(CCCDeciderTestMessages::DeviceRegistered) # evolve only + end + end + + describe '.context_for' do + it 'builds conditions from partition_keys × handled_messages_for_evolve' do + conditions = TestDeviceDecider.context_for(device_id: 'd1') + + # DeviceRegistered and DeviceBound both have device_id + types = conditions.map(&:message_type).uniq.sort + expect(types).to include('decider_test.device.registered') + expect(types).to include('decider_test.device.bound') + expect(conditions.all? { |c| c.key_name == 'device_id' && c.key_value == 'd1' }).to be true + end + end + + describe 'inheritance' do + it 'subclass inherits command handlers' do + subclass = Class.new(TestDeviceDecider) + expect(subclass.handled_commands).to include(CCCDeciderTestMessages::BindDevice) + end + end +end diff --git a/spec/sourced/ccc/evolve_spec.rb b/spec/sourced/ccc/evolve_spec.rb new file mode 100644 index 00000000..61efe6c5 --- /dev/null +++ b/spec/sourced/ccc/evolve_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +module CCCEvolveTestMessages + ItemAdded = Sourced::CCC::Message.define('evolve_test.item.added') do + attribute :item_id, String + attribute :name, String + end + + ItemRemoved = Sourced::CCC::Message.define('evolve_test.item.removed') do + attribute :item_id, String + end + + Unhandled = Sourced::CCC::Message.define('evolve_test.unhandled') do + attribute :foo, String + end +end + +RSpec.describe Sourced::CCC::Evolve do + let(:evolver_class) do + Class.new do + include Sourced::CCC::Evolve + + state do |partition_values| + { items: [], partition_values: partition_values } + end + + evolve CCCEvolveTestMessages::ItemAdded do |state, msg| + state[:items] << { id: msg.payload.item_id, name: msg.payload.name } + end + + evolve CCCEvolveTestMessages::ItemRemoved do |state, msg| + state[:items].reject! { |i| i[:id] == msg.payload.item_id } + end + end + end + + describe '.state' do + it 'initializes state with partition values array' do + instance = evolver_class.new + instance.instance_variable_set(:@partition_values, ['val1', 'val2']) + expect(instance.state[:partition_values]).to eq(['val1', 'val2']) + end + end + + describe '#evolve' do + it 'applies registered handlers in order' do + instance = evolver_class.new + messages = [ + CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }), + CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i2', name: 'Banana' }), + CCCEvolveTestMessages::ItemRemoved.new(payload: { item_id: 'i1' }) + ] + + instance.evolve(messages) + + expect(instance.state[:items]).to eq([{ id: 'i2', name: 'Banana' }]) + end + + it 'skips unregistered message types' do + instance = evolver_class.new + messages = [ + CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }), + CCCEvolveTestMessages::Unhandled.new(payload: { foo: 'bar' }) + ] + + instance.evolve(messages) + + expect(instance.state[:items]).to eq([{ id: 'i1', name: 'Apple' }]) + end + end + + describe '.handled_messages_for_evolve' do + it 'tracks registered classes' do + expect(evolver_class.handled_messages_for_evolve).to contain_exactly( + CCCEvolveTestMessages::ItemAdded, + CCCEvolveTestMessages::ItemRemoved + ) + end + end + + describe 'inheritance' do + it 'subclass inherits evolve handlers' do + subclass = Class.new(evolver_class) + expect(subclass.handled_messages_for_evolve).to contain_exactly( + CCCEvolveTestMessages::ItemAdded, + CCCEvolveTestMessages::ItemRemoved + ) + + instance = subclass.new + instance.evolve([ + CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }) + ]) + expect(instance.state[:items]).to eq([{ id: 'i1', name: 'Apple' }]) + end + end +end diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb new file mode 100644 index 00000000..1631b48c --- /dev/null +++ b/spec/sourced/ccc/projector_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +module CCCProjectorTestMessages + ItemAdded = Sourced::CCC::Message.define('projector_test.item.added') do + attribute :list_id, String + attribute :name, String + end + + ItemArchived = Sourced::CCC::Message.define('projector_test.item.archived') do + attribute :list_id, String + attribute :name, String + end + + NotifyArchive = Sourced::CCC::Message.define('projector_test.notify_archive') do + attribute :list_id, String + end +end + +class TestItemProjector < Sourced::CCC::Projector + partition_by :list_id + consumer_group 'item-projector-test' + + state do |(list_id)| + { list_id: list_id, items: [], synced: false } + end + + evolve CCCProjectorTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end + + evolve CCCProjectorTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + reaction CCCProjectorTestMessages::ItemArchived do |_state, msg| + CCCProjectorTestMessages::NotifyArchive.new(payload: { list_id: msg.payload.list_id }) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + state[:last_replaying] = replaying + end +end + +RSpec.describe Sourced::CCC::Projector do + describe '.handled_messages' do + it 'includes evolve and react types' do + msgs = TestItemProjector.handled_messages + expect(msgs).to include(CCCProjectorTestMessages::ItemAdded) + expect(msgs).to include(CCCProjectorTestMessages::ItemArchived) + end + end + + describe '.handle_batch' do + let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 2) } + + def make_claim(messages, replaying: false) + Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'list_id:L1', + partition_value: { 'list_id' => 'L1' }, + messages: messages, replaying: replaying, guard: guard + ) + end + + it 'evolves from claim.messages and includes sync actions' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ) + ] + claim = make_claim(msgs) + + pairs = TestItemProjector.handle_batch(claim) + + # Last pair should contain sync actions + sync_pair = pairs.last + sync_actions, source_msg = sync_pair + expect(source_msg).to eq(msgs.last) + + sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::Sync) } + expect(sync_action).not_to be_nil + end + + it 'runs reactions when not replaying' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: false) + + pairs = TestItemProjector.handle_batch(claim) + + # Should have reaction pair + sync pair + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions.size).to eq(1) + expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) + end + + it 'skips reactions when replaying' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: true) + + pairs = TestItemProjector.handle_batch(claim) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions).to be_empty + end + + it 'passes replaying to sync blocks' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: true) + + pairs = TestItemProjector.handle_batch(claim) + + # Execute the sync action to verify replaying is passed through + sync_pair = pairs.last + sync_actions = Array(sync_pair.first).select { |a| a.is_a?(Sourced::CCC::Actions::Sync) } + expect(sync_actions).not_to be_empty + + # Call the sync to verify it runs + sync_actions.first.call + end + end + + describe '.context_for' do + it 'builds conditions from partition_keys × handled_messages_for_evolve' do + conditions = TestItemProjector.context_for(list_id: 'L1') + types = conditions.map(&:message_type).sort + expect(types).to include('projector_test.item.added') + expect(types).to include('projector_test.item.archived') + end + end +end diff --git a/spec/sourced/ccc/react_spec.rb b/spec/sourced/ccc/react_spec.rb new file mode 100644 index 00000000..31c99a22 --- /dev/null +++ b/spec/sourced/ccc/react_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +module CCCReactTestMessages + SomethingHappened = Sourced::CCC::Message.define('react_test.something.happened') do + attribute :thing_id, String + end + + DoNext = Sourced::CCC::Message.define('react_test.do_next') do + attribute :thing_id, String + end + + Unhandled = Sourced::CCC::Message.define('react_test.unhandled') do + attribute :foo, String + end +end + +RSpec.describe Sourced::CCC::React do + let(:reactor_class) do + Class.new do + include Sourced::CCC::React + + def state + {} + end + + reaction CCCReactTestMessages::SomethingHappened do |_state, msg| + CCCReactTestMessages::DoNext.new(payload: { thing_id: msg.payload.thing_id }) + end + end + end + + describe '#react' do + it 'returns raw messages (not correlated)' do + instance = reactor_class.new + msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + + result = instance.react(msg) + + expect(result.size).to eq(1) + expect(result.first).to be_a(CCCReactTestMessages::DoNext) + expect(result.first.payload.thing_id).to eq('t1') + # Not correlated — causation_id is its own id + expect(result.first.causation_id).to eq(result.first.id) + end + + it 'returns empty array for unregistered types' do + instance = reactor_class.new + msg = CCCReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) + + result = instance.react(msg) + expect(result).to eq([]) + end + end + + describe '#reacts_to?' do + it 'returns true for registered types' do + instance = reactor_class.new + msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + expect(instance.reacts_to?(msg)).to be true + end + + it 'returns false for unregistered types' do + instance = reactor_class.new + msg = CCCReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) + expect(instance.reacts_to?(msg)).to be false + end + end + + describe '.handled_messages_for_react' do + it 'tracks registered classes' do + expect(reactor_class.handled_messages_for_react).to contain_exactly( + CCCReactTestMessages::SomethingHappened + ) + end + end + + describe 'inheritance' do + it 'subclass inherits reaction handlers' do + subclass = Class.new(reactor_class) + expect(subclass.handled_messages_for_react).to contain_exactly( + CCCReactTestMessages::SomethingHappened + ) + + instance = subclass.new + msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + result = instance.react(msg) + expect(result.size).to eq(1) + end + end +end diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb new file mode 100644 index 00000000..7165c493 --- /dev/null +++ b/spec/sourced/ccc/router_spec.rb @@ -0,0 +1,251 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +module CCCRouterTestMessages + DeviceRegistered = Sourced::CCC::Message.define('router_test.device.registered') do + attribute :device_id, String + attribute :name, String + end + + DeviceBound = Sourced::CCC::Message.define('router_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDevice = Sourced::CCC::Message.define('router_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end + + NotifyBound = Sourced::CCC::Message.define('router_test.notify_bound') do + attribute :device_id, String + end + + # Projector messages + DeviceListed = Sourced::CCC::Message.define('router_test.device.listed') do + attribute :device_id, String + end +end + +# Test decider for router specs +class RouterTestDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'router-test-decider' + + state { |_| { exists: false, bound: false } } + + evolve CCCRouterTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + evolve CCCRouterTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command CCCRouterTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event CCCRouterTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end +end + +# Test projector for router specs +class RouterTestProjector < Sourced::CCC::Projector + partition_by :device_id + consumer_group 'router-test-projector' + + state { |_| { devices: [] } } + + evolve CCCRouterTestMessages::DeviceRegistered do |state, evt| + state[:devices] << evt.payload.name + end + + evolve CCCRouterTestMessages::DeviceBound do |state, _evt| + # nothing + end + + sync do |state:, messages:, replaying:| + # In a real projector, this would persist to DB + state[:synced] = true + end +end + +RSpec.describe Sourced::CCC::Router do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + let(:router) { Sourced::CCC::Router.new(store: store) } + + before do + store.install! + end + + describe '#register' do + it 'creates consumer group and introspects handle_batch signature' do + router.register(RouterTestDecider) + + expect(store.consumer_group_active?('router-test-decider')).to be true + expect(router.reactors).to include(RouterTestDecider) + end + + it 'detects history: for decider, none for projector' do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + + # Decider needs history, projector does not + expect(router.instance_variable_get(:@needs_history)[RouterTestDecider]).to be true + expect(router.instance_variable_get(:@needs_history)[RouterTestProjector]).to be false + end + end + + describe '#handle_next_for' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + end + + it 'returns false when no work available' do + result = router.handle_next_for(RouterTestDecider) + expect(result).to be false + end + + it 'claims, calls handle_batch, executes actions + acks in transaction' do + # Set up: register device first (as history), then send bind command + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + result = router.handle_next_for(RouterTestDecider) + expect(result).to be true + + # DeviceBound event should have been appended to store + conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + read_result = store.read(conds) + expect(read_result.messages.size).to eq(1) + expect(read_result.messages.first).to be_a(CCCRouterTestMessages::DeviceBound) + end + + it 'reads history for decider, skips history for projector' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + # Projector should process without history + result = router.handle_next_for(RouterTestProjector) + expect(result).to be true + end + + it 'releases on ConcurrentAppendError' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + # Stub store.append to raise ConcurrentAppendError after claim + original_append = store.method(:append) + call_count = 0 + allow(store).to receive(:append) do |*args, **kwargs| + call_count += 1 + if call_count > 0 && kwargs[:guard] + raise Sourced::ConcurrentAppendError, 'conflict' + end + original_append.call(*args, **kwargs) + end + + result = router.handle_next_for(RouterTestDecider) + expect(result).to be true + + # Offset should be released (not advanced) — can re-claim + claim = store.claim_next( + 'router-test-decider', + partition_by: ['device_id'], + handled_types: RouterTestDecider.handled_messages.map(&:type), + worker_id: 'w-1' + ) + expect(claim).not_to be_nil + end + + it 'releases on StandardError and calls on_exception' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + # Make handle_batch raise + allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + allow(RouterTestDecider).to receive(:on_exception) + + result = router.handle_next_for(RouterTestDecider) + expect(result).to be true + expect(RouterTestDecider).to have_received(:on_exception) + end + end + + describe '#drain' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + end + + it 'processes all reactors until no work remains' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + router.drain + + # Decider should have produced DeviceBound + conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + read_result = store.read(conds) + expect(read_result.messages.size).to eq(1) + + # Projector should have processed DeviceRegistered and DeviceBound + # (it handles both via evolve) + end + end + + describe 'full integration: append commands → Decider → events → Projector' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + end + + it 'end-to-end flow works' do + # 1. Register device (event — goes to projector directly) + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + # 2. Send bind command (decider will produce DeviceBound event) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + # 3. Drain — decider processes command, projector processes events + router.drain + + # 4. Verify DeviceBound was appended + conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + read_result = store.read(conds) + bound_events = read_result.messages.select { |m| m.is_a?(CCCRouterTestMessages::DeviceBound) } + expect(bound_events.size).to eq(1) + + # Verify causation chain + bound = bound_events.first + expect(bound.causation_id).not_to be_nil + expect(bound.correlation_id).not_to be_nil + end + end +end From 50ac8c956e1220a3ec5315e83280db6ab58e96f4 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 15:52:06 +0000 Subject: [PATCH 014/115] Fix causation chain: reactions correlate with event, not command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decider now pre-correlates events with the command, then passes correlated events as source: to reaction Appends. This gives the correct causation chain: cmd → event → reaction message, with correlation_id tracing back to the command throughout. Append gains source: (override correlation source) and correlated: (skip re-correlation) options. Router integration tests verify exact causation_id and correlation_id at each link in the chain. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/actions.rb | 27 ++++++++++++---- lib/sourced/ccc/decider.rb | 11 ++++--- spec/sourced/ccc/router_spec.rb | 57 ++++++++++++++++++++++----------- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/lib/sourced/ccc/actions.rb b/lib/sourced/ccc/actions.rb index b7e3d630..f2723dbd 100644 --- a/lib/sourced/ccc/actions.rb +++ b/lib/sourced/ccc/actions.rb @@ -8,21 +8,36 @@ module Actions # Append messages to the CCC store with optional consistency guard. # Auto-correlates messages with the source message at execution time. + # + # When +source:+ is provided, it overrides the runtime's source_message + # for correlation (e.g. reactions correlated with the event, not the command). + # + # When +correlated: true+, messages are assumed to be already correlated + # and are appended as-is without re-correlation. class Append - attr_reader :messages, :guard + attr_reader :messages, :guard, :source - def initialize(messages, guard: nil) + def initialize(messages, guard: nil, source: nil, correlated: false) @messages = Array(messages) @guard = guard + @source = source + @correlated = correlated end + def correlated? = @correlated + # @param store [CCC::Store] - # @param source_message [CCC::Message] message to correlate from + # @param source_message [CCC::Message] default message to correlate from # @return [Array] correlated messages that were appended def execute(store, source_message) - correlated = messages.map { |m| source_message.correlate(m) } - store.append(correlated, guard: guard) - correlated + to_append = if @correlated + messages + else + correlate_from = @source || source_message + messages.map { |m| correlate_from.correlate(m) } + end + store.append(to_append, guard: guard) + to_append end end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index c1741a7f..d07f15e0 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -33,18 +33,19 @@ def handle_batch(claim, history:) each_with_partial_ack(claim.messages) do |msg| if handled_commands.include?(msg.class) - events = instance.decide(msg) + raw_events = instance.decide(msg) + correlated_events = raw_events.map { |e| msg.correlate(e) } actions = [] - actions << Actions::Append.new(events, guard: history.guard) if events.any? + actions << Actions::Append.new(correlated_events, guard: history.guard, correlated: true) if correlated_events.any? - events.each do |evt| + correlated_events.each do |evt| next unless instance.reacts_to?(evt) reaction_msgs = Array(instance.react(evt)) - actions << Actions::Append.new(reaction_msgs) if reaction_msgs.any? + actions << Actions::Append.new(reaction_msgs, source: evt) if reaction_msgs.any? end actions += instance.sync_actions( - state: instance.state, messages: [msg], events: events + state: instance.state, messages: [msg], events: raw_events ) [actions, msg] diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 7165c493..129db77b 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -50,6 +50,10 @@ class RouterTestDecider < Sourced::CCC::Decider raise 'Already bound' if state[:bound] event CCCRouterTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id end + + reaction CCCRouterTestMessages::DeviceBound do |_state, evt| + CCCRouterTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) + end end # Test projector for router specs @@ -222,30 +226,47 @@ class RouterTestProjector < Sourced::CCC::Projector router.register(RouterTestProjector) end - it 'end-to-end flow works' do - # 1. Register device (event — goes to projector directly) - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) + it 'events are correlated with the command that produced them' do + reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) - # 2. Send bind command (decider will produce DeviceBound event) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) + cmd = CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + store.append(cmd) - # 3. Drain — decider processes command, projector processes events router.drain - # 4. Verify DeviceBound was appended + # Read the DeviceBound event conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') - read_result = store.read(conds) - bound_events = read_result.messages.select { |m| m.is_a?(CCCRouterTestMessages::DeviceBound) } - expect(bound_events.size).to eq(1) + bound = store.read(conds).messages.find { |m| m.is_a?(CCCRouterTestMessages::DeviceBound) } + + expect(bound).not_to be_nil + expect(bound.causation_id).to eq(cmd.id) + expect(bound.correlation_id).to eq(cmd.correlation_id) + end + + it 'reaction messages are correlated with the event reacted to, not the command' do + reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + cmd = CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + store.append(cmd) + + router.drain + + # Read the DeviceBound event (produced by command handler) + bound_conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + bound = store.read(bound_conds).messages.find { |m| m.is_a?(CCCRouterTestMessages::DeviceBound) } + + # Read the NotifyBound reaction message (produced by reaction handler) + notify_conds = CCCRouterTestMessages::NotifyBound.to_conditions(device_id: 'd1') + notify = store.read(notify_conds).messages.find { |m| m.is_a?(CCCRouterTestMessages::NotifyBound) } - # Verify causation chain - bound = bound_events.first - expect(bound.causation_id).not_to be_nil - expect(bound.correlation_id).not_to be_nil + expect(notify).not_to be_nil + # Reaction is correlated with the event, not the command + expect(notify.causation_id).to eq(bound.id) + expect(notify.correlation_id).to eq(bound.correlation_id) + # The whole chain shares the same correlation_id (the command's) + expect(notify.correlation_id).to eq(cmd.correlation_id) end end end From 9ddcebafb28ca593d24a09437dbd193ad7a649b3 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 16:00:11 +0000 Subject: [PATCH 015/115] Support simple Consumer-only reactors without Decider/Projector Add default empty handled_messages_for_evolve to CCC::Consumer so context_for works for reactors that just extend Consumer, define handled_messages, and implement handle_batch with manual action pairs. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/consumer.rb | 7 +++ spec/sourced/ccc/router_spec.rb | 100 ++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index f9275bb6..7f1f222e 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -21,6 +21,13 @@ def consumer_group(id) @group_id = id end + # Message types this consumer evolves from. Used by {#context_for} + # to build query conditions for history reads. + # Defaults to empty; overridden by CCC::Evolve mixin. + def handled_messages_for_evolve + @handled_messages_for_evolve ||= [] + end + # Build query conditions from partition attributes and handled evolve types. # Override in reactor for custom per-command conditions. def context_for(partition_attrs) diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 129db77b..a806f53c 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -28,6 +28,12 @@ module CCCRouterTestMessages DeviceListed = Sourced::CCC::Message.define('router_test.device.listed') do attribute :device_id, String end + + # Simple reactor messages + DeviceAudited = Sourced::CCC::Message.define('router_test.device.audited') do + attribute :device_id, String + attribute :event_type, String + end end # Test decider for router specs @@ -77,6 +83,28 @@ class RouterTestProjector < Sourced::CCC::Projector end end +# Simple reactor: just extends Consumer, defines handled_messages, implements handle_batch. +# Logs an audit trail message for every DeviceRegistered or DeviceBound it sees. +class RouterTestAuditReactor + extend Sourced::CCC::Consumer + + partition_by :device_id + consumer_group 'router-test-audit' + + def self.handled_messages + [CCCRouterTestMessages::DeviceRegistered, CCCRouterTestMessages::DeviceBound] + end + + def self.handle_batch(claim) + each_with_partial_ack(claim.messages) do |msg| + audit = CCCRouterTestMessages::DeviceAudited.new( + payload: { device_id: msg.payload.device_id, event_type: msg.type } + ) + [Sourced::CCC::Actions::Append.new(audit), msg] + end + end +end + RSpec.describe Sourced::CCC::Router do let(:db) { Sequel.sqlite } let(:store) { Sourced::CCC::Store.new(db) } @@ -269,4 +297,76 @@ class RouterTestProjector < Sourced::CCC::Projector expect(notify.correlation_id).to eq(cmd.correlation_id) end end + + describe 'simple Consumer reactor (no Decider/Projector)' do + before do + router.register(RouterTestAuditReactor) + end + + it 'registers and processes messages through the router' do + reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + router.drain + + # Audit message should have been appended + conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + audits = store.read(conds).messages + expect(audits.size).to eq(1) + expect(audits.first).to be_a(CCCRouterTestMessages::DeviceAudited) + expect(audits.first.payload.event_type).to eq('router_test.device.registered') + end + + it 'appended messages are correlated with the source message' do + reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + router.drain + + conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + audit = store.read(conds).messages.first + + expect(audit.causation_id).to eq(reg.id) + expect(audit.correlation_id).to eq(reg.correlation_id) + end + + it 'handles multiple messages across partitions' do + store.append(CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor A' })) + store.append(CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor B' })) + + router.drain + + d1_conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + d2_conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd2') + + expect(store.read(d1_conds).messages.size).to eq(1) + expect(store.read(d2_conds).messages.size).to eq(1) + end + + it 'context_for returns empty conditions (no evolve types)' do + expect(RouterTestAuditReactor.context_for(device_id: 'd1')).to eq([]) + end + + it 'works alongside deciders and projectors' do + router.register(RouterTestDecider) + + reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + cmd = CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + store.append(cmd) + + router.drain + + # Audit reactor sees DeviceRegistered and DeviceBound + conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + audits = store.read(conds).messages + types = audits.map { |m| m.payload.event_type }.sort + + expect(types).to eq([ + 'router_test.device.bound', + 'router_test.device.registered' + ]) + end + end end From 58a3b804723810a9a34aad817e596e186e7b361e Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 16:29:50 +0000 Subject: [PATCH 016/115] Add CCC.load for synchronous reactor state loading with AND-filtered partition reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces Sourced::CCC.load(reactor_class, store, **partition_attrs) to load a reactor's evolved state from the store. Uses Store#read_partition for SQL-level AND filtering — a message is included only when every partition attribute it declares matches, avoiding loading irrelevant messages into memory. Guard's last_position covers the broader OR-context to prevent false concurrency conflicts. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 29 ++++ lib/sourced/ccc/store.rb | 77 +++++++++++ spec/sourced/ccc/load_spec.rb | 247 ++++++++++++++++++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 spec/sourced/ccc/load_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 2f5bd464..aa59b6ed 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -2,6 +2,35 @@ module Sourced module CCC + # Load a reactor instance from its event history using AND-filtered partition reads. + # Returns the evolved instance and a ReadResult (with .messages and .guard). + # + # Uses {Store#read_partition} which filters at the SQL level: a message is + # included only when every partition attribute it declares matches the given + # value. Messages that don't declare a partition attribute pass through + # (e.g. CourseCreated with only +course_id+ is included even when + # +student_id+ is in the partition). + # + # @param reactor_class [Class] a CCC reactor class (Decider, Projector, or any class + # extending CCC::Consumer that includes CCC::Evolve) + # @param store [CCC::Store] the store to read from + # @param partition_attrs [Hash{Symbol => String}] partition attribute values + # @return [Array(reactor_instance, ReadResult)] + # + # @example + # decider, read_result = Sourced::CCC.load(MyDecider, store, course_id: 'Algebra', student_id: 'joe') + # decider.state # evolved state + # read_result.guard # ConsistencyGuard for subsequent appends + def self.load(reactor_class, store, **partition_attrs) + handled_types = reactor_class.handled_messages_for_evolve.map(&:type).uniq + read_result = store.read_partition(partition_attrs, handled_types: handled_types) + + values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } + instance = reactor_class.new(values) + instance.evolve(read_result.messages) + + [instance, read_result] + end end end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index f492eb76..be75904f 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -204,6 +204,47 @@ def read(conditions, from_position: nil, limit: nil) ReadResult.new(messages: messages, guard: guard) end + # Read messages for a specific partition using AND semantics. + # A message is included only when every partition attribute it declares + # matches the given value. Messages that don't declare a partition + # attribute pass through (same logic as {#claim_next}). + # + # @param partition_attrs [Hash{Symbol|String => String}] partition attribute values + # @param handled_types [Array] message type strings to include + # @param from_position [Integer] fetch messages after this position (default 0) + # @return [ReadResult] messages and a guard for optimistic concurrency + def read_partition(partition_attrs, handled_types:, from_position: 0) + # Resolve key_pair_ids for each partition attribute + key_pair_ids = partition_attrs.filter_map do |name, value| + db[:ccc_key_pairs].where(name: name.to_s, value: value.to_s).get(:id) + end + + # If any key pair doesn't exist in the store, no messages can match + if key_pair_ids.size < partition_attrs.size + guard = ConsistencyGuard.new(conditions: [], last_position: from_position) + return ReadResult.new(messages: [], guard: guard) + end + + messages = fetch_partition_messages(key_pair_ids, from_position, handled_types) + + # Build guard conditions from handled_types, scoped to partition attrs. + # These use OR semantics so the guard detects any concurrent write + # in the broader partition context (e.g. another student enrolling). + partition_sym = partition_attrs.transform_keys(&:to_sym) + guard_conditions = handled_types.filter_map do |type| + klass = Message.registry[type] + klass&.to_conditions(**partition_sym) + end.flatten + + # The guard's last_position must cover the full OR-context, not just + # the AND-filtered messages. Otherwise a message that passes the OR + # conditions but was excluded by AND filtering would look like a conflict. + last_pos = max_position_for(guard_conditions, from_position: from_position) + + guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) + ReadResult.new(messages: messages, guard: guard) + end + # Conflict detection: returns messages matching conditions that appeared # after the given position. Empty array means no conflicts. # @@ -605,6 +646,42 @@ def check_conflicts(conditions, after_position) query_messages(conditions, from_position: after_position) end + # Max position among messages matching the given conditions (OR semantics). + # Returns from_position (or latest_position) if no matches. + # + # @param conditions [Array] + # @param from_position [Integer, nil] + # @return [Integer] + def max_position_for(conditions, from_position: nil) + return from_position || latest_position if conditions.empty? + + key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq + or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } + key_rows = db.fetch("SELECT id, name, value FROM ccc_key_pairs WHERE #{or_clauses.join(' OR ')}").all + + key_pair_index = {} + key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } + + where_parts = conditions.filter_map do |c| + kp_id = key_pair_index[[c.key_name, c.key_value]] + next unless kp_id + "(m.message_type = #{db.literal(c.message_type)} AND mkp.key_pair_id = #{db.literal(kp_id)})" + end + + return from_position || latest_position if where_parts.empty? + + sql = <<~SQL + SELECT MAX(m.position) AS max_pos + FROM ccc_messages m + JOIN ccc_message_key_pairs mkp ON m.position = mkp.message_position + WHERE (#{where_parts.join(' OR ')}) + SQL + sql += " AND m.position > #{db.literal(from_position)}" if from_position + + row = db.fetch(sql).first + row[:max_pos] || from_position || latest_position + end + # Deserialize a database row into a {PositionedMessage}. # Looks up the message class from the registry; falls back to base {Message}. # diff --git a/spec/sourced/ccc/load_spec.rb b/spec/sourced/ccc/load_spec.rb new file mode 100644 index 00000000..ae8bbc9a --- /dev/null +++ b/spec/sourced/ccc/load_spec.rb @@ -0,0 +1,247 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +module CCCLoadTestMessages + CourseCreated = Sourced::CCC::Message.define('load_test.course.created') do + attribute :course_id, String + attribute :title, String + end + + StudentEnrolled = Sourced::CCC::Message.define('load_test.student.enrolled') do + attribute :course_id, String + attribute :student_id, String + end + + AssignmentSubmitted = Sourced::CCC::Message.define('load_test.assignment.submitted') do + attribute :course_id, String + attribute :student_id, String + attribute :grade, String + end +end + +class LoadTestDecider < Sourced::CCC::Decider + partition_by :course_id, :student_id + consumer_group 'load-test-decider' + + state { |_| { enrolled: false, grades: [] } } + + evolve CCCLoadTestMessages::StudentEnrolled do |state, _evt| + state[:enrolled] = true + end + + evolve CCCLoadTestMessages::AssignmentSubmitted do |state, evt| + state[:grades] << evt.payload.grade + end +end + +class LoadTestProjector < Sourced::CCC::Projector + partition_by :course_id + consumer_group 'load-test-projector' + + state do |(course_id)| + { course_id: course_id, title: nil, student_count: 0 } + end + + evolve CCCLoadTestMessages::CourseCreated do |state, evt| + state[:title] = evt.payload.title + end + + evolve CCCLoadTestMessages::StudentEnrolled do |state, _evt| + state[:student_count] += 1 + end +end + +RSpec.describe 'Sourced::CCC.load' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + + before { store.install! } + + describe 'loading a Decider' do + before do + store.append([ + CCCLoadTestMessages::StudentEnrolled.new( + payload: { course_id: 'algebra', student_id: 'joe' } + ), + CCCLoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'A' } + ), + CCCLoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'B' } + ), + # Different partition — should not be loaded + CCCLoadTestMessages::StudentEnrolled.new( + payload: { course_id: 'algebra', student_id: 'jane' } + ) + ]) + end + + it 'returns an evolved instance and a ReadResult' do + instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + expect(instance).to be_a(LoadTestDecider) + expect(read_result).to be_a(Sourced::CCC::ReadResult) + end + + it 'evolves state from matching messages' do + instance, _read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + expect(instance.state[:enrolled]).to be true + expect(instance.state[:grades]).to eq(%w[A B]) + end + + it 'sets partition_values on the instance' do + instance, _read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + expect(instance.partition_values).to eq(%w[algebra joe]) + end + + it 'read_result contains the messages used for evolution' do + _instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + types = read_result.messages.map(&:type) + expect(types).to include('load_test.student.enrolled') + expect(types).to include('load_test.assignment.submitted') + end + + it 'read_result contains a guard for subsequent appends' do + _instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + expect(read_result.guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(read_result.guard.last_position).to be > 0 + end + + it 'guard can be used for optimistic concurrency on append' do + instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + # Append with guard succeeds when no conflicts + new_msg = CCCLoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'C' } + ) + expect { + store.append(new_msg, guard: read_result.guard) + }.not_to raise_error + + # Subsequent append with same guard fails (conflict) + another = CCCLoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'D' } + ) + expect { + store.append(another, guard: read_result.guard) + }.to raise_error(Sourced::ConcurrentAppendError) + end + + it 'excludes messages from other partitions (AND filtering at SQL level)' do + instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + # jane's StudentEnrolled shares course_id=algebra but has student_id=jane. + # AND filtering excludes it at the query level: the message declares + # student_id, and it doesn't match joe. + expect(instance.state[:enrolled]).to be true + expect(instance.state[:grades]).to eq(%w[A B]) + + # read_result.messages only contains joe's messages (3 total) + expect(read_result.messages.size).to eq(3) + student_ids = read_result.messages + .select { |m| m.payload.respond_to?(:student_id) } + .map { |m| m.payload.student_id } + .uniq + expect(student_ids).to eq(['joe']) + end + + it 'guard detects conflicts from concurrent writes to the partition' do + _instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'algebra', student_id: 'joe' + ) + + # Concurrent write to the same partition + store.append(CCCLoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'X' } + )) + + new_msg = CCCLoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'C' } + ) + expect { + store.append(new_msg, guard: read_result.guard) + }.to raise_error(Sourced::ConcurrentAppendError) + end + end + + describe 'loading a Projector' do + before do + store.append([ + CCCLoadTestMessages::CourseCreated.new( + payload: { course_id: 'algebra', title: 'Algebra 101' } + ), + CCCLoadTestMessages::StudentEnrolled.new( + payload: { course_id: 'algebra', student_id: 'joe' } + ), + CCCLoadTestMessages::StudentEnrolled.new( + payload: { course_id: 'algebra', student_id: 'jane' } + ), + # Different course — should not be loaded + CCCLoadTestMessages::CourseCreated.new( + payload: { course_id: 'physics', title: 'Physics 201' } + ) + ]) + end + + it 'evolves projector state from matching messages' do + instance, _read_result = Sourced::CCC.load( + LoadTestProjector, store, + course_id: 'algebra' + ) + + expect(instance.state[:title]).to eq('Algebra 101') + expect(instance.state[:student_count]).to eq(2) + end + + it 'passes partition values to state initializer' do + instance, _read_result = Sourced::CCC.load( + LoadTestProjector, store, + course_id: 'algebra' + ) + + expect(instance.state[:course_id]).to eq('algebra') + end + end + + describe 'empty history' do + it 'returns instance with initial state when no matching messages' do + instance, read_result = Sourced::CCC.load( + LoadTestDecider, store, + course_id: 'nonexistent', student_id: 'nobody' + ) + + expect(instance.state[:enrolled]).to be false + expect(instance.state[:grades]).to eq([]) + expect(read_result.messages).to be_empty + end + end +end From 11d0614352815111d4b3e6c9fc5b1eb8cd153149 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 18:09:09 +0000 Subject: [PATCH 017/115] Add CCC background dispatch infrastructure (Worker, Dispatcher, notifier) Signal-driven dispatch for CCC reactors: Store notifies on append/resume, NotificationQueuer routes types to reactors via WorkQueue, Workers drain partitions in bounded loops. Reuses generic primitives (WorkQueue, CatchUpPoller, InlineNotifier) with CCC-specific Worker and Dispatcher. Also adds batch_size: to Store#claim_next and Router#handle_next_for. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 2 + lib/sourced/ccc/dispatcher.rb | 179 +++++++++++++++ lib/sourced/ccc/router.rb | 5 +- lib/sourced/ccc/store.rb | 20 +- lib/sourced/ccc/worker.rb | 94 ++++++++ spec/sourced/ccc/dispatcher_spec.rb | 344 ++++++++++++++++++++++++++++ 6 files changed, 638 insertions(+), 6 deletions(-) create mode 100644 lib/sourced/ccc/dispatcher.rb create mode 100644 lib/sourced/ccc/worker.rb create mode 100644 spec/sourced/ccc/dispatcher_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index aa59b6ed..abf2f629 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -44,3 +44,5 @@ def self.load(reactor_class, store, **partition_attrs) require 'sourced/ccc/decider' require 'sourced/ccc/projector' require 'sourced/ccc/router' +require 'sourced/ccc/worker' +require 'sourced/ccc/dispatcher' diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb new file mode 100644 index 00000000..ca2f1639 --- /dev/null +++ b/lib/sourced/ccc/dispatcher.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +require 'sourced/work_queue' +require 'sourced/catchup_poller' +require 'sourced/ccc/worker' + +module Sourced + module CCC + # Orchestrator that wires together the signal-driven dispatch pipeline for CCC: + # {WorkQueue}, {NotificationQueuer}, {CatchUpPoller}, store notifier, and CCC {Worker}s. + # + # Mirrors {Sourced::Dispatcher} but uses CCC-specific interfaces: + # - +router.reactors+ instead of +router.async_reactors+ + # - +reactor.group_id+ instead of +reactor.consumer_info.group_id+ + # - +router.store.notifier+ instead of +router.backend.notifier+ + # + # Does not own the process lifecycle — the caller provides the task/fiber + # context via {#spawn_into}, and triggers shutdown via {#stop}. + # + # @example Usage with a task runner + # dispatcher = CCC::Dispatcher.new(router: ccc_router, worker_count: 4) + # executor.start do |task| + # dispatcher.spawn_into(task) + # end + # dispatcher.stop + # + # @example With custom queue for testing + # queue = WorkQueue.new(max_per_reactor: 2, queue: Queue.new) + # dispatcher = CCC::Dispatcher.new(router: ccc_router, work_queue: queue) + class Dispatcher + # Subscriber for the store notifier. Routes events to the {WorkQueue} + # by resolving message types or group IDs to reactor classes. + # + # Handles two events: + # - +'messages_appended'+ — comma-separated type strings; + # maps types to interested reactors and pushes them + # - +'reactor_resumed'+ — a consumer group ID; + # looks up the reactor and pushes it directly + class NotificationQueuer + MESSAGES_APPENDED = 'messages_appended' + REACTOR_RESUMED = 'reactor_resumed' + + # @param work_queue [WorkQueue] queue to push signaled reactors onto + # @param reactors [Array] reactor classes whose +handled_messages+ + # define the type-to-reactor mapping + def initialize(work_queue:, reactors:) + @work_queue = work_queue + @type_to_reactors = build_type_lookup(reactors) + @group_id_to_reactor = build_group_id_lookup(reactors) + end + + # Dispatch a notifier event to the appropriate handler. + # + # @param event_name [String] event name + # @param value [String] event payload + # @return [void] + def call(event_name, value) + case event_name + when MESSAGES_APPENDED + types = value.split(',').map(&:strip) + reactors = types.flat_map { |t| @type_to_reactors.fetch(t, []) }.uniq + reactors.each { |r| @work_queue.push(r) } + when REACTOR_RESUMED + reactor = @group_id_to_reactor[value] + @work_queue.push(reactor) if reactor + end + end + + private + + # @return [Hash{String => Array}] mapping from type string to reactor classes + def build_type_lookup(reactors) + lookup = Hash.new { |h, k| h[k] = [] } + reactors.each do |reactor| + reactor.handled_messages.map(&:type).uniq.each do |type| + lookup[type] << reactor + end + end + lookup + end + + # @return [Hash{String => Class}] mapping from group_id to reactor class + def build_group_id_lookup(reactors) + reactors.each_with_object({}) do |reactor, lookup| + lookup[reactor.group_id] = reactor + end + end + end + + # @return [Array] worker instances managed by this dispatcher + attr_reader :workers + + # @param router [CCC::Router] the CCC router providing reactors and store + # @param worker_count [Integer] number of worker fibers to spawn (default 2) + # @param batch_size [Integer] max messages per claim (default 50) + # @param max_drain_rounds [Integer] max drain iterations before re-enqueue (default 10) + # @param catchup_interval [Numeric] seconds between catch-up polls (default 5) + # @param work_queue [WorkQueue, nil] optional pre-built queue (useful for testing) + # @param logger [Object] logger instance + def initialize( + router:, + worker_count: 2, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 5, + work_queue: nil, + logger: Sourced.config.logger + ) + @logger = logger + @router = router + @workers = [] + + return if worker_count.zero? + + reactors = router.reactors.select { |r| r.handled_messages.any? }.to_a + + @work_queue = work_queue || WorkQueue.new(max_per_reactor: worker_count) + + @workers = worker_count.times.map do |i| + Worker.new( + work_queue: @work_queue, + router: router, + name: "worker-#{i}", + batch_size: batch_size, + max_drain_rounds: max_drain_rounds, + logger: logger + ) + end + + notification_queuer = NotificationQueuer.new(work_queue: @work_queue, reactors: reactors) + @store_notifier = router.store.notifier + @store_notifier.subscribe(notification_queuer) + + @catchup_poller = CatchUpPoller.new( + work_queue: @work_queue, + reactors: reactors, + interval: catchup_interval, + logger: logger + ) + end + + # Spawn all component fibers into the caller's task context. + # Spawns: store notifier (e.g. PG LISTEN), catch-up poller, and N workers. + # + # @param task [Object] an executor task or Async::Task to spawn fibers into + # @return [void] + def spawn_into(task) + return if @workers.empty? + + s = task.respond_to?(:spawn) ? :spawn : :async + + # Store notifier (start — no-op for InlineNotifier) + task.send(s) { @store_notifier.start } + + # CatchUp poller + task.send(s) { @catchup_poller.run } + + # Workers + @workers.each do |w| + task.send(s) { w.run } + end + end + + # Stop all components and close the work queue. + # + # @return [void] + def stop + return if @workers.empty? + + @logger.info "CCC::Dispatcher: stopping #{@workers.size} workers" + @store_notifier.stop + @catchup_poller.stop + @workers.each(&:stop) + @work_queue.close(@workers.size) + @logger.info 'CCC::Dispatcher: all components stopped' + end + end + end +end diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index c9be2d7c..7315afda 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -19,14 +19,15 @@ def register(reactor_class) @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_batch).include?(:history) end - def handle_next_for(reactor_class, worker_id: 'default') + def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) handled_types = reactor_class.handled_messages.map(&:type).uniq claim = store.claim_next( reactor_class.group_id, partition_by: reactor_class.partition_keys.map(&:to_s), handled_types: handled_types, - worker_id: worker_id + worker_id: worker_id, + batch_size: batch_size ) return false unless claim diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index be75904f..9d431ff4 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'json' +require 'sourced/inline_notifier' module Sourced module CCC @@ -40,9 +41,14 @@ class Store # @return [Sequel::SQLite::Database] attr_reader :db + # @return [Sourced::InlineNotifier] + attr_reader :notifier + # @param db [Sequel::SQLite::Database] a Sequel SQLite connection - def initialize(db) + # @param notifier [#notify_new_messages, #notify_reactor_resumed, nil] optional notifier for dispatch signals + def initialize(db, notifier: nil) @db = db + @notifier = notifier || Sourced::InlineNotifier.new @db.run('PRAGMA foreign_keys = ON') @db.run('PRAGMA journal_mode = WAL') @db.run('PRAGMA busy_timeout = 5000') @@ -181,6 +187,8 @@ def append(messages, guard: nil) end end + notifier.notify_new_messages(messages.map(&:type).uniq) + last_position end @@ -292,6 +300,7 @@ def stop_consumer_group(group_id) # @return [void] def start_consumer_group(group_id) db[:ccc_consumer_groups].where(group_id: group_id).update(status: ACTIVE, updated_at: Time.now.iso8601) + notifier.notify_reactor_resumed(group_id) end # Delete all offsets for a consumer group, resetting it to process from the beginning. @@ -324,8 +333,9 @@ def reset_consumer_group(group_id) # @param partition_by [String, Array] attribute name(s) defining partitions # @param handled_types [Array] message type strings this consumer handles # @param worker_id [String] identifier for the claiming worker + # @param batch_size [Integer, nil] max messages to fetch per claim (nil = unlimited) # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, replaying:, guard: }+ or nil - def claim_next(group_id, partition_by:, handled_types:, worker_id:) + def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: nil) partition_by = Array(partition_by).sort cg = db[:ccc_consumer_groups].where(group_id: group_id, status: ACTIVE).first return nil unless cg @@ -339,7 +349,7 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:) .where(offset_id: claimed[:offset_id]) .select_map(:key_pair_id) - messages = fetch_partition_messages(key_pair_ids, claimed[:last_position], handled_types) + messages = fetch_partition_messages(key_pair_ids, claimed[:last_position], handled_types, limit: batch_size) # If no messages pass the conditional AND filter, release and return nil if messages.empty? @@ -560,8 +570,9 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) # @param key_pair_ids [Array] partition key_pair IDs # @param last_position [Integer] fetch messages after this position # @param handled_types [Array] message type strings + # @param limit [Integer, nil] max messages to return (nil = unlimited) # @return [Array] - def fetch_partition_messages(key_pair_ids, last_position, handled_types) + def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: nil) return [] if key_pair_ids.empty? kp_ids_list = key_pair_ids.map { |id| db.literal(id) }.join(', ') @@ -591,6 +602,7 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types) ) ORDER BY m.position ASC SQL + sql += " LIMIT #{db.literal(limit)}" if limit db.fetch(sql).map { |row| deserialize(row) } end diff --git a/lib/sourced/ccc/worker.rb b/lib/sourced/ccc/worker.rb new file mode 100644 index 00000000..62f1f8db --- /dev/null +++ b/lib/sourced/ccc/worker.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # Processes CCC reactors from a {WorkQueue} in a signal-driven drain loop. + # + # Mirrors {Sourced::Worker} but calls {CCC::Router#handle_next_for} instead + # of the Sourced-specific +handle_next_event_for_reactor+. + # + # @example Signal-driven mode (production) + # queue = Sourced::WorkQueue.new(max_per_reactor: 4) + # worker = CCC::Worker.new(work_queue: queue, router: router, name: 'worker-0') + # worker.run # blocks, processing reactors popped from queue + # + # @example Single-tick for testing + # worker = CCC::Worker.new(work_queue: queue, router: router, name: 'test') + # worker.tick(MyReactor) # => true if messages were processed + class Worker + # @return [String] unique identifier for this worker instance + attr_reader :name + + # @param work_queue [WorkQueue] queue to receive reactor signals from + # @param router [CCC::Router] CCC router for dispatching messages + # @param name [String] unique name for this worker + # @param batch_size [Integer] max messages per claim + # @param max_drain_rounds [Integer] max consecutive drain iterations per reactor pickup + # @param logger [Object] logger instance + def initialize( + work_queue:, + router:, + name: SecureRandom.hex(4), + batch_size: 50, + max_drain_rounds: 10, + logger: Sourced.config.logger + ) + @work_queue = work_queue + @router = router + @name = [Process.pid, name].join('-') + @batch_size = batch_size + @max_drain_rounds = max_drain_rounds + @logger = logger + @running = false + end + + # Signal the worker to stop after the current drain completes. + # @return [void] + def stop + @running = false + end + + # Main run loop. Blocks on the {WorkQueue} waiting for reactor signals. + # @return [void] + def run + @running = true + + while @running + reactor = @work_queue.pop + break if reactor.nil? # shutdown sentinel + + drain(reactor) + end + + @logger.info "CCC::Worker #{name}: stopped" + end + + # Drain available messages for a reactor in a bounded loop. + # + # Processes up to +max_drain_rounds+ batches. If all rounds are consumed, + # re-enqueues the reactor for fair scheduling across all reactors. + # + # @param reactor [Class] reactor class to drain messages for + # @return [void] + def drain(reactor) + rounds = 0 + while @running && rounds < @max_drain_rounds + found = @router.handle_next_for(reactor, worker_id: name, batch_size: @batch_size) + break unless found + + rounds += 1 + end + # More work likely — re-enqueue so another worker (or this one) continues + @work_queue.push(reactor) if @running && rounds >= @max_drain_rounds + end + + # Process one tick of work for a specific reactor. Convenience for testing. + # + # @param reactor [Class] reactor class to process + # @return [Boolean] true if messages were processed, false otherwise + def tick(reactor) + @router.handle_next_for(reactor, worker_id: name, batch_size: @batch_size) + end + end + end +end diff --git a/spec/sourced/ccc/dispatcher_spec.rb b/spec/sourced/ccc/dispatcher_spec.rb new file mode 100644 index 00000000..54fbe727 --- /dev/null +++ b/spec/sourced/ccc/dispatcher_spec.rb @@ -0,0 +1,344 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +module CCCDispatcherTestMessages + DeviceRegistered = Sourced::CCC::Message.define('dispatch_test.device.registered') do + attribute :device_id, String + attribute :name, String + end + + DeviceBound = Sourced::CCC::Message.define('dispatch_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDevice = Sourced::CCC::Message.define('dispatch_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end +end + +class DispatchTestDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'dispatch-test-decider' + + state { |_| { exists: false, bound: false } } + + evolve CCCDispatcherTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + evolve CCCDispatcherTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command CCCDispatcherTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event CCCDispatcherTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end +end + +class DispatchTestProjector < Sourced::CCC::Projector + partition_by :device_id + consumer_group 'dispatch-test-projector' + + state { |_| { devices: [] } } + + evolve CCCDispatcherTestMessages::DeviceRegistered do |state, evt| + state[:devices] << evt.payload.name + end + + evolve CCCDispatcherTestMessages::DeviceBound do |state, _evt| + # noop + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + end +end + +RSpec.describe Sourced::CCC::Dispatcher do + let(:db) { Sequel.sqlite } + let(:notifier) { Sourced::InlineNotifier.new } + let(:store) { Sourced::CCC::Store.new(db, notifier: notifier) } + let(:router) { Sourced::CCC::Router.new(store: store) } + let(:logger) { instance_double('Logger', info: nil, warn: nil, debug: nil) } + let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } + + before do + store.install! + router.register(DispatchTestDecider) + router.register(DispatchTestProjector) + end + + describe 'Store notifications' do + it 'append triggers notify_new_messages' do + expect(notifier).to receive(:notify_new_messages).with(['dispatch_test.device.registered']) + + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + end + + it 'start_consumer_group triggers notify_reactor_resumed' do + store.stop_consumer_group('dispatch-test-decider') + + expect(notifier).to receive(:notify_reactor_resumed).with('dispatch-test-decider') + + store.start_consumer_group('dispatch-test-decider') + end + + it 'empty append does not notify' do + expect(notifier).not_to receive(:notify_new_messages) + + store.append([]) + end + end + + describe 'batch_size' do + it 'claim_next with batch_size limits returned messages' do + # Append 5 messages for the same partition + 5.times do |i| + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: "Sensor #{i}" }) + ) + end + + claim = store.claim_next( + 'dispatch-test-projector', + partition_by: ['device_id'], + handled_types: DispatchTestProjector.handled_messages.map(&:type), + worker_id: 'w1', + batch_size: 2 + ) + + expect(claim).not_to be_nil + expect(claim.messages.size).to eq(2) + end + + it 'claim_next without batch_size returns all messages' do + 5.times do |i| + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: "Sensor #{i}" }) + ) + end + + claim = store.claim_next( + 'dispatch-test-projector', + partition_by: ['device_id'], + handled_types: DispatchTestProjector.handled_messages.map(&:type), + worker_id: 'w1' + ) + + expect(claim).not_to be_nil + expect(claim.messages.size).to eq(5) + end + end + + describe Sourced::CCC::Dispatcher::NotificationQueuer do + let(:queuer) do + described_class.new( + work_queue: work_queue, + reactors: [DispatchTestDecider, DispatchTestProjector] + ) + end + + it 'maps message types to interested reactors' do + # DeviceRegistered is handled by projector (via evolve), + # BindDevice is handled by decider (via command) + queuer.call('messages_appended', 'dispatch_test.device.registered,dispatch_test.bind_device') + + popped = [] + popped << work_queue.pop + popped << work_queue.pop + + expect(popped).to contain_exactly(DispatchTestDecider, DispatchTestProjector) + end + + it 'maps group_id to reactor for reactor_resumed' do + queuer.call('reactor_resumed', 'dispatch-test-decider') + + popped = work_queue.pop + expect(popped).to eq(DispatchTestDecider) + end + + it 'ignores unknown message types' do + queuer.call('messages_appended', 'unknown.type') + + # Queue should be empty — push a sentinel to avoid blocking + work_queue.push(nil) + expect(work_queue.pop).to be_nil + end + + it 'ignores unknown group_ids' do + queuer.call('reactor_resumed', 'unknown-group') + + work_queue.push(nil) + expect(work_queue.pop).to be_nil + end + end + + describe Sourced::CCC::Worker do + let(:worker) do + described_class.new( + work_queue: work_queue, + router: router, + name: 'test-worker', + batch_size: 50, + max_drain_rounds: 10, + logger: logger + ) + end + + describe '#tick' do + it 'processes one claim for a reactor' do + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + result = worker.tick(DispatchTestProjector) + expect(result).to be true + end + + it 'returns false when no work available' do + result = worker.tick(DispatchTestDecider) + expect(result).to be false + end + end + + describe '#drain' do + it 'processes until no more work for reactor' do + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor A' }) + ) + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor B' }) + ) + + worker.instance_variable_set(:@running, true) + worker.drain(DispatchTestProjector) + + # Both partitions should have been processed — no more work + result = worker.tick(DispatchTestProjector) + expect(result).to be false + end + + it 're-enqueues reactor when max_drain_rounds reached' do + # Create a worker with max_drain_rounds: 1 + bounded_worker = described_class.new( + work_queue: work_queue, + router: router, + name: 'bounded', + batch_size: 50, + max_drain_rounds: 1, + logger: logger + ) + + # Append messages for 2 partitions + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'A' }) + ) + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'B' }) + ) + + bounded_worker.instance_variable_set(:@running, true) + bounded_worker.drain(DispatchTestProjector) + + # Should have re-enqueued — pop it back + popped = work_queue.pop + expect(popped).to eq(DispatchTestProjector) + end + end + end + + describe 'Dispatcher wiring' do + subject(:dispatcher) do + described_class.new( + router: router, + worker_count: 2, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 5, + work_queue: work_queue, + logger: logger + ) + end + + it 'creates the requested number of workers' do + expect(dispatcher.workers.size).to eq(2) + end + + it 'creates workers with correct names' do + names = dispatcher.workers.map(&:name) + expect(names).to include(match(/worker-0$/)) + expect(names).to include(match(/worker-1$/)) + end + + it 'spawns via #spawn when task responds to spawn' do + task = double('Task') + # 1 notifier + 1 catchup_poller + 2 workers = 4 spawns + expect(task).to receive(:spawn).exactly(4).times + dispatcher.spawn_into(task) + end + + it 'spawns via #async when task does not respond to spawn' do + task = Object.new + def task.async; end + expect(task).to receive(:async).exactly(4).times + dispatcher.spawn_into(task) + end + + it '#stop stops all components' do + dispatcher.stop + + dispatcher.workers.each do |w| + expect(w.instance_variable_get(:@running)).to eq(false) + end + end + + it 'creates zero workers when worker_count is 0' do + d = described_class.new( + router: router, + worker_count: 0, + logger: logger + ) + expect(d.workers).to be_empty + end + end + + describe 'Integration: append → notify → queue → worker' do + it 'InlineNotifier fires synchronously through the full pipeline' do + # Build dispatcher which subscribes NotificationQueuer to the store's notifier + dispatcher = described_class.new( + router: router, + worker_count: 1, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 60, # long interval — we test synchronous path only + work_queue: work_queue, + logger: logger + ) + + # Append triggers notifier → NotificationQueuer → WorkQueue + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + # Pop from queue — should have the reactors that handle this type + popped = work_queue.pop + expect([DispatchTestDecider, DispatchTestProjector]).to include(popped) + + # Worker processes the message + worker = dispatcher.workers.first + result = worker.tick(popped) + expect(result).to be true + + dispatcher.stop + end + end +end From 28c63b7b12c862c474d43c43a213fa34a243c5b3 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 18:59:50 +0000 Subject: [PATCH 018/115] Add CCC stale claim reaper with worker heartbeats Crashed workers leave partitions permanently claimed. Add a ccc_workers table with heartbeat upserts and a StaleClaimReaper that periodically releases claims from workers that stopped heartbeating. Wire the reaper into the Dispatcher alongside the existing notifier and catchup poller. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 1 + lib/sourced/ccc/dispatcher.rb | 17 ++ lib/sourced/ccc/stale_claim_reaper.rb | 77 +++++++ lib/sourced/ccc/store.rb | 49 ++++- spec/sourced/ccc/dispatcher_spec.rb | 6 +- spec/sourced/ccc/stale_claim_reaper_spec.rb | 219 ++++++++++++++++++++ 6 files changed, 365 insertions(+), 4 deletions(-) create mode 100644 lib/sourced/ccc/stale_claim_reaper.rb create mode 100644 spec/sourced/ccc/stale_claim_reaper_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index abf2f629..4cc0957e 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -45,4 +45,5 @@ def self.load(reactor_class, store, **partition_attrs) require 'sourced/ccc/projector' require 'sourced/ccc/router' require 'sourced/ccc/worker' +require 'sourced/ccc/stale_claim_reaper' require 'sourced/ccc/dispatcher' diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb index ca2f1639..a88bd329 100644 --- a/lib/sourced/ccc/dispatcher.rb +++ b/lib/sourced/ccc/dispatcher.rb @@ -3,6 +3,7 @@ require 'sourced/work_queue' require 'sourced/catchup_poller' require 'sourced/ccc/worker' +require 'sourced/ccc/stale_claim_reaper' module Sourced module CCC @@ -95,6 +96,8 @@ def build_group_id_lookup(reactors) # @param batch_size [Integer] max messages per claim (default 50) # @param max_drain_rounds [Integer] max drain iterations before re-enqueue (default 10) # @param catchup_interval [Numeric] seconds between catch-up polls (default 5) + # @param housekeeping_interval [Numeric] seconds between heartbeat/reap cycles (default 30) + # @param claim_ttl_seconds [Integer] stale claim age threshold in seconds (default 120) # @param work_queue [WorkQueue, nil] optional pre-built queue (useful for testing) # @param logger [Object] logger instance def initialize( @@ -103,6 +106,8 @@ def initialize( batch_size: 50, max_drain_rounds: 10, catchup_interval: 5, + housekeeping_interval: 30, + claim_ttl_seconds: 120, work_queue: nil, logger: Sourced.config.logger ) @@ -137,6 +142,14 @@ def initialize( interval: catchup_interval, logger: logger ) + + @stale_claim_reaper = StaleClaimReaper.new( + store: router.store, + interval: housekeeping_interval, + ttl_seconds: claim_ttl_seconds, + worker_ids_provider: -> { @workers.map(&:name) }, + logger: logger + ) end # Spawn all component fibers into the caller's task context. @@ -155,6 +168,9 @@ def spawn_into(task) # CatchUp poller task.send(s) { @catchup_poller.run } + # Stale claim reaper + task.send(s) { @stale_claim_reaper.run } + # Workers @workers.each do |w| task.send(s) { w.run } @@ -170,6 +186,7 @@ def stop @logger.info "CCC::Dispatcher: stopping #{@workers.size} workers" @store_notifier.stop @catchup_poller.stop + @stale_claim_reaper.stop @workers.each(&:stop) @work_queue.close(@workers.size) @logger.info 'CCC::Dispatcher: all components stopped' diff --git a/lib/sourced/ccc/stale_claim_reaper.rb b/lib/sourced/ccc/stale_claim_reaper.rb new file mode 100644 index 00000000..bf7dd024 --- /dev/null +++ b/lib/sourced/ccc/stale_claim_reaper.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # Periodic loop that heartbeats active workers and releases claims + # held by workers that have stopped heartbeating (crashed or killed). + # + # Combines heartbeating and reaping in one loop since CCC doesn't have + # a separate HouseKeeper like the main Sourced module. + # + # +worker_ids_provider+ is a proc that returns current worker names — + # injected by the {Dispatcher} which owns the Worker instances. + # + # @example + # reaper = StaleClaimReaper.new( + # store: store, + # interval: 30, + # ttl_seconds: 120, + # worker_ids_provider: -> { workers.map(&:name) }, + # logger: logger + # ) + # # In a fiber/thread: + # reaper.run # blocks, heartbeating + reaping every 30s + # # From another fiber/thread: + # reaper.stop # breaks the loop + class StaleClaimReaper + # @param store [CCC::Store] the CCC store + # @param interval [Numeric] seconds between heartbeat/reap cycles (default 30) + # @param ttl_seconds [Integer] age threshold for stale claims (default 120) + # @param worker_ids_provider [Proc] returns Array of active worker IDs + # @param logger [Object] logger instance + def initialize(store:, interval: 30, ttl_seconds: 120, worker_ids_provider: -> { [] }, logger: Sourced.config.logger) + @store = store + @interval = interval + @ttl_seconds = ttl_seconds + @worker_ids_provider = worker_ids_provider + @logger = logger + @running = false + end + + # Run the heartbeat/reap loop. Blocks until {#stop} is called. + # Reaps on startup (from previous runs where workers were killed). + # + # @return [void] + def run + @running = true + reap # reap on startup for claims left by previously killed workers + while @running + sleep @interval + heartbeat if @running + reap if @running + end + @logger.info 'CCC::StaleClaimReaper: stopped' + end + + # Signal the reaper to stop after the current sleep cycle. + # + # @return [void] + def stop + @running = false + end + + private + + def heartbeat + ids = Array(@worker_ids_provider.call).uniq + count = @store.worker_heartbeat(ids) + @logger.debug "CCC::StaleClaimReaper: heartbeated #{count} workers" if count > 0 + end + + def reap + released = @store.release_stale_claims(ttl_seconds: @ttl_seconds) + @logger.info "CCC::StaleClaimReaper: released #{released} stale claims" if released > 0 + end + end + end +end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 9d431ff4..03a0c926 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -62,7 +62,8 @@ def installed? db.table_exists?(:ccc_message_key_pairs) && db.table_exists?(:ccc_consumer_groups) && db.table_exists?(:ccc_offsets) && - db.table_exists?(:ccc_offset_key_pairs) + db.table_exists?(:ccc_offset_key_pairs) && + db.table_exists?(:ccc_workers) end # Create all required tables and indexes. Idempotent. @@ -134,6 +135,13 @@ def install! PRIMARY KEY (offset_id, key_pair_id) ) SQL + + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_workers ( + id TEXT PRIMARY KEY, + last_seen TEXT NOT NULL + ) + SQL end # Append messages to the store. Extracts and indexes key-value pairs @@ -436,6 +444,44 @@ def release(group_id, offset_id:) ) end + # Upsert heartbeat timestamps for active workers. + # + # @param worker_ids [Array] worker identifiers + # @param at [Time] timestamp to record (default Time.now) + # @return [Integer] number of workers heartbeated + def worker_heartbeat(worker_ids, at: Time.now) + ids = Array(worker_ids).uniq + return 0 if ids.empty? + + now = at.iso8601 + ids.each do |id| + db.run(<<~SQL) + INSERT INTO ccc_workers (id, last_seen) VALUES (#{db.literal(id)}, #{db.literal(now)}) + ON CONFLICT(id) DO UPDATE SET last_seen = #{db.literal(now)} + SQL + end + ids.size + end + + # Release claims held by workers that haven't heartbeated within ttl_seconds. + # + # @param ttl_seconds [Integer] age threshold + # @return [Integer] number of claims released + def release_stale_claims(ttl_seconds: 120) + cutoff = (Time.now - ttl_seconds).iso8601 + + stale_worker_ids = db[:ccc_workers] + .where(Sequel.lit('last_seen <= ?', cutoff)) + .select_map(:id) + + return 0 if stale_worker_ids.empty? + + db[:ccc_offsets] + .where(claimed: 1) + .where(claimed_by: stale_worker_ids) + .update(claimed: 0, claimed_at: nil, claimed_by: nil) + end + # Current max position in the message log. # # @return [Integer] max position, or 0 if the store is empty @@ -453,6 +499,7 @@ def clear! db[:ccc_message_key_pairs].delete db[:ccc_key_pairs].delete db[:ccc_messages].delete + db[:ccc_workers].delete db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) end diff --git a/spec/sourced/ccc/dispatcher_spec.rb b/spec/sourced/ccc/dispatcher_spec.rb index 54fbe727..b1ec83fa 100644 --- a/spec/sourced/ccc/dispatcher_spec.rb +++ b/spec/sourced/ccc/dispatcher_spec.rb @@ -281,15 +281,15 @@ class DispatchTestProjector < Sourced::CCC::Projector it 'spawns via #spawn when task responds to spawn' do task = double('Task') - # 1 notifier + 1 catchup_poller + 2 workers = 4 spawns - expect(task).to receive(:spawn).exactly(4).times + # 1 notifier + 1 catchup_poller + 1 stale_claim_reaper + 2 workers = 5 spawns + expect(task).to receive(:spawn).exactly(5).times dispatcher.spawn_into(task) end it 'spawns via #async when task does not respond to spawn' do task = Object.new def task.async; end - expect(task).to receive(:async).exactly(4).times + expect(task).to receive(:async).exactly(5).times dispatcher.spawn_into(task) end diff --git a/spec/sourced/ccc/stale_claim_reaper_spec.rb b/spec/sourced/ccc/stale_claim_reaper_spec.rb new file mode 100644 index 00000000..b78e490b --- /dev/null +++ b/spec/sourced/ccc/stale_claim_reaper_spec.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +module StaleClaimReaperTestMessages + DeviceRegistered = Sourced::CCC::Message.define('reaper_test.device.registered') do + attribute :device_id, String + attribute :name, String + end +end + +class ReaperTestProjector < Sourced::CCC::Projector + partition_by :device_id + consumer_group 'reaper-test-projector' + + state { |_| { devices: [] } } + + evolve StaleClaimReaperTestMessages::DeviceRegistered do |state, evt| + state[:devices] << evt.payload.name + end +end + +RSpec.describe 'Store worker heartbeat and stale claim release' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + + before { store.install! } + + describe '#worker_heartbeat' do + it 'inserts workers with last_seen' do + count = store.worker_heartbeat(['w1', 'w2']) + expect(count).to eq(2) + + rows = db[:ccc_workers].all + expect(rows.size).to eq(2) + expect(rows.map { |r| r[:id] }).to contain_exactly('w1', 'w2') + rows.each { |r| expect(r[:last_seen]).not_to be_nil } + end + + it 'updates existing workers last_seen' do + early = Time.now - 300 + store.worker_heartbeat(['w1'], at: early) + + later = Time.now + store.worker_heartbeat(['w1'], at: later) + + row = db[:ccc_workers].where(id: 'w1').first + expect(row[:last_seen]).to eq(later.iso8601) + end + + it 'deduplicates worker IDs' do + count = store.worker_heartbeat(['w1', 'w1', 'w1']) + expect(count).to eq(1) + expect(db[:ccc_workers].count).to eq(1) + end + + it 'returns 0 for empty array' do + count = store.worker_heartbeat([]) + expect(count).to eq(0) + end + end + + describe '#release_stale_claims' do + before do + store.register_consumer_group('reaper-test-projector') + + # Append a message to create a partition + store.append( + StaleClaimReaperTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + # Claim the partition + @claim = store.claim_next( + 'reaper-test-projector', + partition_by: ['device_id'], + handled_types: ['reaper_test.device.registered'], + worker_id: 'w1' + ) + end + + it 'releases claims from stale workers' do + # Heartbeat far in the past + store.worker_heartbeat(['w1'], at: Time.now - 300) + + released = store.release_stale_claims(ttl_seconds: 120) + expect(released).to eq(1) + + # Verify the offset is unclaimed + offset = db[:ccc_offsets].where(id: @claim.offset_id).first + expect(offset[:claimed]).to eq(0) + expect(offset[:claimed_at]).to be_nil + expect(offset[:claimed_by]).to be_nil + end + + it 'leaves claims from recently-heartbeated workers' do + # Heartbeat just now + store.worker_heartbeat(['w1'], at: Time.now) + + released = store.release_stale_claims(ttl_seconds: 120) + expect(released).to eq(0) + + # Verify the offset is still claimed + offset = db[:ccc_offsets].where(id: @claim.offset_id).first + expect(offset[:claimed]).to eq(1) + expect(offset[:claimed_by]).to eq('w1') + end + + it 'returns 0 when no stale claims exist' do + released = store.release_stale_claims(ttl_seconds: 120) + # No heartbeat for w1, so no record in ccc_workers, so no stale workers found + expect(released).to eq(0) + end + + it 'only releases claims from stale workers, not healthy ones' do + # Append another message for a different partition + store.append( + StaleClaimReaperTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor 2' }) + ) + + # Ack the first claim so d1 partition is free, then claim d2 with w2 + store.ack('reaper-test-projector', offset_id: @claim.offset_id, position: @claim.messages.last.position) + + claim2 = store.claim_next( + 'reaper-test-projector', + partition_by: ['device_id'], + handled_types: ['reaper_test.device.registered'], + worker_id: 'w2' + ) + + # Re-claim d1 with w1 + claim1 = store.claim_next( + 'reaper-test-projector', + partition_by: ['device_id'], + handled_types: ['reaper_test.device.registered'], + worker_id: 'w1' + ) + + # w1 is stale, w2 is healthy + store.worker_heartbeat(['w1'], at: Time.now - 300) + store.worker_heartbeat(['w2'], at: Time.now) + + released = store.release_stale_claims(ttl_seconds: 120) + + # Only w1's claims should be released + if claim1 + expect(released).to eq(1) + offset1 = db[:ccc_offsets].where(id: claim1.offset_id).first + expect(offset1[:claimed]).to eq(0) + end + + offset2 = db[:ccc_offsets].where(id: claim2.offset_id).first + expect(offset2[:claimed]).to eq(1) + expect(offset2[:claimed_by]).to eq('w2') + end + end +end + +RSpec.describe Sourced::CCC::StaleClaimReaper do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + let(:logger) { instance_double('Logger', info: nil, warn: nil, debug: nil) } + + before { store.install! } + + describe '#heartbeat' do + it 'calls worker_heartbeat with worker IDs from provider' do + reaper = described_class.new( + store: store, + interval: 30, + ttl_seconds: 120, + worker_ids_provider: -> { ['w-0', 'w-1'] }, + logger: logger + ) + + reaper.send(:heartbeat) + + rows = db[:ccc_workers].all + expect(rows.size).to eq(2) + expect(rows.map { |r| r[:id] }).to contain_exactly('w-0', 'w-1') + end + end + + describe '#reap' do + it 'calls release_stale_claims with configured TTL' do + reaper = described_class.new( + store: store, + interval: 30, + ttl_seconds: 60, + logger: logger + ) + + expect(store).to receive(:release_stale_claims).with(ttl_seconds: 60).and_return(0) + reaper.send(:reap) + end + end + + describe '#run and #stop' do + it 'reaps on startup then stops' do + reaper = described_class.new( + store: store, + interval: 0.01, + ttl_seconds: 120, + worker_ids_provider: -> { ['w-0'] }, + logger: logger + ) + + expect(store).to receive(:release_stale_claims).at_least(:once).and_return(0) + + thread = Thread.new { reaper.run } + sleep 0.05 + reaper.stop + thread.join(1) + + expect(logger).to have_received(:info).with('CCC::StaleClaimReaper: stopped') + end + end +end From 0cce1b5797fb651a2d54c2d452dc91e2b6402f7c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 20:46:42 +0000 Subject: [PATCH 019/115] Add CCC::Supervisor as top-level process entry point Mirrors Sourced::Supervisor but simpler: no separate HouseKeepers since StaleClaimReaper is already embedded in the CCC Dispatcher. Takes router + config kwargs, sets signal handlers, and spawns Dispatcher into an executor. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 1 + lib/sourced/ccc/supervisor.rb | 101 +++++++++++++++++ spec/sourced/ccc/supervisor_spec.rb | 162 ++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 lib/sourced/ccc/supervisor.rb create mode 100644 spec/sourced/ccc/supervisor_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 4cc0957e..fd1d33a9 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -47,3 +47,4 @@ def self.load(reactor_class, store, **partition_attrs) require 'sourced/ccc/worker' require 'sourced/ccc/stale_claim_reaper' require 'sourced/ccc/dispatcher' +require 'sourced/ccc/supervisor' diff --git a/lib/sourced/ccc/supervisor.rb b/lib/sourced/ccc/supervisor.rb new file mode 100644 index 00000000..80bc67df --- /dev/null +++ b/lib/sourced/ccc/supervisor.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'sourced/ccc/dispatcher' + +module Sourced + module CCC + # Top-level process entry point for CCC background workers. + # Creates a {Dispatcher} (which embeds Workers, CatchUpPoller, notifier, + # and StaleClaimReaper) and spawns it into an executor. + # + # Mirrors {Sourced::Supervisor} but simpler: no separate HouseKeepers, + # since housekeeping (heartbeat + stale claim reaping) is embedded in + # the CCC Dispatcher's StaleClaimReaper. + # + # @example Start with defaults + # CCC::Supervisor.start(router: my_ccc_router) + # + # @example Create and start manually + # supervisor = CCC::Supervisor.new(router: my_ccc_router, count: 4) + # supervisor.start + class Supervisor + # Start a new supervisor instance with the given options. + # + # @param args [Hash] Arguments passed to {#initialize} + # @return [void] This method blocks until the supervisor is stopped + def self.start(...) + new(...).start + end + + # @param router [CCC::Router] the CCC router providing reactors and store + # @param logger [Object] Logger instance for supervisor output + # @param count [Integer] Number of worker fibers to spawn + # @param batch_size [Integer] Messages per backend fetch + # @param max_drain_rounds [Integer] Max drain iterations per reactor pickup + # @param catchup_interval [Numeric] Seconds between catch-up polls + # @param housekeeping_interval [Numeric] Seconds between heartbeat/reap cycles + # @param claim_ttl_seconds [Integer] Stale claim age threshold in seconds + # @param executor [Object] Executor instance for running concurrent workers + def initialize( + router:, + logger: Sourced.config.logger, + count: 2, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 5, + housekeeping_interval: 30, + claim_ttl_seconds: 120, + executor: Sourced.config.executor + ) + @router = router + @logger = logger + @count = count + @batch_size = batch_size + @max_drain_rounds = max_drain_rounds + @catchup_interval = catchup_interval + @housekeeping_interval = housekeeping_interval + @claim_ttl_seconds = claim_ttl_seconds + @executor = executor + end + + # Start the supervisor and dispatcher. + # This method blocks until the supervisor receives a shutdown signal. + def start + logger.info("CCC::Supervisor: starting with #{@count} workers and #{@executor} executor") + set_signal_handlers + + @dispatcher = Dispatcher.new( + router: @router, + worker_count: @count, + batch_size: @batch_size, + max_drain_rounds: @max_drain_rounds, + catchup_interval: @catchup_interval, + housekeeping_interval: @housekeeping_interval, + claim_ttl_seconds: @claim_ttl_seconds, + logger: logger + ) + + @executor.start do |task| + @dispatcher.spawn_into(task) + end + end + + # Stop all components gracefully. + def stop + logger.info('CCC::Supervisor: stopping dispatcher') + @dispatcher&.stop + logger.info('CCC::Supervisor: all workers stopped') + end + + # Set up signal handlers for graceful shutdown. + def set_signal_handlers + Signal.trap('INT') { stop } + Signal.trap('TERM') { stop } + end + + private + + attr_reader :logger + end + end +end diff --git a/spec/sourced/ccc/supervisor_spec.rb b/spec/sourced/ccc/supervisor_spec.rb new file mode 100644 index 00000000..49af5808 --- /dev/null +++ b/spec/sourced/ccc/supervisor_spec.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +RSpec.describe Sourced::CCC::Supervisor do + let(:executor) { double('Executor') } + let(:logger) { instance_double('Logger', info: nil, warn: nil) } + let(:store_notifier) { Sourced::InlineNotifier.new } + let(:store) { double('Store', notifier: store_notifier) } + let(:reactor1) { double('Reactor1', handled_messages: [double(type: 'event1')], group_id: 'Reactor1', partition_keys: [:id]) } + let(:reactors) { [reactor1] } + let(:router) { instance_double(Sourced::CCC::Router, store: store, reactors: reactors) } + let(:task) { double('Task', spawn: nil) } + let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } + + before do + allow(Sourced::WorkQueue).to receive(:new).and_return(work_queue) + allow(executor).to receive(:start).and_yield(task) + allow(Signal).to receive(:trap) + end + + describe '.start' do + it 'creates a new supervisor instance and starts it' do + supervisor_instance = instance_double(described_class) + expect(described_class).to receive(:new).with( + router: router, + logger: logger, + count: 3 + ).and_return(supervisor_instance) + expect(supervisor_instance).to receive(:start) + + described_class.start(router: router, logger: logger, count: 3) + end + end + + describe '#start' do + subject(:supervisor) do + described_class.new( + router: router, + logger: logger, + count: 2, + executor: executor + ) + end + + it 'sets up INT and TERM signal handlers' do + expect(Signal).to receive(:trap).with('INT') + expect(Signal).to receive(:trap).with('TERM') + supervisor.start + end + + it 'creates Dispatcher with correct params' do + expect(Sourced::CCC::Dispatcher).to receive(:new).with( + router: router, + worker_count: 2, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 5, + housekeeping_interval: 30, + claim_ttl_seconds: 120, + logger: logger + ).and_call_original + + supervisor.start + end + + it 'passes custom params through to Dispatcher' do + custom_supervisor = described_class.new( + router: router, + logger: logger, + count: 4, + batch_size: 100, + max_drain_rounds: 20, + catchup_interval: 10, + housekeeping_interval: 60, + claim_ttl_seconds: 300, + executor: executor + ) + + expect(Sourced::CCC::Dispatcher).to receive(:new).with( + router: router, + worker_count: 4, + batch_size: 100, + max_drain_rounds: 20, + catchup_interval: 10, + housekeeping_interval: 60, + claim_ttl_seconds: 300, + logger: logger + ).and_call_original + + custom_supervisor.start + end + + it 'spawns via executor (notifier + catchup + reaper + 2 workers = 5 spawns)' do + expect(executor).to receive(:start).and_yield(task) + # 1 notifier + 1 catchup_poller + 1 stale_claim_reaper + 2 workers = 5 spawns + expect(task).to receive(:spawn).exactly(5).times + + supervisor.start + end + end + + describe '#stop' do + subject(:supervisor) do + described_class.new( + router: router, + logger: logger, + count: 2, + executor: executor + ) + end + + before do + supervisor.start + end + + it 'logs shutdown information' do + expect(logger).to receive(:info).with('CCC::Supervisor: stopping dispatcher') + expect(logger).to receive(:info).with('CCC::Supervisor: all workers stopped') + # Dispatcher also logs + allow(logger).to receive(:info) + supervisor.stop + end + + it 'stops the dispatcher' do + dispatcher = supervisor.instance_variable_get(:@dispatcher) + expect(dispatcher).to receive(:stop) + # Suppress logs from Supervisor#stop + allow(logger).to receive(:info) + supervisor.stop + end + end + + describe 'signal handling' do + subject(:supervisor) do + described_class.new( + router: router, + logger: logger, + executor: executor + ) + end + + it 'traps INT and TERM signals to call stop' do + int_handler = nil + term_handler = nil + + allow(Signal).to receive(:trap) do |signal, &block| + int_handler = block if signal == 'INT' + term_handler = block if signal == 'TERM' + end + + supervisor.start + + expect(supervisor).to receive(:stop) + int_handler.call + + expect(supervisor).to receive(:stop) + term_handler.call + end + end +end From 0f5e3119ba39016d2c4586a6ffaebf0f5fd9e1d0 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 21:54:51 +0000 Subject: [PATCH 020/115] Add CCC::Configuration and module-level API (config, configure, register, store, router, reset!) Wire CCC components (Supervisor, Dispatcher, Worker, Consumer, StaleClaimReaper) to pull defaults from CCC.config instead of Sourced.config, keeping executor on Sourced. Change CCC.load signature from positional store to keyword store: defaulting to global. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 46 +++++- lib/sourced/ccc/configuration.rb | 55 +++++++ lib/sourced/ccc/consumer.rb | 2 +- lib/sourced/ccc/dispatcher.rb | 2 +- lib/sourced/ccc/stale_claim_reaper.rb | 2 +- lib/sourced/ccc/supervisor.rb | 16 +- lib/sourced/ccc/worker.rb | 2 +- spec/sourced/ccc/configuration_spec.rb | 198 +++++++++++++++++++++++++ spec/sourced/ccc/load_spec.rb | 22 +-- spec/sourced/ccc/supervisor_spec.rb | 4 + 10 files changed, 323 insertions(+), 26 deletions(-) create mode 100644 lib/sourced/ccc/configuration.rb create mode 100644 spec/sourced/ccc/configuration_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index fd1d33a9..f1c8be4f 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -2,6 +2,44 @@ module Sourced module CCC + # @return [Configuration] the global CCC configuration instance + def self.config + @config ||= Configuration.new + end + + # Configure the CCC module. Calls setup! after yielding. + # @yieldparam config [Configuration] + def self.configure(&) + yield config if block_given? + config.setup! + config.freeze + end + + # Register a reactor class with the global router. + # Triggers setup! if not already done. + # @param reactor [Class] a CCC reactor class + def self.register(reactor) + config.setup! + config.router.register(reactor) + end + + # @return [CCC::Store] the global store (triggers setup! if needed) + def self.store + config.setup! + config.store + end + + # @return [CCC::Router] the global router (triggers setup! if needed) + def self.router + config.setup! + config.router + end + + # Reset the global configuration. For test teardown. + def self.reset! + @config = nil + end + # Load a reactor instance from its event history using AND-filtered partition reads. # Returns the evolved instance and a ReadResult (with .messages and .guard). # @@ -13,15 +51,16 @@ module CCC # # @param reactor_class [Class] a CCC reactor class (Decider, Projector, or any class # extending CCC::Consumer that includes CCC::Evolve) - # @param store [CCC::Store] the store to read from + # @param store [CCC::Store, nil] the store to read from (defaults to CCC.store) # @param partition_attrs [Hash{Symbol => String}] partition attribute values # @return [Array(reactor_instance, ReadResult)] # # @example - # decider, read_result = Sourced::CCC.load(MyDecider, store, course_id: 'Algebra', student_id: 'joe') + # decider, read_result = Sourced::CCC.load(MyDecider, course_id: 'Algebra', student_id: 'joe') # decider.state # evolved state # read_result.guard # ConsistencyGuard for subsequent appends - def self.load(reactor_class, store, **partition_attrs) + def self.load(reactor_class, store: nil, **partition_attrs) + store ||= self.store handled_types = reactor_class.handled_messages_for_evolve.map(&:type).uniq read_result = store.read_partition(partition_attrs, handled_types: handled_types) @@ -34,6 +73,7 @@ def self.load(reactor_class, store, **partition_attrs) end end +require 'sourced/ccc/configuration' require 'sourced/ccc/message' require 'sourced/ccc/store' require 'sourced/ccc/actions' diff --git a/lib/sourced/ccc/configuration.rb b/lib/sourced/ccc/configuration.rb new file mode 100644 index 00000000..b04f2cde --- /dev/null +++ b/lib/sourced/ccc/configuration.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Sourced + module CCC + class Configuration + attr_accessor :logger, :worker_count, :batch_size, + :catchup_interval, :max_drain_rounds, + :claim_ttl_seconds, :housekeeping_interval + + attr_reader :store, :router + + def initialize + @logger = Sourced.config.logger + @worker_count = 2 + @batch_size = 50 + @catchup_interval = 5 + @max_drain_rounds = 10 + @claim_ttl_seconds = 120 + @housekeeping_interval = 30 + @store = nil + @router = nil + @error_strategy = nil + @setup = false + end + + # Accepts either a CCC::Store instance or a Sequel::SQLite::Database connection. + # When given a DB connection, wraps it in CCC::Store.new(db). + def store=(s) + @store = case s + when Store then s + else Store.new(s) + end + end + + def error_strategy=(strategy) + raise ArgumentError, 'Must respond to #call' unless strategy.respond_to?(:call) + + @error_strategy = strategy + end + + def error_strategy + @error_strategy || Sourced.config.error_strategy + end + + def setup! + return if @setup + + @store ||= Store.new(Sequel.sqlite) + @store.install! + @router ||= Router.new(store: @store) + @setup = true + end + end + end +end diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index 7f1f222e..bd82f688 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -37,7 +37,7 @@ def context_for(partition_attrs) end def on_exception(exception, message, group) - Sourced.config.error_strategy.call(exception, message, group) + CCC.config.error_strategy.call(exception, message, group) end # Iterate messages collecting [actions, message] pairs. diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb index a88bd329..f90c466e 100644 --- a/lib/sourced/ccc/dispatcher.rb +++ b/lib/sourced/ccc/dispatcher.rb @@ -109,7 +109,7 @@ def initialize( housekeeping_interval: 30, claim_ttl_seconds: 120, work_queue: nil, - logger: Sourced.config.logger + logger: CCC.config.logger ) @logger = logger @router = router diff --git a/lib/sourced/ccc/stale_claim_reaper.rb b/lib/sourced/ccc/stale_claim_reaper.rb index bf7dd024..2f729625 100644 --- a/lib/sourced/ccc/stale_claim_reaper.rb +++ b/lib/sourced/ccc/stale_claim_reaper.rb @@ -29,7 +29,7 @@ class StaleClaimReaper # @param ttl_seconds [Integer] age threshold for stale claims (default 120) # @param worker_ids_provider [Proc] returns Array of active worker IDs # @param logger [Object] logger instance - def initialize(store:, interval: 30, ttl_seconds: 120, worker_ids_provider: -> { [] }, logger: Sourced.config.logger) + def initialize(store:, interval: 30, ttl_seconds: 120, worker_ids_provider: -> { [] }, logger: CCC.config.logger) @store = store @interval = interval @ttl_seconds = ttl_seconds diff --git a/lib/sourced/ccc/supervisor.rb b/lib/sourced/ccc/supervisor.rb index 80bc67df..cc98b028 100644 --- a/lib/sourced/ccc/supervisor.rb +++ b/lib/sourced/ccc/supervisor.rb @@ -37,14 +37,14 @@ def self.start(...) # @param claim_ttl_seconds [Integer] Stale claim age threshold in seconds # @param executor [Object] Executor instance for running concurrent workers def initialize( - router:, - logger: Sourced.config.logger, - count: 2, - batch_size: 50, - max_drain_rounds: 10, - catchup_interval: 5, - housekeeping_interval: 30, - claim_ttl_seconds: 120, + router: CCC.router, + logger: CCC.config.logger, + count: CCC.config.worker_count, + batch_size: CCC.config.batch_size, + max_drain_rounds: CCC.config.max_drain_rounds, + catchup_interval: CCC.config.catchup_interval, + housekeeping_interval: CCC.config.housekeeping_interval, + claim_ttl_seconds: CCC.config.claim_ttl_seconds, executor: Sourced.config.executor ) @router = router diff --git a/lib/sourced/ccc/worker.rb b/lib/sourced/ccc/worker.rb index 62f1f8db..3b4754a4 100644 --- a/lib/sourced/ccc/worker.rb +++ b/lib/sourced/ccc/worker.rb @@ -31,7 +31,7 @@ def initialize( name: SecureRandom.hex(4), batch_size: 50, max_drain_rounds: 10, - logger: Sourced.config.logger + logger: CCC.config.logger ) @work_queue = work_queue @router = router diff --git a/spec/sourced/ccc/configuration_spec.rb b/spec/sourced/ccc/configuration_spec.rb new file mode 100644 index 00000000..f408bad0 --- /dev/null +++ b/spec/sourced/ccc/configuration_spec.rb @@ -0,0 +1,198 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +RSpec.describe Sourced::CCC::Configuration do + after { Sourced::CCC.reset! } + + describe 'CCC.config' do + it 'returns a Configuration with sensible defaults' do + config = Sourced::CCC.config + expect(config).to be_a(described_class) + expect(config.worker_count).to eq(2) + expect(config.batch_size).to eq(50) + expect(config.catchup_interval).to eq(5) + expect(config.max_drain_rounds).to eq(10) + expect(config.claim_ttl_seconds).to eq(120) + expect(config.housekeeping_interval).to eq(30) + expect(config.logger).to eq(Sourced.config.logger) + end + + it 'returns the same instance on repeated calls' do + expect(Sourced::CCC.config).to be(Sourced::CCC.config) + end + end + + describe 'CCC.configure' do + it 'yields the config and freezes it after setup' do + Sourced::CCC.configure do |c| + c.worker_count = 4 + c.batch_size = 100 + end + + expect(Sourced::CCC.config.worker_count).to eq(4) + expect(Sourced::CCC.config.batch_size).to eq(100) + expect(Sourced::CCC.config).to be_frozen + end + + it 'calls setup! which creates store and router' do + Sourced::CCC.configure {} + + expect(Sourced::CCC.config.store).to be_a(Sourced::CCC::Store) + expect(Sourced::CCC.config.router).to be_a(Sourced::CCC::Router) + end + end + + describe 'CCC.register' do + let(:reactor_class) do + Class.new(Sourced::CCC::Projector) do + def self.name = 'TestConfigReactor' + + consumer_group 'test-config-reactor' + partition_by :thing_id + + state { |_| {} } + end + end + + it 'triggers setup and delegates to router.register' do + Sourced::CCC.register(reactor_class) + + expect(Sourced::CCC.router.reactors).to include(reactor_class) + end + end + + describe 'CCC.store' do + it 'triggers setup and returns the store' do + store = Sourced::CCC.store + expect(store).to be_a(Sourced::CCC::Store) + expect(store.installed?).to be true + end + end + + describe 'CCC.router' do + it 'triggers setup and returns the router' do + router = Sourced::CCC.router + expect(router).to be_a(Sourced::CCC::Router) + expect(router.store).to be(Sourced::CCC.store) + end + end + + describe 'CCC.reset!' do + it 'clears the singleton config' do + original = Sourced::CCC.config + Sourced::CCC.reset! + expect(Sourced::CCC.config).not_to be(original) + end + end + + describe '#store=' do + it 'accepts a CCC::Store instance directly' do + db = Sequel.sqlite + store = Sourced::CCC::Store.new(db) + + config = described_class.new + config.store = store + expect(config.store).to be(store) + end + + it 'wraps a Sequel::SQLite::Database in a Store' do + db = Sequel.sqlite + + config = described_class.new + config.store = db + expect(config.store).to be_a(Sourced::CCC::Store) + expect(config.store.db).to be(db) + end + end + + describe '#error_strategy' do + it 'falls through to Sourced.config.error_strategy by default' do + config = described_class.new + expect(config.error_strategy).to eq(Sourced.config.error_strategy) + end + + it 'can be overridden with a custom callable' do + custom = ->(_e, _m, _g) {} + config = described_class.new + config.error_strategy = custom + expect(config.error_strategy).to be(custom) + end + + it 'raises if assigned a non-callable' do + config = described_class.new + expect { config.error_strategy = 'not callable' }.to raise_error(ArgumentError) + end + end + + describe '#setup!' do + it 'is idempotent' do + config = described_class.new + config.setup! + store1 = config.store + router1 = config.router + config.setup! + expect(config.store).to be(store1) + expect(config.router).to be(router1) + end + + it 'defaults to in-memory SQLite store when none configured' do + config = described_class.new + config.setup! + expect(config.store).to be_a(Sourced::CCC::Store) + expect(config.store.installed?).to be true + end + + it 'uses configured store when set' do + db = Sequel.sqlite + store = Sourced::CCC::Store.new(db) + store.install! + + config = described_class.new + config.store = store + config.setup! + expect(config.store).to be(store) + end + end + + describe 'CCC.load with global store' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + + let(:decider_class) do + Class.new(Sourced::CCC::Decider) do + def self.name = 'ConfigLoadDecider' + + partition_by :thing_id + consumer_group 'config-load-decider' + + state { |_| { count: 0 } } + end + end + + before do + store.install! + Sourced::CCC.configure do |c| + c.store = store + end + end + + it 'uses global store when store: not provided' do + instance, read_result = Sourced::CCC.load(decider_class, thing_id: 'abc') + expect(instance.state[:count]).to eq(0) + expect(read_result.messages).to be_empty + end + + it 'uses override store when store: provided' do + other_db = Sequel.sqlite + other_store = Sourced::CCC::Store.new(other_db) + other_store.install! + + instance, read_result = Sourced::CCC.load(decider_class, store: other_store, thing_id: 'abc') + expect(instance.state[:count]).to eq(0) + expect(read_result.messages).to be_empty + end + end +end diff --git a/spec/sourced/ccc/load_spec.rb b/spec/sourced/ccc/load_spec.rb index ae8bbc9a..4fcb603c 100644 --- a/spec/sourced/ccc/load_spec.rb +++ b/spec/sourced/ccc/load_spec.rb @@ -81,7 +81,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'returns an evolved instance and a ReadResult' do instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -91,7 +91,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'evolves state from matching messages' do instance, _read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -101,7 +101,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'sets partition_values on the instance' do instance, _read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -110,7 +110,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'read_result contains the messages used for evolution' do _instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -121,7 +121,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'read_result contains a guard for subsequent appends' do _instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -131,7 +131,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'guard can be used for optimistic concurrency on append' do instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -154,7 +154,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'excludes messages from other partitions (AND filtering at SQL level)' do instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -175,7 +175,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'guard detects conflicts from concurrent writes to the partition' do _instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -214,7 +214,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'evolves projector state from matching messages' do instance, _read_result = Sourced::CCC.load( - LoadTestProjector, store, + LoadTestProjector, store: store, course_id: 'algebra' ) @@ -224,7 +224,7 @@ class LoadTestProjector < Sourced::CCC::Projector it 'passes partition values to state initializer' do instance, _read_result = Sourced::CCC.load( - LoadTestProjector, store, + LoadTestProjector, store: store, course_id: 'algebra' ) @@ -235,7 +235,7 @@ class LoadTestProjector < Sourced::CCC::Projector describe 'empty history' do it 'returns instance with initial state when no matching messages' do instance, read_result = Sourced::CCC.load( - LoadTestDecider, store, + LoadTestDecider, store: store, course_id: 'nonexistent', student_id: 'nobody' ) diff --git a/spec/sourced/ccc/supervisor_spec.rb b/spec/sourced/ccc/supervisor_spec.rb index 49af5808..4e823a4f 100644 --- a/spec/sourced/ccc/supervisor_spec.rb +++ b/spec/sourced/ccc/supervisor_spec.rb @@ -20,6 +20,10 @@ allow(Signal).to receive(:trap) end + after do + Sourced::CCC.reset! + end + describe '.start' do it 'creates a new supervisor instance and starts it' do supervisor_instance = instance_double(described_class) From 2fbf355ae6eecfae66b069bd7774368143049487 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 21 Feb 2026 21:59:31 +0000 Subject: [PATCH 021/115] Add StoreInterface to guard CCC::Configuration#store= assignment Uses Types::Interface to validate custom store objects implement the 12 required methods. Store instances and raw Sequel SQLite connections are still accepted directly. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/configuration.rb | 20 +++++++++++++++++++- spec/sourced/ccc/configuration_spec.rb | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/sourced/ccc/configuration.rb b/lib/sourced/ccc/configuration.rb index b04f2cde..fbe5ea34 100644 --- a/lib/sourced/ccc/configuration.rb +++ b/lib/sourced/ccc/configuration.rb @@ -3,6 +3,21 @@ module Sourced module CCC class Configuration + StoreInterface = Types::Interface[ + :installed?, + :install!, + :append, + :read, + :read_partition, + :claim_next, + :ack, + :release, + :register_consumer_group, + :worker_heartbeat, + :release_stale_claims, + :notifier + ] + attr_accessor :logger, :worker_count, :batch_size, :catchup_interval, :max_drain_rounds, :claim_ttl_seconds, :housekeeping_interval @@ -25,10 +40,13 @@ def initialize # Accepts either a CCC::Store instance or a Sequel::SQLite::Database connection. # When given a DB connection, wraps it in CCC::Store.new(db). + # Accepts a CCC::Store, a Sequel::SQLite::Database (auto-wrapped), + # or any object implementing StoreInterface. def store=(s) @store = case s when Store then s - else Store.new(s) + when ->(v) { v.class.name == 'Sequel::SQLite::Database' } then Store.new(s) + else StoreInterface.parse(s) end end diff --git a/spec/sourced/ccc/configuration_spec.rb b/spec/sourced/ccc/configuration_spec.rb index f408bad0..a394c3dc 100644 --- a/spec/sourced/ccc/configuration_spec.rb +++ b/spec/sourced/ccc/configuration_spec.rb @@ -106,6 +106,24 @@ def self.name = 'TestConfigReactor' expect(config.store).to be_a(Sourced::CCC::Store) expect(config.store.db).to be(db) end + + it 'accepts any object implementing StoreInterface' do + fake_store = double('CustomStore', + installed?: true, install!: nil, append: nil, read: nil, + read_partition: nil, claim_next: nil, ack: nil, release: nil, + register_consumer_group: nil, worker_heartbeat: nil, + release_stale_claims: nil, notifier: nil + ) + + config = described_class.new + config.store = fake_store + expect(config.store).to be(fake_store) + end + + it 'raises for objects not implementing StoreInterface' do + config = described_class.new + expect { config.store = Object.new }.to raise_error(Plumb::ParseError) + end end describe '#error_strategy' do From ee383e3c5491dbecf66186e04df69b0cd754473f Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 22 Feb 2026 00:32:04 +0000 Subject: [PATCH 022/115] StateStored and EventSourced projectors for CCC --- lib/sourced/ccc/projector.rb | 32 +++++- spec/sourced/ccc/configuration_spec.rb | 2 +- spec/sourced/ccc/dispatcher_spec.rb | 2 +- spec/sourced/ccc/load_spec.rb | 2 +- spec/sourced/ccc/projector_spec.rb | 120 +++++++++++++++++++- spec/sourced/ccc/router_spec.rb | 2 +- spec/sourced/ccc/stale_claim_reaper_spec.rb | 2 +- 7 files changed, 152 insertions(+), 10 deletions(-) diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index b2e61895..923bfd8a 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -14,12 +14,14 @@ def handled_messages (handled_messages_for_evolve + handled_messages_for_react).uniq end - # No history: — uses claim.messages directly. - def handle_batch(claim) + private + + def build_instance(claim) values = partition_keys.map { |k| claim.partition_value[k.to_s] } - instance = new(values) - instance.evolve(claim.messages) + new(values) + end + def build_action_pairs(instance, claim) sync_actions = instance.sync_actions( state: instance.state, messages: claim.messages, replaying: claim.replaying ) @@ -43,6 +45,28 @@ def handle_batch(claim) def initialize(partition_values = []) @partition_values = partition_values end + + # StateStored: loads persisted state via `state` block, evolves only new messages. + class StateStored < self + class << self + def handle_batch(claim) + instance = build_instance(claim) + instance.evolve(claim.messages) + build_action_pairs(instance, claim) + end + end + end + + # EventSourced: rebuilds state from full history each time. + class EventSourced < self + class << self + def handle_batch(claim, history:) + instance = build_instance(claim) + instance.evolve(history.messages) + build_action_pairs(instance, claim) + end + end + end end end end diff --git a/spec/sourced/ccc/configuration_spec.rb b/spec/sourced/ccc/configuration_spec.rb index a394c3dc..eeb0fd30 100644 --- a/spec/sourced/ccc/configuration_spec.rb +++ b/spec/sourced/ccc/configuration_spec.rb @@ -47,7 +47,7 @@ describe 'CCC.register' do let(:reactor_class) do - Class.new(Sourced::CCC::Projector) do + Class.new(Sourced::CCC::Projector::StateStored) do def self.name = 'TestConfigReactor' consumer_group 'test-config-reactor' diff --git a/spec/sourced/ccc/dispatcher_spec.rb b/spec/sourced/ccc/dispatcher_spec.rb index b1ec83fa..4d0f242a 100644 --- a/spec/sourced/ccc/dispatcher_spec.rb +++ b/spec/sourced/ccc/dispatcher_spec.rb @@ -42,7 +42,7 @@ class DispatchTestDecider < Sourced::CCC::Decider end end -class DispatchTestProjector < Sourced::CCC::Projector +class DispatchTestProjector < Sourced::CCC::Projector::StateStored partition_by :device_id consumer_group 'dispatch-test-projector' diff --git a/spec/sourced/ccc/load_spec.rb b/spec/sourced/ccc/load_spec.rb index 4fcb603c..00d5b030 100644 --- a/spec/sourced/ccc/load_spec.rb +++ b/spec/sourced/ccc/load_spec.rb @@ -37,7 +37,7 @@ class LoadTestDecider < Sourced::CCC::Decider end end -class LoadTestProjector < Sourced::CCC::Projector +class LoadTestProjector < Sourced::CCC::Projector::StateStored partition_by :course_id consumer_group 'load-test-projector' diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb index 1631b48c..ca1d6577 100644 --- a/spec/sourced/ccc/projector_spec.rb +++ b/spec/sourced/ccc/projector_spec.rb @@ -19,7 +19,7 @@ module CCCProjectorTestMessages end end -class TestItemProjector < Sourced::CCC::Projector +class TestItemProjector < Sourced::CCC::Projector::StateStored partition_by :list_id consumer_group 'item-projector-test' @@ -45,6 +45,32 @@ class TestItemProjector < Sourced::CCC::Projector end end +class TestItemESProjector < Sourced::CCC::Projector::EventSourced + partition_by :list_id + consumer_group 'item-es-projector-test' + + state do |(list_id)| + { list_id: list_id, items: [], synced: false } + end + + evolve CCCProjectorTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end + + evolve CCCProjectorTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + reaction CCCProjectorTestMessages::ItemArchived do |_state, msg| + CCCProjectorTestMessages::NotifyArchive.new(payload: { list_id: msg.payload.list_id }) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + state[:last_replaying] = replaying + end +end + RSpec.describe Sourced::CCC::Projector do describe '.handled_messages' do it 'includes evolve and react types' do @@ -141,6 +167,98 @@ def make_claim(messages, replaying: false) end end + describe 'EventSourced' do + let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 5) } + + def make_claim(messages, replaying: false) + Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'list_id:L1', + partition_value: { 'list_id' => 'L1' }, + messages: messages, replaying: replaying, guard: guard + ) + end + + def make_history(messages) + Sourced::CCC::ReadResult.new(messages: messages, guard: guard) + end + + it 'evolves from full history, not just claim messages' do + history_msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 3 + ) + ] + # Claim only contains the latest message + claim_msgs = [history_msgs.last] + claim = make_claim(claim_msgs) + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(claim, history: history) + + # Sync pair should be the last one, acked against claim's last message + sync_pair = pairs.last + _sync_actions, source_msg = sync_pair + expect(source_msg).to eq(claim_msgs.last) + end + + it 'runs reactions only on claim messages, not full history' do + history_msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Old' }), 1 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'New' }), 2 + ) + ] + # Only the second message is in the claim + claim_msgs = [history_msgs.last] + claim = make_claim(claim_msgs, replaying: false) + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(claim, history: history) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + # Only 1 reaction (for the claim message), not 2 + expect(append_actions.size).to eq(1) + expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) + end + + it 'skips reactions when replaying' do + history_msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(history_msgs, replaying: true) + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(claim, history: history) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions).to be_empty + end + + it 'is detected by Injector as needing history' do + needs = Sourced::Injector.resolve_args(TestItemESProjector, :handle_batch) + expect(needs).to include(:history) + end + + it 'StateStored is not detected as needing history' do + needs = Sourced::Injector.resolve_args(TestItemProjector, :handle_batch) + expect(needs).not_to include(:history) + end + end + describe '.context_for' do it 'builds conditions from partition_keys × handled_messages_for_evolve' do conditions = TestItemProjector.context_for(list_id: 'L1') diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index a806f53c..59c94834 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -63,7 +63,7 @@ class RouterTestDecider < Sourced::CCC::Decider end # Test projector for router specs -class RouterTestProjector < Sourced::CCC::Projector +class RouterTestProjector < Sourced::CCC::Projector::StateStored partition_by :device_id consumer_group 'router-test-projector' diff --git a/spec/sourced/ccc/stale_claim_reaper_spec.rb b/spec/sourced/ccc/stale_claim_reaper_spec.rb index b78e490b..a32b1047 100644 --- a/spec/sourced/ccc/stale_claim_reaper_spec.rb +++ b/spec/sourced/ccc/stale_claim_reaper_spec.rb @@ -11,7 +11,7 @@ module StaleClaimReaperTestMessages end end -class ReaperTestProjector < Sourced::CCC::Projector +class ReaperTestProjector < Sourced::CCC::Projector::StateStored partition_by :device_id consumer_group 'reaper-test-projector' From d48dc4eaa54f283ebb90effd0d228879ae4c11e0 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 22 Feb 2026 17:09:02 +0000 Subject: [PATCH 023/115] Add CCC error handling parity with SequelBackend Replace NullGroup with GroupUpdater that accumulates stop/retry mutations for atomic persistence. Add Store#updating_consumer_group to load, yield, and persist group state. Gate claim_next on retry_at so retries are honoured at the database level. Clear retry_at and error_context in start_consumer_group. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/consumer.rb | 28 +++++++++ lib/sourced/ccc/router.rb | 8 ++- lib/sourced/ccc/store.rb | 42 +++++++++++-- spec/sourced/ccc/router_spec.rb | 56 +++++++++++++++++ spec/sourced/ccc/store_spec.rb | 104 ++++++++++++++++++++++++++++++++ 5 files changed, 232 insertions(+), 6 deletions(-) diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index bd82f688..67f00a8e 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -2,6 +2,34 @@ module Sourced module CCC + # Accumulates mutations to a consumer group row for atomic persistence. + # Mirrors Sourced::Backends::SequelBackend::GroupUpdater. + # Used by {Store#updating_consumer_group}. + class GroupUpdater + attr_reader :group_id, :updates, :error_context + + def initialize(group_id, row, logger) + @group_id = group_id + @logger = logger + @error_context = row[:error_context] + @updates = { error_context: @error_context.dup } + end + + def stop(exception:, message:) + @logger.error "CCC: stopping consumer group #{group_id} message: '#{message.type}' (#{message.id}). #{exception.class}: #{exception.message}" + @updates[:status] = Store::STOPPED + @updates[:retry_at] = nil + @updates[:updated_at] = Time.now.iso8601 + end + + def retry(time, **ctx) + @logger.warn "CCC: retrying consumer group #{group_id} at #{time}" + @updates[:updated_at] = Time.now.iso8601 + @updates[:retry_at] = time.iso8601 + @updates[:error_context].merge!(ctx) + end + end + # Shared consumer configuration for CCC reactors. # Extended (not included) onto reactor classes. module Consumer diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index 7315afda..5481a3cf 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -51,14 +51,18 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) rescue Sourced::PartialBatchError => e execute_actions(e.action_pairs, claim, reactor_class.group_id) - reactor_class.on_exception(e, e.failed_message, nil) + store.updating_consumer_group(reactor_class.group_id) do |group| + reactor_class.on_exception(e, e.failed_message, group) + end true rescue Sourced::ConcurrentAppendError store.release(reactor_class.group_id, offset_id: claim.offset_id) true rescue StandardError => e store.release(reactor_class.group_id, offset_id: claim.offset_id) - reactor_class.on_exception(e, claim.messages.first, nil) + store.updating_consumer_group(reactor_class.group_id) do |group| + reactor_class.on_exception(e, claim.messages.first, group) + end true end end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 03a0c926..4d0e54cc 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -44,11 +44,16 @@ class Store # @return [Sourced::InlineNotifier] attr_reader :notifier + # @return [Logger] + attr_reader :logger + # @param db [Sequel::SQLite::Database] a Sequel SQLite connection # @param notifier [#notify_new_messages, #notify_reactor_resumed, nil] optional notifier for dispatch signals - def initialize(db, notifier: nil) + # @param logger [Logger, nil] optional logger (defaults to Sourced.config.logger) + def initialize(db, notifier: nil, logger: nil) @db = db @notifier = notifier || Sourced::InlineNotifier.new + @logger = logger || Sourced.config.logger @db.run('PRAGMA foreign_keys = ON') @db.run('PRAGMA journal_mode = WAL') @db.run('PRAGMA busy_timeout = 5000') @@ -302,15 +307,40 @@ def stop_consumer_group(group_id) db[:ccc_consumer_groups].where(group_id: group_id).update(status: STOPPED, updated_at: Time.now.iso8601) end - # Re-activate a stopped consumer group. + # Re-activate a stopped consumer group, clearing retry state. # # @param group_id [String] # @return [void] def start_consumer_group(group_id) - db[:ccc_consumer_groups].where(group_id: group_id).update(status: ACTIVE, updated_at: Time.now.iso8601) + db[:ccc_consumer_groups] + .where(group_id: group_id) + .update(status: ACTIVE, retry_at: nil, error_context: nil, updated_at: Time.now.iso8601) notifier.notify_reactor_resumed(group_id) end + # Load a consumer group row, yield a {GroupUpdater} for mutation, + # then persist the accumulated updates atomically. + # Mirrors SequelBackend#updating_consumer_group. + # + # @param group_id [String] + # @yieldparam group [CCC::GroupUpdater] + # @return [void] + def updating_consumer_group(group_id) + dataset = db[:ccc_consumer_groups].where(group_id: group_id) + row = dataset.first + raise ArgumentError, "Consumer group #{group_id} not found" unless row + + ctx = row[:error_context] ? JSON.parse(row[:error_context], symbolize_names: true) : {} + row[:error_context] = ctx + + group = CCC::GroupUpdater.new(group_id, row, logger) + yield group + + updates = group.updates.dup + updates[:error_context] = JSON.dump(updates[:error_context]) + dataset.update(updates) + end + # Delete all offsets for a consumer group, resetting it to process from the beginning. # # @param group_id [String] @@ -345,7 +375,11 @@ def reset_consumer_group(group_id) # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, replaying:, guard: }+ or nil def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: nil) partition_by = Array(partition_by).sort - cg = db[:ccc_consumer_groups].where(group_id: group_id, status: ACTIVE).first + now = Time.now.iso8601 + cg = db[:ccc_consumer_groups] + .where(group_id: group_id, status: ACTIVE) + .where { Sequel.|({retry_at: nil}, Sequel.lit('retry_at <= ?', now)) } + .first return nil unless cg bootstrap_offsets(cg[:id], partition_by) diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 59c94834..9dbcee2e 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -220,6 +220,62 @@ def self.handle_batch(claim) expect(result).to be true expect(RouterTestDecider).to have_received(:on_exception) end + + it 'on_exception stops consumer group when default strategy' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + + router.handle_next_for(RouterTestDecider) + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false + end + + it 'on_exception persists error_context in the database' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + + router.handle_next_for(RouterTestDecider) + + row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:error_context]).not_to be_nil + expect(row[:status]).to eq('stopped') + end + + it 'on_exception with retry strategy sets retry_at on consumer group' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + retry_strategy = Sourced::ErrorStrategy.new do |s| + s.retry(times: 3, after: 5) + end + allow(Sourced::CCC).to receive_message_chain(:config, :error_strategy).and_return(retry_strategy) + + allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + + router.handle_next_for(RouterTestDecider) + + row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:retry_at]).not_to be_nil + expect(row[:status]).to eq('active') + + ctx = JSON.parse(row[:error_context], symbolize_names: true) + expect(ctx[:retry_count]).to eq(2) + end end describe '#drain' do diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 82dbeecb..71565ef0 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -538,6 +538,82 @@ module CCCStoreTestMessages store.start_consumer_group('my-group') expect(store.consumer_group_active?('my-group')).to be true end + + it 'start_consumer_group clears retry_at and error_context' do + store.register_consumer_group('my-group') + + # Set retry_at and error_context via updating_consumer_group + store.updating_consumer_group('my-group') do |group| + group.retry(Time.now + 60, retry_count: 1) + end + + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + expect(row[:retry_at]).not_to be_nil + expect(row[:error_context]).not_to be_nil + + store.start_consumer_group('my-group') + + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + expect(row[:retry_at]).to be_nil + expect(row[:error_context]).to be_nil + expect(row[:status]).to eq('active') + end + end + + describe '#updating_consumer_group' do + before do + store.register_consumer_group('my-group') + end + + it 'yields a GroupUpdater and persists error_context' do + store.updating_consumer_group('my-group') do |group| + expect(group).to be_a(Sourced::CCC::GroupUpdater) + group.retry(Time.now + 30, retry_count: 1) + end + + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + ctx = JSON.parse(row[:error_context], symbolize_names: true) + expect(ctx[:retry_count]).to eq(1) + expect(row[:retry_at]).not_to be_nil + end + + it 'persists error_context across successive calls (retry_count increments)' do + store.updating_consumer_group('my-group') do |group| + group.retry(Time.now + 30, retry_count: 1) + end + + store.updating_consumer_group('my-group') do |group| + expect(group.error_context[:retry_count]).to eq(1) + group.retry(Time.now + 60, retry_count: 2) + end + + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + ctx = JSON.parse(row[:error_context], symbolize_names: true) + expect(ctx[:retry_count]).to eq(2) + end + + it 'raises ArgumentError for nonexistent group' do + expect { + store.updating_consumer_group('nonexistent') { |_| } + }.to raise_error(ArgumentError, /nonexistent/) + end + + it 'stop sets status to STOPPED and clears retry_at' do + # First set a retry_at + store.updating_consumer_group('my-group') do |group| + group.retry(Time.now + 30, retry_count: 1) + end + + err = RuntimeError.new('test error') + msg = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + store.updating_consumer_group('my-group') do |group| + group.stop(exception: err, message: msg) + end + + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + expect(row[:status]).to eq('stopped') + expect(row[:retry_at]).to be_nil + end end describe '#reset_consumer_group' do @@ -667,6 +743,34 @@ module CCCStoreTestMessages expect(result).to be_nil end + it 'returns nil when retry_at is in the future' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + # Set retry_at to the future + store.updating_consumer_group(group_id) do |group| + group.retry(Time.now + 3600, retry_count: 1) + end + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result).to be_nil + end + + it 'returns claims when retry_at is in the past' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + # Set retry_at to the past + store.updating_consumer_group(group_id) do |group| + group.retry(Time.now - 1, retry_count: 1) + end + + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result).not_to be_nil + end + it 'prioritizes partition with earliest pending message' do store.append( CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) From 9b9bc465e9f2df3ddbf2e4c9e3e36779df93a444 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 22 Feb 2026 17:10:53 +0000 Subject: [PATCH 024/115] Error logging --- lib/sourced/ccc/consumer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index 67f00a8e..9170a70b 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -16,7 +16,7 @@ def initialize(group_id, row, logger) end def stop(exception:, message:) - @logger.error "CCC: stopping consumer group #{group_id} message: '#{message.type}' (#{message.id}). #{exception.class}: #{exception.message}" + @logger.error "CCC: stopping consumer group #{group_id} message: '#{message&.type}' (#{message&.id}). #{exception&.class}: #{exception&.message}" @updates[:status] = Store::STOPPED @updates[:retry_at] = nil @updates[:updated_at] = Time.now.iso8601 From 381cbf708b1f54e1401d8555cfa268e9aaa62dee Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 22 Feb 2026 18:33:32 +0000 Subject: [PATCH 025/115] CCC::Command and CCC::Event subclasses with their own registries --- lib/sourced/ccc/message.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 489693ad..cde6d401 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -214,5 +214,8 @@ def prepare_attributes(attrs) attrs end end + + class Command < Message; end + class Event < Message; end end end From 9253938cdb56921cb49733a068aa4352e577b208 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 22 Feb 2026 18:43:06 +0000 Subject: [PATCH 026/115] Add CCC.handle! for synchronous command handling in web controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encapsulates the validate → load history → decide → append → ACK flow into a single call. Returns a HandleResult supporting destructuring: cmd, reactor, events = CCC.handle!(cmd, MyDecider) Adds Store#advance_offset to move consumer group offsets without a prior claim, so background workers skip already-handled commands. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 102 +++++++++++++ lib/sourced/ccc/store.rb | 30 ++++ spec/sourced/ccc/handle_spec.rb | 257 ++++++++++++++++++++++++++++++++ spec/sourced/ccc/store_spec.rb | 165 ++++++++++++++++++++ 4 files changed, 554 insertions(+) create mode 100644 spec/sourced/ccc/handle_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index f1c8be4f..96b7ebe0 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'sourced/injector' + module Sourced module CCC # @return [Configuration] the global CCC configuration instance @@ -40,6 +42,69 @@ def self.reset! @config = nil end + # Returned by {.handle!} with command, reactor instance, and new events. + # Supports array destructuring: +cmd, reactor, events = CCC.handle!(cmd, MyDecider)+ + HandleResult = Data.define(:command, :reactor, :events) do + def to_ary = [command, reactor, events] + end + + # Handle a command synchronously: validate, load history, decide, append, and ACK. + # + # 1. Validates the command via +command.valid?+ + # 2. If invalid, returns immediately with the command, an uninitialized reactor, and empty events + # 3. Loads the reactor's history from the command's partition attributes + # 4. Evolves the reactor from history and runs the decider + # 5. Appends the command and correlated events to the store with optimistic concurrency + # 6. Advances consumer group offsets for registered reactors so background workers skip + # the already-handled command + # + # @param reactor_class [Class] a CCC::Decider (or any reactor extending Consumer + Evolve) + # @param command [CCC::Command] the command to handle (must respond to +valid?+) + # @param store [CCC::Store, nil] the store to use (defaults to CCC.store) + # @return [HandleResult] supports destructuring: +cmd, reactor, events = result+ + # @raise [Sourced::ConcurrentAppendError] if conflicting messages found after history read + # @raise [RuntimeError] if the decider raises a domain error (invariant violation) + # + # @example + # cmd = CourseApp::CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) + # cmd, decider, events = Sourced::CCC.handle!(CourseApp::CourseDecider, cmd) + # if cmd.valid? + # # events were appended, offsets advanced + # else + # # cmd.errors has validation details + # end + def self.handle!(reactor_class, command, store: nil) + store ||= self.store + + partition_attrs = extract_partition_attrs(command, reactor_class) + values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } + instance = reactor_class.new(values) + + unless command.valid? + return HandleResult.new(command: command, reactor: instance, events: []) + end + + # Load history if the reactor needs it (Deciders always do) + needs_history = Injector.resolve_args(reactor_class, :handle_batch).include?(:history) + if needs_history + instance, read_result = load(reactor_class, store: store, **partition_attrs) + end + + # Decide + raw_events = instance.decide(command) + correlated_events = raw_events.map { |e| command.correlate(e) } + + # Append command + events in one transaction with consistency guard + guard = read_result&.guard + to_append = [command] + correlated_events + last_position = store.append(to_append, guard: guard) + + # Advance offsets for registered consumer groups + advance_registered_offsets(store, reactor_class, partition_attrs, last_position) + + HandleResult.new(command: command, reactor: instance, events: correlated_events) + end + # Load a reactor instance from its event history using AND-filtered partition reads. # Returns the evolved instance and a ReadResult (with .messages and .guard). # @@ -70,6 +135,43 @@ def self.load(reactor_class, store: nil, **partition_attrs) [instance, read_result] end + + # Extract partition attribute values from a command's payload, + # scoped to the reactor's declared partition_keys. + # + # @param command [CCC::Command] + # @param reactor_class [Class] + # @return [Hash{Symbol => String}] + private_class_method def self.extract_partition_attrs(command, reactor_class) + reactor_class.partition_keys.each_with_object({}) do |key, h| + value = command.payload&.respond_to?(key) ? command.payload.send(key) : nil + h[key] = value if value + end + end + + # Advance consumer group offsets for all reactors registered in the global router + # that handle the given reactor_class's messages, so background workers skip + # the already-handled command. + # + # @param store [CCC::Store] + # @param reactor_class [Class] + # @param partition_attrs [Hash{Symbol => String}] + # @param position [Integer] + private_class_method def self.advance_registered_offsets(store, reactor_class, partition_attrs, position) + return unless config.router + + partition = partition_attrs.transform_keys(&:to_s) + + config.router.reactors.each do |registered_reactor| + next unless registered_reactor == reactor_class + + store.advance_offset( + registered_reactor.group_id, + partition: partition, + position: position + ) + end + end end end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 4d0e54cc..967a3597 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -516,6 +516,36 @@ def release_stale_claims(ttl_seconds: 120) .update(claimed: 0, claimed_at: nil, claimed_by: nil) end + # Advance a consumer group's offset for a specific partition to at least +position+. + # Bootstraps the offset row if it doesn't exist yet. + # Unlike {#ack}, this does not require a prior claim. + # + # @param group_id [String] consumer group identifier + # @param partition [Hash{String => String}] partition attribute names and values + # @param position [Integer] advance offset to at least this position + # @return [void] + def advance_offset(group_id, partition:, position:) + cg = db[:ccc_consumer_groups].where(group_id: group_id).first + return unless cg + + partition_by = partition.keys.sort + bootstrap_offsets(cg[:id], partition_by) + + partition_key = build_partition_key(partition_by, partition) + offset = db[:ccc_offsets].where(consumer_group_id: cg[:id], partition_key: partition_key).first + return unless offset + return if offset[:last_position] >= position + + db[:ccc_offsets].where(id: offset[:id]).update(last_position: position) + + if position > cg[:highest_position] + db[:ccc_consumer_groups].where(id: cg[:id]).update( + highest_position: position, + updated_at: Time.now.iso8601 + ) + end + end + # Current max position in the message log. # # @return [Integer] max position, or 0 if the store is empty diff --git a/spec/sourced/ccc/handle_spec.rb b/spec/sourced/ccc/handle_spec.rb new file mode 100644 index 00000000..128107fb --- /dev/null +++ b/spec/sourced/ccc/handle_spec.rb @@ -0,0 +1,257 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sequel' + +module CCCHandleTestMessages + CreateDevice = Sourced::CCC::Command.define('handle_test.create_device') do + attribute :device_id, String + attribute :name, Sourced::Types::String.present + end + + DeviceCreated = Sourced::CCC::Event.define('handle_test.device_created') do + attribute :device_id, String + attribute :name, String + end + + ActivateDevice = Sourced::CCC::Command.define('handle_test.activate_device') do + attribute :device_id, String + end + + DeviceActivated = Sourced::CCC::Event.define('handle_test.device_activated') do + attribute :device_id, String + end +end + +class HandleTestDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'handle-test-decider' + + state { |_| { exists: false, active: false } } + + evolve CCCHandleTestMessages::DeviceCreated do |state, _evt| + state[:exists] = true + end + + evolve CCCHandleTestMessages::DeviceActivated do |state, _evt| + state[:active] = true + end + + command CCCHandleTestMessages::CreateDevice do |state, cmd| + raise 'Already exists' if state[:exists] + event CCCHandleTestMessages::DeviceCreated, device_id: cmd.payload.device_id, name: cmd.payload.name + end + + command CCCHandleTestMessages::ActivateDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already active' if state[:active] + event CCCHandleTestMessages::DeviceActivated, device_id: cmd.payload.device_id + end +end + +RSpec.describe 'Sourced::CCC.handle!' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + + before { store.install! } + + describe 'valid command, no prior history' do + it 'returns command, reactor, and events' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 'Sensor' } + ) + + result = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + expect(result).to be_a(Sourced::CCC::HandleResult) + expect(result.command).to eq(cmd) + expect(result.reactor).to be_a(HandleTestDecider) + expect(result.events.size).to eq(1) + expect(result.events.first).to be_a(CCCHandleTestMessages::DeviceCreated) + end + + it 'supports array destructuring' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 'Sensor' } + ) + + cmd_out, reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + expect(cmd_out).to eq(cmd) + expect(reactor).to be_a(HandleTestDecider) + expect(events.size).to eq(1) + end + + it 'evolves reactor state' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 'Sensor' } + ) + + _cmd, reactor, _events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + expect(reactor.state[:exists]).to be true + end + + it 'appends command and correlated events to the store' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 'Sensor' } + ) + + _cmd, _reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + # Both command and event should be in the store + all = store.db[:ccc_messages].order(:position).all + expect(all.size).to eq(2) + expect(all[0][:message_type]).to eq('handle_test.create_device') + expect(all[1][:message_type]).to eq('handle_test.device_created') + end + + it 'correlates events with the command' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 'Sensor' } + ) + + _cmd, _reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + expect(events.first.causation_id).to eq(cmd.id) + expect(events.first.correlation_id).to eq(cmd.correlation_id) + end + end + + describe 'valid command with prior history' do + before do + # Create device first + Sourced::CCC.handle!( + HandleTestDecider, + CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + store: store + ) + end + + it 'loads history and evolves before deciding' do + cmd = CCCHandleTestMessages::ActivateDevice.new( + payload: { device_id: 'd1' } + ) + + _cmd, reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + expect(events.size).to eq(1) + expect(events.first).to be_a(CCCHandleTestMessages::DeviceActivated) + expect(reactor.state[:exists]).to be true + expect(reactor.state[:active]).to be true + end + end + + describe 'invalid command' do + it 'returns immediately without appending' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 123 } # name should be a present string + ) + + cmd_out, reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + expect(cmd_out.valid?).to be false + expect(reactor).to be_a(HandleTestDecider) + expect(events).to eq([]) + + # Nothing appended + expect(store.db[:ccc_messages].count).to eq(0) + end + end + + describe 'domain invariant violation' do + it 'raises the domain error' do + cmd = CCCHandleTestMessages::ActivateDevice.new( + payload: { device_id: 'd1' } + ) + + expect { + Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + }.to raise_error(RuntimeError, 'Not found') + end + end + + describe 'optimistic concurrency' do + it 'guard prevents concurrent writes within the same partition' do + # Create the device first so there is history (and thus guard conditions) + Sourced::CCC.handle!( + HandleTestDecider, + CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + store: store + ) + + # Simulate a concurrent write to the same partition between load and append + # by directly appending an event after the first handle! + store.append( + CCCHandleTestMessages::DeviceActivated.new(payload: { device_id: 'd1' }) + ) + + # A second handle! that also needs to write to the same partition + # should detect the concurrent write. Since the decider also sees the + # activated state, it raises an invariant error first. + cmd = CCCHandleTestMessages::ActivateDevice.new(payload: { device_id: 'd1' }) + expect { + Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + }.to raise_error(RuntimeError, 'Already active') + end + + it 'passes guard through to store.append for conflict detection' do + # Verify handle! plumbs the guard through by checking that the + # guard from load is used when appending + allow(store).to receive(:append).and_call_original + + Sourced::CCC.handle!( + HandleTestDecider, + CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + store: store + ) + + expect(store).to have_received(:append).with( + anything, + guard: an_instance_of(Sourced::CCC::ConsistencyGuard) + ) + end + end + + describe 'offset advancement for registered reactors' do + let(:router) { Sourced::CCC::Router.new(store: store) } + + before do + router.register(HandleTestDecider) + allow(Sourced::CCC).to receive(:config).and_return( + instance_double(Sourced::CCC::Configuration, router: router) + ) + end + + it 'advances offsets so background workers skip handled commands' do + cmd = CCCHandleTestMessages::CreateDevice.new( + payload: { device_id: 'd1', name: 'Sensor' } + ) + + Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + + # Background worker should find no work for this partition + handled = router.handle_next_for(HandleTestDecider, worker_id: 'test-worker') + expect(handled).to be false + end + + it 'advances offsets after multiple commands on same partition' do + Sourced::CCC.handle!( + HandleTestDecider, + CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + store: store + ) + + Sourced::CCC.handle!( + HandleTestDecider, + CCCHandleTestMessages::ActivateDevice.new(payload: { device_id: 'd1' }), + store: store + ) + + # Background worker should still find no work + handled = router.handle_next_for(HandleTestDecider, worker_id: 'test-worker') + expect(handled).to be false + end + end +end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 71565ef0..3022b291 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1130,6 +1130,171 @@ module CCCStoreTestMessages end end + describe '#advance_offset' do + let(:group_id) { 'advance-test' } + let(:handled_types) { ['store_test.device.registered', 'store_test.device.bound'] } + + before do + store.register_consumer_group(group_id) + end + + it 'bootstraps and advances offset for a new partition' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + + offset = db[:ccc_offsets].join(:ccc_consumer_groups, id: :consumer_group_id) + .where(Sequel[:ccc_consumer_groups][:group_id] => group_id) + .first + expect(offset[:last_position]).to eq(1) + end + + it 'advances highest_position on the consumer group' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + + cg = db[:ccc_consumer_groups].where(group_id: group_id).first + expect(cg[:highest_position]).to eq(1) + end + + it 'never decreases the offset' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + ]) + + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 2 + ) + + # Try to go backwards + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + + offset = db[:ccc_offsets].join(:ccc_consumer_groups, id: :consumer_group_id) + .where(Sequel[:ccc_consumer_groups][:group_id] => group_id) + .first + expect(offset[:last_position]).to eq(2) + end + + it 'never decreases highest_position' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 2 + ) + + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-2' }, + position: 1 + ) + + cg = db[:ccc_consumer_groups].where(group_id: group_id).first + expect(cg[:highest_position]).to eq(2) + end + + it 'causes claim_next to skip the advanced partition' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + + result = store.claim_next(group_id, + partition_by: 'device_id', + handled_types: handled_types, + worker_id: 'w-1' + ) + expect(result).to be_nil + end + + it 'only skips messages up to the advanced position' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a1' }) + ]) + + # Advance past only the first message + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + + result = store.claim_next(group_id, + partition_by: 'device_id', + handled_types: handled_types, + worker_id: 'w-1' + ) + expect(result).not_to be_nil + expect(result.messages.size).to eq(1) + expect(result.messages.first.type).to eq('store_test.device.bound') + end + + it 'is a no-op for nonexistent consumer group' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + expect { + store.advance_offset('nonexistent', + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + }.not_to raise_error + + expect(db[:ccc_offsets].count).to eq(0) + end + + it 'is a no-op when partition has no messages in the store' do + expect { + store.advance_offset(group_id, + partition: { 'device_id' => 'dev-1' }, + position: 1 + ) + }.not_to raise_error + + expect(db[:ccc_offsets].count).to eq(0) + end + + it 'works with composite partitions' do + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ) + + store.advance_offset(group_id, + partition: { 'course_name' => 'Algebra', 'user_id' => 'joe' }, + position: 1 + ) + + offset = db[:ccc_offsets].join(:ccc_consumer_groups, id: :consumer_group_id) + .where(Sequel[:ccc_consumer_groups][:group_id] => group_id) + .first + expect(offset[:last_position]).to eq(1) + expect(offset[:partition_key]).to eq('course_name:Algebra|user_id:joe') + end + end + describe '#release' do let(:group_id) { 'release-test' } From 432caeb9da1df02ee881eed63a6a391b2278c2d8 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 28 Feb 2026 16:20:27 +0000 Subject: [PATCH 027/115] Add CCC module README with API docs and usage examples Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/README.md | 319 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 lib/sourced/ccc/README.md diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md new file mode 100644 index 00000000..743a7652 --- /dev/null +++ b/lib/sourced/ccc/README.md @@ -0,0 +1,319 @@ +# Sourced::CCC — Stream-less Event Sourcing + +CCC ("Command Context Consistency") is an experimental module for aggregateless, stream-less event sourcing. Events go into a flat, globally-ordered log. Consistency context is assembled dynamically by querying relevant facts via key-value pairs extracted from event payloads, rather than being pre-assigned to fixed streams. + +## Core Concepts + +- **No streams or aggregates** — all messages share a single append-only log with auto-increment positions. +- **Partitioning by attributes** — reactors declare which payload attributes define their consistency boundary (e.g. `partition_by :course_id`). The store uses these to build query conditions and claim work. +- **Decide → Evolve → React** — same pattern as core Sourced, but without stream IDs or sequence numbers. +- **Optimistic concurrency** — reads return a `ConsistencyGuard` that detects conflicting writes at append time. + +## Messages + +CCC has its own message hierarchy, separate from `Sourced::Message`. Messages have no `stream_id` or `seq` — they get a global `position` when stored. + +```ruby +# Define base classes for your domain (optional but recommended) +class MyEvent < Sourced::CCC::Event; end +class MyCommand < Sourced::CCC::Command; end + +# Define typed messages with payload attributes +CreateCourse = MyCommand.define('courses.create') do + attribute :course_id, String + attribute :course_name, String +end + +CourseCreated = MyEvent.define('courses.created') do + attribute :course_id, String + attribute :course_name, String +end +``` + +### Message features + +- **Auto-generated UUIDs** for `id`, `causation_id`, and `correlation_id` +- **Causal tracing** via `#correlate(other_message)` — sets `causation_id` and `correlation_id` +- **Auto-indexed keys** — `#extracted_keys` returns `[["course_id", "c1"], ["course_name", "Algebra"]]` from payload attributes, used by the store to index messages for querying +- **Own registry** — `CCC::Message.registry` is separate from `Sourced::Message.registry`. Use `.from(type: "courses.created", payload: {...})` to instantiate from a type string. +- **Typed payloads** — uses the same Plumb/Types DSL as core Sourced for attribute coercion and validation + +## Store + +`CCC::Store` is an SQLite-backed store (via Sequel) providing the flat message log, key-pair indexing, and consumer group management. + +```ruby +require 'sequel' + +db = Sequel.sqlite('my_app.db') +store = Sourced::CCC::Store.new(db) +store.install! # creates tables (idempotent) +``` + +### Appending messages + +```ruby +cmd = CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) +position = store.append(cmd) # returns the assigned position +``` + +### Reading with query conditions + +```ruby +# Build conditions for a specific course +conditions = CourseCreated.to_conditions(course_id: 'c1') +# => [QueryCondition('courses.created', 'course_id', 'c1')] + +# Read matching messages +result = store.read(conditions) +result.messages # => [PositionedMessage, ...] +result.guard # => ConsistencyGuard (for optimistic concurrency) +``` + +### Optimistic concurrency + +```ruby +result = store.read(conditions) + +# ... later, append with conflict detection +store.append(new_events, guard: result.guard) +# raises Sourced::ConcurrentAppendError if conflicting messages +# were appended after the read +``` + +### Partition reads + +`read_partition` uses AND semantics — a message is included only when every partition attribute it declares matches the given value. + +```ruby +result = store.read_partition( + { course_id: 'c1' }, + handled_types: ['courses.created', 'courses.enrolled'] +) +``` + +## Deciders + +Deciders handle commands, enforce invariants, and produce events. They rebuild state from event history before each decision. + +```ruby +class CourseDecider < Sourced::CCC::Decider + # Defines the consistency boundary + partition_by :course_name + + # Initial state factory (receives partition values array) + state do |_partition_values| + { name_taken: false } + end + + # Evolve state from events (rebuilds history) + evolve CourseCreated do |state, _event| + state[:name_taken] = true + end + + # Command handler — enforce invariants, then produce events + command CreateCourse do |state, cmd| + raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] + + event CourseCreated, + course_id: cmd.payload.course_id, + course_name: cmd.payload.course_name + end +end +``` + +### Synchronous command handling + +`CCC.handle!` loads history, runs the decider, appends the command + events, and advances consumer group offsets — all in one call. Designed for web controllers. + +```ruby +cmd = CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) +cmd, decider, events = Sourced::CCC.handle!(CourseDecider, cmd) + +if cmd.valid? + # Success — events were appended +else + # Validation failure — cmd.errors has details +end +``` + +Raises `Sourced::ConcurrentAppendError` on conflicts, or `RuntimeError` on domain invariant violations (e.g. "Course already exists"). + +### Loading a decider's state + +```ruby +decider, read_result = Sourced::CCC.load(CourseDecider, course_name: 'Algebra') +decider.state # => { name_taken: true } +``` + +## Projectors + +Projectors consume events to build read models. Two flavours: + +### EventSourced projector + +Rebuilds state from full history on every batch (like deciders). + +```ruby +class CourseCatalogProjector < Sourced::CCC::Projector::EventSourced + partition_by :course_id + + state do |_partition_values| + { course_id: nil, course_name: nil, students: [] } + end + + evolve CourseCreated do |state, event| + state[:course_id] = event.payload.course_id + state[:course_name] = event.payload.course_name + end + + evolve StudentEnrolled do |state, event| + state[:students] << event.payload.student_id + end + + # Sync block runs within the store transaction after evolving + sync do |state:, messages:, **| + next unless state[:course_id] + # Write projection to disk, database, cache, etc. + File.write("projections/#{state[:course_id]}.json", state.to_json) + end +end +``` + +### StateStored projector + +Loads persisted state via the `state` block, evolves only new (unprocessed) messages. + +```ruby +class MyProjector < Sourced::CCC::Projector::StateStored + partition_by :course_id + + state do |partition_values| + # Load existing state from your storage + existing = MyDB.find(partition_values.first) + existing || { course_id: nil, students: [] } + end + + evolve StudentEnrolled do |state, event| + state[:students] << event.payload.student_id + end + + sync do |state:, messages:, **| + MyDB.upsert(state) + end +end +``` + +## Reactions + +Both deciders and projectors can react to events to produce new commands or events, enabling workflow orchestration. + +```ruby +class EnrolmentDecider < Sourced::CCC::Decider + partition_by :course_id + + # ... evolve and command handlers ... + + # React to an event by producing new messages + reaction StudentEnrolled do |state, event| + NotifyStudent.new(payload: { student_id: event.payload.student_id }) + end +end +``` + +Reactions are skipped during replay (when `replaying: true`), so side effects don't re-fire. + +## Configuration + +```ruby +require 'sourced/ccc' + +Sourced::CCC.configure do |c| + # Pass a Sequel SQLite connection or a CCC::Store instance + c.store = Sequel.sqlite('my_app.db') + + # Optional settings + c.worker_count = 4 # background worker fibers (default: 2) + c.batch_size = 50 # messages per claim (default: 50) + c.catchup_interval = 5 # seconds between catch-up polls (default: 5) + c.max_drain_rounds = 10 # max drain iterations per pickup (default: 10) + c.claim_ttl_seconds = 120 # stale claim threshold (default: 120) + c.housekeeping_interval = 30 # heartbeat/reap cycle (default: 30) +end +``` + +## Registering reactors + +```ruby +Sourced::CCC.register(CourseDecider) +Sourced::CCC.register(EnrolmentDecider) +Sourced::CCC.register(CourseCatalogProjector) +``` + +This registers the reactor's consumer group with the store and adds it to the global router. + +## Background processing + +The supervisor starts workers that claim partitions, process messages, and ack offsets. + +```ruby +# Start blocking (handles INT/TERM signals for graceful shutdown) +Sourced::CCC::Supervisor.start + +# Or create and start manually +supervisor = Sourced::CCC::Supervisor.new( + router: Sourced::CCC.router, + count: 4 +) +supervisor.start +``` + +### How it works + +1. **Store** appends messages and notifies listeners of new message types +2. **Dispatcher** routes notifications to a `WorkQueue`, mapping message types to interested reactors +3. **Workers** pop reactors from the queue, claim a partition via `Router#handle_next_for`, process messages, and ack +4. **CatchUpPoller** periodically pushes all reactors as a safety net (handles missed notifications) +5. **StaleClaimReaper** releases claims held by dead workers + +### Router (direct usage) + +The router can also be used directly for testing or scripting: + +```ruby +router = Sourced::CCC.router + +# Process one batch for a specific reactor +router.handle_next_for(CourseDecider) + +# Drain all pending work across all reactors +router.drain +``` + +## Consumer groups + +Each reactor class is a consumer group. The store tracks per-partition offsets so multiple reactors process the same events independently. + +```ruby +store = Sourced::CCC.store + +# Manual consumer group management +store.stop_consumer_group('CourseDecider') +store.start_consumer_group('CourseDecider') +store.reset_consumer_group('CourseDecider') # reprocess from beginning + +store.consumer_group_active?('CourseDecider') # => true/false +``` + +## Full example + +See `examples/ccc_app/` for a complete Sinatra application with: +- Two deciders (course creation with name uniqueness, student enrolment with capacity limits) +- An event-sourced projector writing JSON files +- Synchronous command handling via `CCC.handle!` in HTTP endpoints +- Background worker processing via Falcon + +## Design reference + +- Design article: `plans/ccc/ccc.md` +- TypeScript reference: [Boundless SQLite storage](https://github.com/SBortz/boundless) From 926fd12b0fba103416d60473318951195f9a0e71 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 28 Feb 2026 16:43:03 +0000 Subject: [PATCH 028/115] Accept #group_id objects in CCC Store consumer group methods stop_consumer_group, start_consumer_group, reset_consumer_group, and consumer_group_active? now accept either a String or any object responding to #group_id (e.g. a reactor class). Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/README.md | 14 +++++++++----- lib/sourced/ccc/store.rb | 21 +++++++++++++++++---- spec/sourced/ccc/store_spec.rb | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 743a7652..be4d5729 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -294,15 +294,19 @@ router.drain Each reactor class is a consumer group. The store tracks per-partition offsets so multiple reactors process the same events independently. +The lifecycle methods (`stop_consumer_group`, `start_consumer_group`, `reset_consumer_group`, `consumer_group_active?`) accept either a String group ID or any object responding to `#group_id` (e.g. a reactor class). + ```ruby store = Sourced::CCC.store -# Manual consumer group management -store.stop_consumer_group('CourseDecider') -store.start_consumer_group('CourseDecider') -store.reset_consumer_group('CourseDecider') # reprocess from beginning +# Pass reactor classes directly +store.stop_consumer_group(CourseDecider) +store.start_consumer_group(CourseDecider) +store.reset_consumer_group(CourseDecider) # reprocess from beginning +store.consumer_group_active?(CourseDecider) # => true/false -store.consumer_group_active?('CourseDecider') # => true/false +# Or use plain strings +store.stop_consumer_group('CourseApp::CourseDecider') ``` ## Full example diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 967a3597..3ad322a8 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -290,9 +290,10 @@ def register_consumer_group(group_id) # Whether the consumer group exists and is active. # - # @param group_id [String] + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ # @return [Boolean] def consumer_group_active?(group_id) + group_id = resolve_group_id(group_id) row = db[:ccc_consumer_groups].where(group_id: group_id).select(:status).first return false unless row @@ -301,17 +302,19 @@ def consumer_group_active?(group_id) # Stop a consumer group. Stopped groups are skipped by {#claim_next}. # - # @param group_id [String] + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ # @return [void] def stop_consumer_group(group_id) + group_id = resolve_group_id(group_id) db[:ccc_consumer_groups].where(group_id: group_id).update(status: STOPPED, updated_at: Time.now.iso8601) end # Re-activate a stopped consumer group, clearing retry state. # - # @param group_id [String] + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ # @return [void] def start_consumer_group(group_id) + group_id = resolve_group_id(group_id) db[:ccc_consumer_groups] .where(group_id: group_id) .update(status: ACTIVE, retry_at: nil, error_context: nil, updated_at: Time.now.iso8601) @@ -343,9 +346,10 @@ def updating_consumer_group(group_id) # Delete all offsets for a consumer group, resetting it to process from the beginning. # - # @param group_id [String] + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ # @return [void] def reset_consumer_group(group_id) + group_id = resolve_group_id(group_id) cg = db[:ccc_consumer_groups].where(group_id: group_id).first return unless cg @@ -569,6 +573,15 @@ def clear! private + # Resolve a group_id argument that is either a String + # or an object responding to +#group_id+. + # + # @param group_id [String, #group_id] + # @return [String] + def resolve_group_id(group_id) + group_id.respond_to?(:group_id) ? group_id.group_id : group_id + end + # Build canonical partition key string from attribute names and values. # Sorted by attribute name for deterministic uniqueness. # diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 3022b291..44f61b5f 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -527,6 +527,12 @@ module CCCStoreTestMessages it 'returns false for nonexistent group' do expect(store.consumer_group_active?('nope')).to be false end + + it 'accepts an object responding to #group_id' do + store.register_consumer_group('my-group') + reactor = double('reactor', group_id: 'my-group') + expect(store.consumer_group_active?(reactor)).to be true + end end describe '#stop/start_consumer_group' do @@ -539,6 +545,17 @@ module CCCStoreTestMessages expect(store.consumer_group_active?('my-group')).to be true end + it 'accepts an object responding to #group_id' do + store.register_consumer_group('my-group') + reactor = double('reactor', group_id: 'my-group') + + store.stop_consumer_group(reactor) + expect(store.consumer_group_active?('my-group')).to be false + + store.start_consumer_group(reactor) + expect(store.consumer_group_active?('my-group')).to be true + end + it 'start_consumer_group clears retry_at and error_context' do store.register_consumer_group('my-group') @@ -631,6 +648,22 @@ module CCCStoreTestMessages store.reset_consumer_group('my-group') expect(db[:ccc_offsets].count).to eq(0) end + + it 'accepts an object responding to #group_id' do + store.register_consumer_group('my-group') + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + store.claim_next('my-group', + partition_by: 'device_id', + handled_types: ['store_test.device.registered'], + worker_id: 'w-1') + + reactor = double('reactor', group_id: 'my-group') + expect(db[:ccc_offsets].count).to be > 0 + store.reset_consumer_group(reactor) + expect(db[:ccc_offsets].count).to eq(0) + end end describe '#claim_next (single attribute partition)' do From ce8239626bce0dd9ef0fb588c00f055f4e0fb44b Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 11 Mar 2026 16:42:34 +0000 Subject: [PATCH 029/115] Syntax --- lib/sourced/ccc/store.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 3ad322a8..4c2d2cbe 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -215,14 +215,14 @@ def append(messages, guard: nil) def read(conditions, from_position: nil, limit: nil) conditions = Array(conditions) if conditions.empty? - guard = ConsistencyGuard.new(conditions: conditions, last_position: from_position || latest_position) - return ReadResult.new(messages: [], guard: guard) + guard = ConsistencyGuard.new(conditions:, last_position: from_position || latest_position) + return ReadResult.new(messages: [], guard:) end - messages = query_messages(conditions, from_position: from_position, limit: limit) + messages = query_messages(conditions, from_position:, limit:) last_pos = messages.any? ? messages.last.position : (from_position || latest_position) - guard = ConsistencyGuard.new(conditions: conditions, last_position: last_pos) - ReadResult.new(messages: messages, guard: guard) + guard = ConsistencyGuard.new(conditions:, last_position:) + ReadResult.new(messages:, guard:) end # Read messages for a specific partition using AND semantics. @@ -242,8 +242,8 @@ def read_partition(partition_attrs, handled_types:, from_position: 0) # If any key pair doesn't exist in the store, no messages can match if key_pair_ids.size < partition_attrs.size - guard = ConsistencyGuard.new(conditions: [], last_position: from_position) - return ReadResult.new(messages: [], guard: guard) + guard = ConsistencyGuard.new(conditions: [], last_position:) + return ReadResult.new(messages: [], guard:) end messages = fetch_partition_messages(key_pair_ids, from_position, handled_types) From 0e1901cb9631e783855f29eda1b3d340acab2c79 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 11 Mar 2026 16:43:11 +0000 Subject: [PATCH 030/115] Design refs not in repo --- lib/sourced/ccc/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index be4d5729..816194e6 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -316,8 +316,3 @@ See `examples/ccc_app/` for a complete Sinatra application with: - An event-sourced projector writing JSON files - Synchronous command handling via `CCC.handle!` in HTTP endpoints - Background worker processing via Falcon - -## Design reference - -- Design article: `plans/ccc/ccc.md` -- TypeScript reference: [Boundless SQLite storage](https://github.com/SBortz/boundless) From 7e579aeffb77aa2f665c91ad07d244e7f92cc727 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 11 Mar 2026 17:21:04 +0000 Subject: [PATCH 031/115] Fix missing vars --- lib/sourced/ccc/store.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 4c2d2cbe..1db7cd19 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -220,7 +220,7 @@ def read(conditions, from_position: nil, limit: nil) end messages = query_messages(conditions, from_position:, limit:) - last_pos = messages.any? ? messages.last.position : (from_position || latest_position) + last_position = messages.any? ? messages.last.position : (from_position || latest_position) guard = ConsistencyGuard.new(conditions:, last_position:) ReadResult.new(messages:, guard:) end @@ -242,7 +242,7 @@ def read_partition(partition_attrs, handled_types:, from_position: 0) # If any key pair doesn't exist in the store, no messages can match if key_pair_ids.size < partition_attrs.size - guard = ConsistencyGuard.new(conditions: [], last_position:) + guard = ConsistencyGuard.new(conditions: [], last_position: from_position) return ReadResult.new(messages: [], guard:) end From 84ea18f1390af8ba6e8cc955c4e4061c5c5124c2 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 11 Mar 2026 17:44:03 +0000 Subject: [PATCH 032/115] Add #dispatch helper to CCC::Decider, allow resolving message classes by symbol name --- lib/sourced/ccc/consumer.rb | 41 +++++++++ lib/sourced/ccc/decider.rb | 20 +++- lib/sourced/ccc/message.rb | 14 +++ lib/sourced/ccc/react.rb | 153 ++++++++++++++++++++++++++++--- spec/sourced/ccc/decider_spec.rb | 29 ++++++ spec/sourced/ccc/react_spec.rb | 125 +++++++++++++++++++++++++ 6 files changed, 367 insertions(+), 15 deletions(-) diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index 9170a70b..6b806ad2 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -33,6 +33,11 @@ def retry(time, **ctx) # Shared consumer configuration for CCC reactors. # Extended (not included) onto reactor classes. module Consumer + def self.extended(base) + super + base.extend ClassMethods + end + def partition_keys @partition_keys ||= [] end @@ -82,6 +87,42 @@ def each_with_partial_ack(messages) end results end + + module ClassMethods + # Resolve a CCC message class from a symbol or type-like string. + # + # Symbols are normalized by replacing dots with underscores before + # matching against registered message types. For example, + # +:course_created+ matches "course.created" and + # "course_created". + # + # @param message_symbol [Symbol, String] symbolic message identifier + # @return [Class, nil] matching CCC message class, or +nil+ if none found + # + # @example + # CourseDecider[:courses_created] + # # => CourseCreated + def [](message_symbol) + normalized = message_symbol.to_s.tr('.', '_') + find_registered_message_class(normalized) + end + + private + + def find_registered_message_class(normalized_name, base = CCC::Message) + base.registry.keys.each do |type| + klass = base.registry[type] + return klass if type.tr('.', '_') == normalized_name + end + + base.subclasses.each do |subclass| + klass = find_registered_message_class(normalized_name, subclass) + return klass if klass + end + + nil + end + end end end end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index d07f15e0..34969a0a 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -77,8 +77,26 @@ def decide(command) @uncommitted_events.dup end - # Called from within command handlers to produce events. + # Produce a new event from within a command handler and apply it + # to the decider's in-memory state immediately. + # + # Accepts either a CCC message class or a symbol resolved via .[]. + # + # @param event_class [Class, Symbol] event class or symbolic event name + # @param payload [Hash] payload attributes for the event + # @return [CCC::Message] the newly built event + # + # @example Produce by class + # command RegisterDevice do |_state, cmd| + # event DeviceRegistered, device_id: cmd.payload.device_id + # end + # + # @example Produce by symbol + # command RegisterDevice do |_state, cmd| + # event :device_registered, device_id: cmd.payload.device_id + # end def event(event_class, payload = {}) + event_class = self.class[event_class] if event_class.is_a?(Symbol) evt = event_class.new(payload: payload) @uncommitted_events << evt evolve([evt]) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index cde6d401..63d3ccbf 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -143,6 +143,20 @@ def initialize(attrs = {}) super(attrs) end + def with_metadata(meta = {}) + return self if meta.empty? + + with(metadata: metadata.merge(meta)) + end + + def at(datetime) + if datetime < created_at + raise Sourced::PastMessageDateError, "Message #{type} can't be delayed to a date in the past" + end + + with(created_at: datetime) + end + # Set causation and correlation IDs on another message, establishing # a causal link from this message to +message+. Merges metadata. # diff --git a/lib/sourced/ccc/react.rb b/lib/sourced/ccc/react.rb index 165951ed..f55adf2f 100644 --- a/lib/sourced/ccc/react.rb +++ b/lib/sourced/ccc/react.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true +require 'set' + module Sourced module CCC # React mixin for CCC reactors. - # Reactions return raw CCC::Message instances (not correlated). - # The runtime handles auto-correlation via Actions::Append#execute. + # Supports the same dispatch-based reaction DSL as Sourced::React, + # adapted to CCC's stream-less messages. module React PREFIX = 'ccc_reaction' EMPTY_ARRAY = [].freeze @@ -14,14 +16,16 @@ def self.included(base) base.extend ClassMethods end - # Run the reaction handler for a single message. - # Returns raw messages (not correlated). - def react(message) - method_name = Sourced.message_method_name(PREFIX, message.class.to_s) - if respond_to?(method_name) - Array(send(method_name, state, message)).compact - else - EMPTY_ARRAY + # Run reaction handlers for one or more messages. + # Supports both explicit message returns and dispatch(...) calls. + def react(messages) + __handling_reactions(Array(messages)) do |message| + method_name = Sourced.message_method_name(PREFIX, message.class.to_s) + if respond_to?(method_name) + Array(send(method_name, state, message)).compact + else + EMPTY_ARRAY + end end end @@ -29,22 +33,143 @@ def reacts_to?(message) self.class.handled_messages_for_react.include?(message.class) end + private + + def __handling_reactions(messages) + messages.flat_map do |message| + @__reaction_dispatchers = [] + @__message_for_reaction = message + explicit = Array(yield(message)).compact.reject { |value| value.is_a?(Dispatcher) } + dispatched = @__reaction_dispatchers.map(&:message) + explicit + dispatched + end + ensure + @__reaction_dispatchers = [] + @__message_for_reaction = nil + end + + class Dispatcher + attr_reader :message + + def initialize(message) + @message = message + end + + def inspect = %(<#{self.class} #{@message}>) + + def at(datetime) + @message = @message.at(datetime) + self + end + + def with_metadata(attrs = {}) + @message = @message.with_metadata(attrs) + self + end + end + + # Queue a follow-up message from within a reaction block. + # + # The returned {Dispatcher} can be chained to delay the message + # or add metadata before it is appended. + # + # @param message_class [Class, Symbol] CCC message class, or a symbol + # resolved via .[] + # @param payload [Hash] message payload attributes + # @return [Dispatcher] chainable wrapper around the dispatched message + # + # @example Dispatch by class + # reaction StudentEnrolled do |_state, event| + # dispatch(NotifyStudent, student_id: event.payload.student_id) + # end + # + # @example Dispatch by symbol with delay and metadata + # reaction StudentEnrolled do |_state, event| + # dispatch(:notify_student, student_id: event.payload.student_id) + # .with_metadata(channel: 'email') + # .at(Time.now + 300) + # end + def dispatch(message_class, payload = {}) + message_class = self.class[message_class] if message_class.is_a?(Symbol) + message = @__message_for_reaction + .correlate(message_class.new(payload: payload)) + .with_metadata(producer: self.class.group_id) + + dispatcher = Dispatcher.new(message) + @__reaction_dispatchers << dispatcher + dispatcher + end + module ClassMethods def inherited(subclass) super handled_messages_for_react.each do |klass| subclass.handled_messages_for_react << klass end + catch_all_react_events.each do |klass| + subclass.catch_all_react_events << klass + end end def handled_messages_for_react @handled_messages_for_react ||= [] end - # Register a reaction handler for a CCC::Message subclass. - def reaction(message_class, &block) - handled_messages_for_react << message_class - define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) + def catch_all_react_events + @catch_all_react_events ||= Set.new + end + + # Register a reaction handler for one or more CCC message types. + # + # Accepts message classes, symbols resolved via .[], + # multiple arguments, or no arguments for a catch-all reaction across + # all evolve types without an explicit handler. + # + # @example React to a specific event class + # reaction StudentEnrolled do |state, event| + # dispatch(NotifyStudent, student_id: event.payload.student_id) + # end + # + # @example React to a symbol-resolved message class + # reaction :student_enrolled do |state, event| + # dispatch(:notify_student, student_id: event.payload.student_id) + # end + def reaction(*args, &block) + case args + in [] + handled_messages_for_evolve.each do |message_class| + method_name = Sourced.message_method_name(PREFIX, message_class.to_s) + next if instance_methods.include?(method_name.to_sym) + + catch_all_react_events << message_class + reaction(message_class, &block) + end + + in [Symbol => message_symbol] + message_class = self[message_symbol].tap do |klass| + raise( + ArgumentError, + "Cannot resolve message symbol #{message_symbol.inspect} for #{self}.reaction" + ) unless klass + end + + reaction(message_class, &block) + in [Class => message_class] if message_class < CCC::Message + __validate_message_for_reaction!(message_class) + handled_messages_for_react << message_class + define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) if block_given? + in Array => values if values.none?(&:nil?) + values.each { |value| reaction(value, &block) } + else + raise( + ArgumentError, + "Invalid arguments #{args.inspect} for #{self}.reaction" + ) + end + end + + def __validate_message_for_reaction!(_message_class) + # no-op end end end diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/sourced/ccc/decider_spec.rb index 0229f8bc..b7dc1c5d 100644 --- a/spec/sourced/ccc/decider_spec.rb +++ b/spec/sourced/ccc/decider_spec.rb @@ -23,6 +23,16 @@ module CCCDeciderTestMessages NotifyBound = Sourced::CCC::Message.define('decider_test.notify_bound') do attribute :device_id, String end + + SymbolicBound = Sourced::CCC::Message.define('decider_test.symbolic_bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDeviceWithSymbol = Sourced::CCC::Message.define('decider_test.bind_device_with_symbol') do + attribute :device_id, String + attribute :asset_id, String + end end class TestDeviceDecider < Sourced::CCC::Decider @@ -45,6 +55,12 @@ class TestDeviceDecider < Sourced::CCC::Decider event CCCDeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id end + command CCCDeciderTestMessages::BindDeviceWithSymbol do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event :decider_test_symbolic_bound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end + reaction CCCDeciderTestMessages::DeviceBound do |_state, evt| CCCDeciderTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) end @@ -54,6 +70,7 @@ class TestDeviceDecider < Sourced::CCC::Decider describe '.command' do it 'registers handler and #decide runs it' do expect(TestDeviceDecider.handled_commands).to include(CCCDeciderTestMessages::BindDevice) + expect(TestDeviceDecider.handled_commands).to include(CCCDeciderTestMessages::BindDeviceWithSymbol) instance = TestDeviceDecider.new instance.instance_variable_set(:@state, { exists: true, bound: false }) @@ -79,6 +96,18 @@ class TestDeviceDecider < Sourced::CCC::Decider expect(events.size).to eq(1) expect(instance.state[:bound]).to be true end + + it 'resolves event classes from symbols' do + instance = TestDeviceDecider.new + instance.instance_variable_set(:@state, { exists: true, bound: false }) + + events = instance.decide( + CCCDeciderTestMessages::BindDeviceWithSymbol.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + expect(events.size).to eq(1) + expect(events.first).to be_a(CCCDeciderTestMessages::SymbolicBound) + end end describe '.handle_batch' do diff --git a/spec/sourced/ccc/react_spec.rb b/spec/sourced/ccc/react_spec.rb index 31c99a22..962c355c 100644 --- a/spec/sourced/ccc/react_spec.rb +++ b/spec/sourced/ccc/react_spec.rb @@ -15,12 +15,29 @@ module CCCReactTestMessages Unhandled = Sourced::CCC::Message.define('react_test.unhandled') do attribute :foo, String end + + Wildcarded = Sourced::CCC::Message.define('react_test.wildcarded') do + attribute :thing_id, String + end + + DelayedCommand = Sourced::CCC::Message.define('react_test.delayed.command') do + attribute :thing_id, String + end + + MultiReaction = Sourced::CCC::Message.define('react_test.multi.reaction') do + attribute :thing_id, String + end + + AnotherMultiReaction = Sourced::CCC::Message.define('react_test.another.multi.reaction') do + attribute :thing_id, String + end end RSpec.describe Sourced::CCC::React do let(:reactor_class) do Class.new do include Sourced::CCC::React + extend Sourced::CCC::Consumer def state {} @@ -46,6 +63,13 @@ def state expect(result.first.causation_id).to eq(result.first.id) end + it 'accepts a single message or an array of messages' do + instance = reactor_class.new + msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + + expect(instance.react([msg]).map(&:class)).to eq([CCCReactTestMessages::DoNext]) + end + it 'returns empty array for unregistered types' do instance = reactor_class.new msg = CCCReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) @@ -90,4 +114,105 @@ def state expect(result.size).to eq(1) end end + + describe 'dispatch DSL' do + let(:dsl_reactor_class) do + Class.new do + include Sourced::CCC::React + extend Sourced::CCC::Consumer + + consumer_group 'ccc-reactor' + + def state + { source: 'state' } + end + + def self.handled_messages_for_evolve + [CCCReactTestMessages::Wildcarded] + end + + reaction CCCReactTestMessages::SomethingHappened do |_state, msg| + dispatch(CCCReactTestMessages::DoNext, thing_id: msg.payload.thing_id) + end + + reaction :react_test_delayed_command do |_state, msg| + dispatch(:react_test_delayed_command, thing_id: msg.payload.thing_id) + .with_metadata(foo: 'bar') + .at(Time.now + 10) + end + + reaction CCCReactTestMessages::MultiReaction, CCCReactTestMessages::AnotherMultiReaction do |_state, msg| + dispatch(CCCReactTestMessages::DoNext, thing_id: msg.payload.thing_id) + end + + reaction do |state, msg| + dispatch(CCCReactTestMessages::DoNext, thing_id: "#{state[:source]}-#{msg.payload.thing_id}") + end + end + end + + it 'supports dispatch with correlation, producer metadata, metadata chaining, and delays' do + now = Time.now + Timecop.freeze(now) do + instance = dsl_reactor_class.new + source = CCCReactTestMessages::DelayedCommand.new(payload: { thing_id: 't1' }) + + result = instance.react(source) + + expect(result.map(&:class)).to eq([CCCReactTestMessages::DelayedCommand]) + expect(result.first.causation_id).to eq(source.id) + expect(result.first.correlation_id).to eq(source.correlation_id) + expect(result.first.metadata[:producer]).to eq('ccc-reactor') + expect(result.first.metadata[:foo]).to eq('bar') + expect(result.first.created_at).to eq(now + 10) + end + end + + it 'supports dispatching multiple messages from one reaction block' do + klass = Class.new do + include Sourced::CCC::React + extend Sourced::CCC::Consumer + + def state + {} + end + + reaction CCCReactTestMessages::SomethingHappened do |_state, msg| + dispatch(CCCReactTestMessages::DoNext, thing_id: msg.payload.thing_id) + dispatch(CCCReactTestMessages::DelayedCommand, thing_id: msg.payload.thing_id) + end + end + + result = klass.new.react( + CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + ) + + expect(result.map(&:class)).to eq([ + CCCReactTestMessages::DoNext, + CCCReactTestMessages::DelayedCommand + ]) + end + + it 'supports wildcard reactions for evolve types without explicit handlers' do + result = dsl_reactor_class.new.react( + CCCReactTestMessages::Wildcarded.new(payload: { thing_id: 't1' }) + ) + + expect(result.map(&:class)).to eq([CCCReactTestMessages::DoNext]) + expect(result.first.payload.thing_id).to eq('state-t1') + expect(dsl_reactor_class.catch_all_react_events).to eq(Set[ + CCCReactTestMessages::Wildcarded + ]) + end + + it 'supports reactions registered for multiple message classes' do + instance = dsl_reactor_class.new + + first = instance.react(CCCReactTestMessages::MultiReaction.new(payload: { thing_id: 't1' })) + second = instance.react(CCCReactTestMessages::AnotherMultiReaction.new(payload: { thing_id: 't2' })) + + expect(first.map(&:class)).to eq([CCCReactTestMessages::DoNext]) + expect(second.map(&:class)).to eq([CCCReactTestMessages::DoNext]) + end + end end From 68df28d9c703fce5d675e7ab13b97d77c4705905 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 11 Mar 2026 18:16:30 +0000 Subject: [PATCH 033/115] Add CCC scheduled message support Port delayed dispatch support to the CCC runtime. This adds a dedicated scheduled message path for reactions that return messages with a future created_at, such as: reaction SomeEvent do |_state, evt| dispatch(SomeCommand, ...).at(Time.now + 10) end What changed: - add CCC::Actions.build_for to split produced messages into immediate append actions and delayed schedule actions - add CCC::Actions::Schedule for persisting delayed CCC messages - update CCC deciders and projectors to use the action builder so delayed reaction output is scheduled instead of appended immediately - add ccc_scheduled_messages storage plus Store#schedule_messages and Store#update_schedule! to persist and later promote due messages into the flat CCC log - add CCC::ScheduledMessagePoller and wire it into the CCC dispatcher so due scheduled messages are promoted in the background alongside catch-up polling and stale-claim reaping - extend CCC specs to cover store-level scheduling, delayed reaction action generation, dispatcher/supervisor wiring, and promotion of scheduled messages into the main log - add YARD comments to the public methods and modules touched by this work How it is used: - reaction blocks can continue to use dispatch(...).at(time) - when the returned CCC message has a future created_at, CCC now stores it in the scheduled message table instead of appending it immediately - the scheduled message poller periodically calls Store#update_schedule! to move due messages into the main CCC log - promoted messages are appended through the normal CCC store path, so they get positions, key extraction/indexing, and notifier fanout just like regular appended messages - once promoted, existing CCC workers can claim and process them without any further API changes in reactor code --- lib/sourced/ccc/actions.rb | 67 +++++++++++++++++++ lib/sourced/ccc/decider.rb | 29 +++++++- lib/sourced/ccc/dispatcher.rb | 11 +++ lib/sourced/ccc/projector.rb | 16 ++++- lib/sourced/ccc/scheduled_message_poller.rb | 38 +++++++++++ lib/sourced/ccc/store.rb | 74 +++++++++++++++++++++ spec/sourced/ccc/decider_spec.rb | 53 +++++++++++++++ spec/sourced/ccc/dispatcher_spec.rb | 37 ++++++++++- spec/sourced/ccc/projector_spec.rb | 39 +++++++++++ spec/sourced/ccc/store_spec.rb | 48 +++++++++++++ spec/sourced/ccc/supervisor_spec.rb | 6 +- 11 files changed, 406 insertions(+), 12 deletions(-) create mode 100644 lib/sourced/ccc/scheduled_message_poller.rb diff --git a/lib/sourced/ccc/actions.rb b/lib/sourced/ccc/actions.rb index f2723dbd..39de31f2 100644 --- a/lib/sourced/ccc/actions.rb +++ b/lib/sourced/ccc/actions.rb @@ -2,10 +2,34 @@ module Sourced module CCC + # Action builders and executable action types for CCC reactors. module Actions OK = :ok RETRY = :retry + # Split produced messages into immediate append actions and delayed schedule actions. + # + # @param messages [CCC::Message, Array] messages produced by a reactor + # @param guard [ConsistencyGuard, nil] optional concurrency guard for immediate appends + # @param source [CCC::Message, nil] source message used for correlation when executing + # @param correlated [Boolean] whether +messages+ are already correlated + # @return [Array] executable actions in append/schedule groups + def self.build_for(messages, guard: nil, source: nil, correlated: false) + actions = [] + messages = Array(messages) + return actions if messages.empty? + + now = Time.now + to_schedule, to_append = messages.partition { |message| message.created_at > now } + + actions << Append.new(to_append, guard: guard, source: source, correlated: correlated) if to_append.any? + to_schedule.group_by(&:created_at).each do |at, scheduled_messages| + actions << Schedule.new(scheduled_messages, at:, source: source, correlated: correlated) + end + + actions + end + # Append messages to the CCC store with optional consistency guard. # Auto-correlates messages with the source message at execution time. # @@ -17,6 +41,10 @@ module Actions class Append attr_reader :messages, :guard, :source + # @param messages [CCC::Message, Array] messages to append + # @param guard [ConsistencyGuard, nil] optional optimistic concurrency guard + # @param source [CCC::Message, nil] explicit correlation source + # @param correlated [Boolean] whether +messages+ are already correlated def initialize(messages, guard: nil, source: nil, correlated: false) @messages = Array(messages) @guard = guard @@ -24,6 +52,7 @@ def initialize(messages, guard: nil, source: nil, correlated: false) @correlated = correlated end + # @return [Boolean] whether messages should be appended without re-correlation def correlated? = @correlated # @param store [CCC::Store] @@ -41,14 +70,52 @@ def execute(store, source_message) end end + # Schedule messages for future promotion into the main CCC log. + class Schedule + attr_reader :messages, :at, :source + + # @param messages [CCC::Message, Array] messages to schedule + # @param at [Time] when the messages should become available for promotion + # @param source [CCC::Message, nil] explicit correlation source + # @param correlated [Boolean] whether +messages+ are already correlated + def initialize(messages, at:, source: nil, correlated: false) + @messages = Array(messages) + @at = at + @source = source + @correlated = correlated + end + + # @return [Boolean] whether messages should be scheduled without re-correlation + def correlated? = @correlated + + # @param store [CCC::Store] + # @param source_message [CCC::Message] default message to correlate from + # @return [Array] correlated messages that were scheduled + def execute(store, source_message) + to_schedule = if @correlated + messages + else + correlate_from = @source || source_message + messages.map { |message| correlate_from.correlate(message) } + end + store.schedule_messages(to_schedule, at: at) + to_schedule + end + end + # Execute a synchronous side effect within the current transaction. class Sync + # @param work [#call] callable to execute def initialize(work) @work = work end + # @return [Object] the callable's return value def call = @work.call + # @param _store [Object] unused + # @param _source_message [Object] unused + # @return [nil] def execute(_store, _source_message) call nil diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index 34969a0a..4130e7b3 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -2,6 +2,7 @@ module Sourced module CCC + # Reactor base class for command-handling workflows in CCC. class Decider include CCC::Evolve include CCC::React @@ -9,23 +10,34 @@ class Decider extend CCC::Consumer class << self + # @return [Array] command message classes handled by this decider def handled_commands @handled_commands ||= [] end # Messages to claim: commands to decide on + events to react to. # Evolve types are NOT included — they are only for context_for(). + # + # @return [Array] command and reaction message classes def handled_messages handled_commands + handled_messages_for_react end # Register a command handler. + # + # @param message_class [Class] CCC command class to handle + # @yield [state, message] command handler block + # @return [void] def command(message_class, &block) handled_commands << message_class define_method(Sourced.message_method_name('ccc_decide', message_class.to_s), &block) end - # Reactor interface — requests history: via signature. + # Build executable actions for a claimed batch. + # + # @param claim [ClaimResult] claimed partition batch + # @param history [ReadResult] event history for the partition + # @return [Array, PositionedMessage)>] action/source pairs def handle_batch(claim, history:) values = partition_keys.map { |k| claim.partition_value[k.to_s] } instance = new(values) @@ -36,12 +48,14 @@ def handle_batch(claim, history:) raw_events = instance.decide(msg) correlated_events = raw_events.map { |e| msg.correlate(e) } actions = [] - actions << Actions::Append.new(correlated_events, guard: history.guard, correlated: true) if correlated_events.any? + actions.concat( + Actions.build_for(correlated_events, guard: history.guard, correlated: true) + ) correlated_events.each do |evt| next unless instance.reacts_to?(evt) reaction_msgs = Array(instance.react(evt)) - actions << Actions::Append.new(reaction_msgs, source: evt) if reaction_msgs.any? + actions.concat(Actions.build_for(reaction_msgs, source: evt)) end actions += instance.sync_actions( @@ -55,6 +69,10 @@ def handle_batch(claim, history:) end end + # Copy registered command handlers into subclasses. + # + # @param subclass [Class] subclass being created + # @return [void] def inherited(subclass) super handled_commands.each do |cmd_class| @@ -65,11 +83,16 @@ def inherited(subclass) attr_reader :partition_values + # @param partition_values [Array] values for the decider's partition keys def initialize(partition_values = []) @partition_values = partition_values @uncommitted_events = [] end + # Decide a command against the decider's current in-memory state. + # + # @param command [CCC::Message] command to handle + # @return [Array] newly produced events def decide(command) @uncommitted_events = [] method_name = Sourced.message_method_name('ccc_decide', command.class.to_s) diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb index f90c466e..d268b4fc 100644 --- a/lib/sourced/ccc/dispatcher.rb +++ b/lib/sourced/ccc/dispatcher.rb @@ -3,6 +3,7 @@ require 'sourced/work_queue' require 'sourced/catchup_poller' require 'sourced/ccc/worker' +require 'sourced/ccc/scheduled_message_poller' require 'sourced/ccc/stale_claim_reaper' module Sourced @@ -143,6 +144,12 @@ def initialize( logger: logger ) + @scheduled_message_poller = ScheduledMessagePoller.new( + store: router.store, + interval: catchup_interval, + logger: logger + ) + @stale_claim_reaper = StaleClaimReaper.new( store: router.store, interval: housekeeping_interval, @@ -168,6 +175,9 @@ def spawn_into(task) # CatchUp poller task.send(s) { @catchup_poller.run } + # Scheduled message poller + task.send(s) { @scheduled_message_poller.run } + # Stale claim reaper task.send(s) { @stale_claim_reaper.run } @@ -186,6 +196,7 @@ def stop @logger.info "CCC::Dispatcher: stopping #{@workers.size} workers" @store_notifier.stop @catchup_poller.stop + @scheduled_message_poller.stop @stale_claim_reaper.stop @workers.each(&:stop) @work_queue.close(@workers.size) diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index 923bfd8a..a0b8b92f 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -2,6 +2,7 @@ module Sourced module CCC + # Reactor base class for CCC read-model projectors. class Projector include CCC::Evolve include CCC::React @@ -10,6 +11,8 @@ class Projector class << self # Projectors claim events they evolve from + events they react to. + # + # @return [Array] evolved and reacted-to message classes def handled_messages (handled_messages_for_evolve + handled_messages_for_react).uniq end @@ -32,7 +35,8 @@ def build_action_pairs(instance, claim) each_with_partial_ack(claim.messages) do |msg| next unless instance.reacts_to?(msg) reaction_msgs = Array(instance.react(msg)) - reaction_msgs.any? ? [Actions::Append.new(reaction_msgs), msg] : nil + actions = Actions.build_for(reaction_msgs) + actions.any? ? [actions, msg] : nil end end @@ -42,13 +46,16 @@ def build_action_pairs(instance, claim) attr_reader :partition_values + # @param partition_values [Array] values for the projector's partition keys def initialize(partition_values = []) @partition_values = partition_values end - # StateStored: loads persisted state via `state` block, evolves only new messages. + # Projector variant that evolves only the claimed messages on top of stored state. class StateStored < self class << self + # @param claim [ClaimResult] claimed partition batch + # @return [Array, PositionedMessage)>] action/source pairs def handle_batch(claim) instance = build_instance(claim) instance.evolve(claim.messages) @@ -57,9 +64,12 @@ def handle_batch(claim) end end - # EventSourced: rebuilds state from full history each time. + # Projector variant that rebuilds state from full history each time. class EventSourced < self class << self + # @param claim [ClaimResult] claimed partition batch + # @param history [ReadResult] full partition history + # @return [Array, PositionedMessage)>] action/source pairs def handle_batch(claim, history:) instance = build_instance(claim) instance.evolve(history.messages) diff --git a/lib/sourced/ccc/scheduled_message_poller.rb b/lib/sourced/ccc/scheduled_message_poller.rb new file mode 100644 index 00000000..93a8e62a --- /dev/null +++ b/lib/sourced/ccc/scheduled_message_poller.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Sourced + module CCC + # Periodically promotes due scheduled messages into the main CCC log. + class ScheduledMessagePoller + # @param store [CCC::Store] the CCC store containing scheduled messages + # @param interval [Numeric] polling interval in seconds + # @param logger [Object] logger instance + def initialize(store:, interval: 5, logger: CCC.config.logger) + @store = store + @interval = interval + @logger = logger + @running = false + end + + # Run the polling loop until {#stop} is called. + # + # @return [void] + def run + @running = true + while @running + promoted = @store.update_schedule! + @logger.info "CCC::ScheduledMessagePoller: appended #{promoted} scheduled messages" if promoted > 0 + sleep @interval + end + @logger.info 'CCC::ScheduledMessagePoller: stopped' + end + + # Signal the poller to stop after the current sleep cycle. + # + # @return [void] + def stop + @running = false + end + end + end +end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 1db7cd19..31d18d2d 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -9,6 +9,8 @@ module CCC class PositionedMessage < SimpleDelegator attr_reader :position + # @param message [CCC::Message] the wrapped message instance + # @param position [Integer] global log position def initialize(message, position) super(message) @position = position @@ -65,6 +67,7 @@ def installed? db.table_exists?(:ccc_messages) && db.table_exists?(:ccc_key_pairs) && db.table_exists?(:ccc_message_key_pairs) && + db.table_exists?(:ccc_scheduled_messages) && db.table_exists?(:ccc_consumer_groups) && db.table_exists?(:ccc_offsets) && db.table_exists?(:ccc_offset_key_pairs) && @@ -107,6 +110,16 @@ def install! SQL db.run('CREATE INDEX IF NOT EXISTS idx_ccc_mkp_key ON ccc_message_key_pairs(key_pair_id, message_position)') + db.run(<<~SQL) + CREATE TABLE IF NOT EXISTS ccc_scheduled_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + available_at TEXT NOT NULL, + message TEXT NOT NULL + ) + SQL + db.run('CREATE INDEX IF NOT EXISTS idx_ccc_scheduled_available_at ON ccc_scheduled_messages(available_at)') + db.run(<<~SQL) CREATE TABLE IF NOT EXISTS ccc_consumer_groups ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -205,6 +218,66 @@ def append(messages, guard: nil) last_position end + # Persist messages for future promotion into the main CCC log. + # + # @param messages [CCC::Message, Array] one or more delayed messages + # @param at [Time] when the messages should become available + # @return [Boolean] false when no messages were provided, true otherwise + def schedule_messages(messages, at:) + messages = Array(messages) + return false if messages.empty? + + now = Time.now + rows = messages.map do |message| + data = message.to_h + data[:metadata] = message.metadata.merge(scheduled_at: now) + { + created_at: now.iso8601, + available_at: at.iso8601, + message: JSON.dump(data) + } + end + + db.transaction do + db[:ccc_scheduled_messages].multi_insert(rows) + end + + true + end + + # Promote due scheduled messages into the main CCC log. + # + # Appended messages are re-inserted through {#append} so they are indexed, + # assigned fresh positions, and announced through the store notifier. + # + # @return [Integer] number of scheduled messages promoted + def update_schedule! + now = Time.now + + db.transaction do + rows = db[:ccc_scheduled_messages] + .where { available_at <= now.iso8601 } + .order(:id) + .limit(100) + .all + + return 0 if rows.empty? + + messages = rows.map do |row| + data = JSON.parse(row[:message], symbolize_names: true) + data[:created_at] = now + Message.from(data) + end + + append(messages) + + row_ids = rows.map { |row| row[:id] } + db[:ccc_scheduled_messages].where(id: row_ids).delete + + rows.size + end + end + # Query messages by conditions. Each condition matches on # (message_type AND key_name AND key_value). Multiple conditions are OR'd. # @@ -567,6 +640,7 @@ def clear! db[:ccc_message_key_pairs].delete db[:ccc_key_pairs].delete db[:ccc_messages].delete + db[:ccc_scheduled_messages].delete db[:ccc_workers].delete db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) end diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/sourced/ccc/decider_spec.rb index b7dc1c5d..a6fd616f 100644 --- a/spec/sourced/ccc/decider_spec.rb +++ b/spec/sourced/ccc/decider_spec.rb @@ -24,6 +24,10 @@ module CCCDeciderTestMessages attribute :device_id, String end + DelayedNotifyBound = Sourced::CCC::Message.define('decider_test.delayed_notify_bound') do + attribute :device_id, String + end + SymbolicBound = Sourced::CCC::Message.define('decider_test.symbolic_bound') do attribute :device_id, String attribute :asset_id, String @@ -66,6 +70,32 @@ class TestDeviceDecider < Sourced::CCC::Decider end end +class TestDelayedReactionDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'device-delayed-decider-test' + + state { |_| { exists: false, bound: false } } + + evolve CCCDeciderTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + evolve CCCDeciderTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command CCCDeciderTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event CCCDeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end + + reaction CCCDeciderTestMessages::DeviceBound do |_state, evt| + dispatch(CCCDeciderTestMessages::DelayedNotifyBound, device_id: evt.payload.device_id) + .at(Time.now + 10) + end +end + RSpec.describe Sourced::CCC::Decider do describe '.command' do it 'registers handler and #decide runs it' do @@ -183,6 +213,29 @@ class TestDeviceDecider < Sourced::CCC::Decider expect(source_msg).to eq(reg_positioned) end + it 'returns schedule actions for delayed reaction dispatches' do + reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + history_msgs = [Sourced::CCC::PositionedMessage.new(reg, 1)] + guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 1) + history = Sourced::CCC::ReadResult.new(messages: history_msgs, guard: guard) + + cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + claim = Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [Sourced::CCC::PositionedMessage.new(cmd, 2)], + replaying: false, + guard: guard + ) + + pairs = TestDelayedReactionDecider.handle_batch(claim, history: history) + actions = pairs.first.first + schedule_action = Array(actions).find { |action| action.is_a?(Sourced::CCC::Actions::Schedule) } + + expect(schedule_action).not_to be_nil + expect(schedule_action.messages.first).to be_a(CCCDeciderTestMessages::DelayedNotifyBound) + end + it 'invariant violation propagates as error' do guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 0) history = Sourced::CCC::ReadResult.new(messages: [], guard: guard) diff --git a/spec/sourced/ccc/dispatcher_spec.rb b/spec/sourced/ccc/dispatcher_spec.rb index 4d0f242a..5d3cb26e 100644 --- a/spec/sourced/ccc/dispatcher_spec.rb +++ b/spec/sourced/ccc/dispatcher_spec.rb @@ -19,6 +19,10 @@ module CCCDispatcherTestMessages attribute :device_id, String attribute :asset_id, String end + + DelayedNotify = Sourced::CCC::Message.define('dispatch_test.delayed_notify') do + attribute :device_id, String + end end class DispatchTestDecider < Sourced::CCC::Decider @@ -40,6 +44,11 @@ class DispatchTestDecider < Sourced::CCC::Decider raise 'Already bound' if state[:bound] event CCCDispatcherTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id end + + reaction CCCDispatcherTestMessages::DeviceBound do |_state, evt| + dispatch(CCCDispatcherTestMessages::DelayedNotify, device_id: evt.payload.device_id) + .at(Time.now + 2) + end end class DispatchTestProjector < Sourced::CCC::Projector::StateStored @@ -281,15 +290,15 @@ class DispatchTestProjector < Sourced::CCC::Projector::StateStored it 'spawns via #spawn when task responds to spawn' do task = double('Task') - # 1 notifier + 1 catchup_poller + 1 stale_claim_reaper + 2 workers = 5 spawns - expect(task).to receive(:spawn).exactly(5).times + # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns + expect(task).to receive(:spawn).exactly(6).times dispatcher.spawn_into(task) end it 'spawns via #async when task does not respond to spawn' do task = Object.new def task.async; end - expect(task).to receive(:async).exactly(5).times + expect(task).to receive(:async).exactly(6).times dispatcher.spawn_into(task) end @@ -341,4 +350,26 @@ def task.async; end dispatcher.stop end end + + describe 'scheduled message promotion' do + it 'promotes delayed reactions into the main log when due' do + store.append( + CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + CCCDispatcherTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + expect(router.handle_next_for(DispatchTestDecider)).to be true + expect(db[:ccc_scheduled_messages].count).to eq(1) + + Timecop.freeze(Time.now + 3) do + expect(store.update_schedule!).to eq(1) + end + + conds = CCCDispatcherTestMessages::DelayedNotify.to_conditions(device_id: 'd1') + result = store.read(conds) + expect(result.messages.map(&:class)).to include(CCCDispatcherTestMessages::DelayedNotify) + end + end end diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb index ca1d6577..44aeeea6 100644 --- a/spec/sourced/ccc/projector_spec.rb +++ b/spec/sourced/ccc/projector_spec.rb @@ -17,6 +17,10 @@ module CCCProjectorTestMessages NotifyArchive = Sourced::CCC::Message.define('projector_test.notify_archive') do attribute :list_id, String end + + DelayedNotifyArchive = Sourced::CCC::Message.define('projector_test.delayed_notify_archive') do + attribute :list_id, String + end end class TestItemProjector < Sourced::CCC::Projector::StateStored @@ -71,6 +75,24 @@ class TestItemESProjector < Sourced::CCC::Projector::EventSourced end end +class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored + partition_by :list_id + consumer_group 'delayed-item-projector-test' + + state do |(list_id)| + { list_id: list_id, items: [] } + end + + evolve CCCProjectorTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + reaction CCCProjectorTestMessages::ItemArchived do |_state, msg| + dispatch(CCCProjectorTestMessages::DelayedNotifyArchive, list_id: msg.payload.list_id) + .at(Time.now + 10) + end +end + RSpec.describe Sourced::CCC::Projector do describe '.handled_messages' do it 'includes evolve and react types' do @@ -131,6 +153,23 @@ def make_claim(messages, replaying: false) expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) end + it 'returns schedule actions for delayed reactions' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: false) + + pairs = TestDelayedItemProjector.handle_batch(claim) + + schedule_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |action| action.is_a?(Sourced::CCC::Actions::Schedule) } + + expect(schedule_actions.size).to eq(1) + expect(schedule_actions.first.messages.first).to be_a(CCCProjectorTestMessages::DelayedNotifyArchive) + end + it 'skips reactions when replaying' do msgs = [ Sourced::CCC::PositionedMessage.new( diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 44f61b5f..14d7461b 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -162,6 +162,54 @@ module CCCStoreTestMessages end end + describe '#schedule_messages and #update_schedule!' do + it 'stores delayed messages outside the main log until due' do + now = Time.now + delayed = CCCStoreTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' } + ).at(now + 60) + + expect(store.schedule_messages([delayed], at: delayed.created_at)).to be true + expect(store.latest_position).to eq(0) + expect(db[:ccc_scheduled_messages].count).to eq(1) + expect(store.update_schedule!).to eq(0) + end + + it 'promotes due messages into the flat log and preserves metadata' do + now = Time.now + due = CCCStoreTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { source: 'test' } + ).at(now + 2) + + store.schedule_messages([due], at: due.created_at) + + Timecop.freeze(now + 3) do + expect(store.update_schedule!).to eq(1) + end + + expect(db[:ccc_scheduled_messages].count).to eq(0) + expect(store.latest_position).to eq(1) + + cond = Sourced::CCC::QueryCondition.new( + message_type: due.type, + key_name: 'device_id', + key_value: 'dev-1' + ) + result = store.read([cond]) + msg = result.messages.first + + expect(msg).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(msg.created_at).to be >= Time.at((now + 3).to_i) + expect(msg.metadata[:source]).to eq('test') + expect(Time.parse(msg.metadata[:scheduled_at])).to be_a(Time) + end + + it 'returns false when asked to schedule no messages' do + expect(store.schedule_messages([], at: Time.now + 5)).to be false + end + end + describe '#append with guard (conditional append)' do let(:cond) do Sourced::CCC::QueryCondition.new( diff --git a/spec/sourced/ccc/supervisor_spec.rb b/spec/sourced/ccc/supervisor_spec.rb index 4e823a4f..ad190f17 100644 --- a/spec/sourced/ccc/supervisor_spec.rb +++ b/spec/sourced/ccc/supervisor_spec.rb @@ -96,10 +96,10 @@ custom_supervisor.start end - it 'spawns via executor (notifier + catchup + reaper + 2 workers = 5 spawns)' do + it 'spawns via executor (notifier + catchup + scheduler + reaper + 2 workers = 6 spawns)' do expect(executor).to receive(:start).and_yield(task) - # 1 notifier + 1 catchup_poller + 1 stale_claim_reaper + 2 workers = 5 spawns - expect(task).to receive(:spawn).exactly(5).times + # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns + expect(task).to receive(:spawn).exactly(6).times supervisor.start end From 7251e03c2c9f7ff771ba7fa0ca00e22ca241fdca Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 11 Mar 2026 18:31:19 +0000 Subject: [PATCH 034/115] Document CCC retries and backoff --- lib/sourced/ccc/README.md | 58 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 816194e6..052c5e3a 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -242,6 +242,59 @@ Sourced::CCC.configure do |c| end ``` +## Failure handling and retries + +CCC already supports consumer-group retries on failure. + +- On reactor errors, `Router#handle_next_for` calls the reactor's `on_exception` hook. +- By default, that hook uses `CCC.config.error_strategy`. +- The default `Sourced::ErrorStrategy` stops the consumer group immediately. +- If you configure a retrying error strategy, CCC stores the next retry time in the consumer group's `retry_at` column and skips claiming work for that group until that time has passed. + +So retries are built in already, but they are opt-in via the error strategy configuration. + +### Example: exponential backoff retries + +```ruby +require 'sourced/ccc' + +Sourced::CCC.configure do |c| + c.store = Sequel.sqlite('my_app.db') + + c.error_strategy = Sourced::ErrorStrategy.new do |s| + s.retry( + times: 5, + after: 2, + backoff: ->(retry_after, retry_count) { retry_after * (2**(retry_count - 1)) } + ) + + s.on_retry do |retry_count, exception, message, later| + LOGGER.warn( + "CCC retry ##{retry_count} for #{message.type} (#{message.id}) " \ + "at #{later}: #{exception.class}: #{exception.message}" + ) + end + + s.on_stop do |exception, message| + LOGGER.error( + "CCC stopping consumer group after retries for #{message.type} (#{message.id}): " \ + "#{exception.class}: #{exception.message}" + ) + end + end +end +``` + +With the configuration above, failures retry after: + +- retry 1: 2 seconds +- retry 2: 4 seconds +- retry 3: 8 seconds +- retry 4: 16 seconds +- retry 5: 32 seconds + +After the configured retries are exhausted, the consumer group is stopped. + ## Registering reactors ```ruby @@ -274,7 +327,8 @@ supervisor.start 2. **Dispatcher** routes notifications to a `WorkQueue`, mapping message types to interested reactors 3. **Workers** pop reactors from the queue, claim a partition via `Router#handle_next_for`, process messages, and ack 4. **CatchUpPoller** periodically pushes all reactors as a safety net (handles missed notifications) -5. **StaleClaimReaper** releases claims held by dead workers +5. **ScheduledMessagePoller** promotes due delayed messages into the main CCC log +6. **StaleClaimReaper** releases claims held by dead workers ### Router (direct usage) @@ -309,6 +363,8 @@ store.consumer_group_active?(CourseDecider) # => true/false store.stop_consumer_group('CourseApp::CourseDecider') ``` +When retries are configured via `CCC.config.error_strategy`, failed consumer groups remain active but paused until their `retry_at` time. Once that time passes, they become claimable again automatically. + ## Full example See `examples/ccc_app/` for a complete Sinatra application with: From dc5058c6603e56f772c0613590b6c957a7e67c47 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 00:05:06 +0000 Subject: [PATCH 035/115] New 'failed' consumer group status, and ErrorStrategy#on_fail Error strategy moves consumer group to 'failed' after failed retries, instead of 'stopped'. 'stopped' remains but it's for manual, voluntary stops. --- README.md | 10 +++--- lib/sourced.rb | 2 +- lib/sourced/backends/sequel_backend.rb | 10 +++--- .../backends/sequel_backend/group_updater.rb | 15 ++++++-- lib/sourced/backends/test_backend.rb | 7 ++-- lib/sourced/backends/test_backend/group.rb | 12 +++++-- lib/sourced/configuration.rb | 2 +- lib/sourced/consumer.rb | 4 +-- lib/sourced/error_strategy.rb | 18 +++++----- spec/consumer_spec.rb | 6 ++-- spec/error_strategy_spec.rb | 35 ++++++++++++------- spec/shared_examples/backend_examples.rb | 20 +++++++++-- 12 files changed, 94 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index e15b9c86..9a1ac9e5 100644 --- a/README.md +++ b/README.md @@ -1381,9 +1381,9 @@ Sourced workflows are eventually-consistent by default. This means that commands Most "domain errors" in command handlers should be handled by the developer and recorded as domain events, so that the domain can react and/or compensate for them. -To handle true _exceptions_ (code or data bugs, network or IO exceptions) Sourced provides a default error strategy that will "stop" the affected consumer group (the Postgres backend will log the exception and offending message in the `consumer_groups` table). +To handle true _exceptions_ (code or data bugs, network or IO exceptions) Sourced provides a default error strategy that will mark the affected consumer group as failed (the Postgres backend will log the exception in the `consumer_groups` table). -You can configure the error strategy with retries and exponential backoff, as well as `on_retry` and `on_stop` callbacks. +You can configure the error strategy with retries and exponential backoff, as well as `on_retry` and `on_fail` callbacks. ```ruby Sourced.configure do |config| @@ -1394,7 +1394,7 @@ Sourced.configure do |config| times: 3, # Wait 5 seconds before retrying after: 5, - # Custom backoff: given after=5, retries in 5, 10 and 15 seconds before stopping + # Custom backoff: given after=5, retries in 5, 10 and 15 seconds before failing backoff: ->(retry_after, retry_count) { retry_after * retry_count } ) @@ -1404,8 +1404,8 @@ Sourced.configure do |config| end # Finally, trigger this callback - # after all retries have failed and the consumer group is stopped. - s.on_stop do |exception, message| + # after all retries have failed and the consumer group is failed. + s.on_fail do |exception, message| Sentry.capture_exception(exception) end end diff --git a/lib/sourced.rb b/lib/sourced.rb index da90d7d1..79e94d7c 100644 --- a/lib/sourced.rb +++ b/lib/sourced.rb @@ -102,7 +102,7 @@ def self.config # config.backend = Sequel.connect(ENV['DATABASE_URL']) # config.error_strategy do |s| # s.retry(times: 3, after: 5) - # s.on_stop { |e, msg| Sentry.capture_exception(e) } + # s.on_fail { |e, msg| Sentry.capture_exception(e) } # end # end def self.configure(&) diff --git a/lib/sourced/backends/sequel_backend.rb b/lib/sourced/backends/sequel_backend.rb index be3ad385..3eba63b9 100644 --- a/lib/sourced/backends/sequel_backend.rb +++ b/lib/sourced/backends/sequel_backend.rb @@ -34,6 +34,8 @@ class SequelBackend ACTIVE = 'active' # Consumer group status indicating stopped processing STOPPED = 'stopped' + # Consumer group status indicating failed processing + FAILED = 'failed' # Pre-allocated return for empty/failed batch claims NO_BATCH = [nil, nil].freeze @@ -573,7 +575,7 @@ def updating_consumer_group(group_id, &) dataset.update(updates) end - # Start a consumer group that has been stopped. + # Start a consumer group that has been stopped or failed. # Signals the notifier so workers pick up the reactor immediately. # # @param group_id [String] @@ -585,11 +587,11 @@ def start_consumer_group(group_id) end # @param group_id [String] - # @param reason [#inspect, NilClass] - def stop_consumer_group(group_id, reason = nil) + # @param message [#to_s, NilClass] + def stop_consumer_group(group_id, message = nil) group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) updating_consumer_group(group_id) do |group| - group.stop(reason) + group.stop(message:) end end diff --git a/lib/sourced/backends/sequel_backend/group_updater.rb b/lib/sourced/backends/sequel_backend/group_updater.rb index d668c32d..96d993bd 100644 --- a/lib/sourced/backends/sequel_backend/group_updater.rb +++ b/lib/sourced/backends/sequel_backend/group_updater.rb @@ -14,12 +14,23 @@ def initialize(group_id, row, logger) @updates = { error_context: @error_context.dup } end - def stop(reason = nil) + def stop(message: nil) @logger.error "stopping consumer group #{group_id}" @updates[:status] = STOPPED @updates[:retry_at] = nil @updates[:updated_at] = Time.now - @updates[:error_context][:reason] = reason if reason + @updates[:error_context][:message] = message if message + end + + def fail(exception: nil) + @logger.error "failing consumer group #{group_id}" + @updates[:status] = FAILED + @updates[:retry_at] = nil + @updates[:updated_at] = Time.now + if exception + @updates[:error_context][:exception_class] = exception.class.to_s + @updates[:error_context][:exception_message] = exception.message + end end def retry(time, ctx = {}) diff --git a/lib/sourced/backends/test_backend.rb b/lib/sourced/backends/test_backend.rb index df2b26f0..331f6f6d 100644 --- a/lib/sourced/backends/test_backend.rb +++ b/lib/sourced/backends/test_backend.rb @@ -8,6 +8,7 @@ module Backends class TestBackend ACTIVE = 'active' STOPPED = 'stopped' + FAILED = 'failed' def initialize clear! @@ -98,7 +99,7 @@ def updating_consumer_group(group_id, &) end end - # Start a consumer group that has been stopped. + # Start a consumer group that has been stopped or failed. # Signals the notifier so workers pick up the reactor immediately. # # @param group_id [String] @@ -113,11 +114,11 @@ def start_consumer_group(group_id) notifier.notify_reactor_resumed(group_id) end - def stop_consumer_group(group_id, error = nil) + def stop_consumer_group(group_id, message = nil) group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) transaction do group = @state.groups[group_id] - group.stop(error) + group.stop(message:) end end diff --git a/lib/sourced/backends/test_backend/group.rb b/lib/sourced/backends/test_backend/group.rb index 80913cc9..e31eff74 100644 --- a/lib/sourced/backends/test_backend/group.rb +++ b/lib/sourced/backends/test_backend/group.rb @@ -21,11 +21,19 @@ def initialize(group_id, backend) def active? = @status == :active - def stop(reason = nil) - @error_context[:reason] = reason if reason + def stop(message: nil) + @error_context[:message] = message if message @status = :stopped end + def fail(exception: nil) + if exception + @error_context[:exception_class] = exception.class.to_s + @error_context[:exception_message] = exception.message + end + @status = :failed + end + def reset! @offsets = {} reindex diff --git a/lib/sourced/configuration.rb b/lib/sourced/configuration.rb index 328578ca..855f0428 100644 --- a/lib/sourced/configuration.rb +++ b/lib/sourced/configuration.rb @@ -169,7 +169,7 @@ def error_strategy=(strategy) # puts "Retrying #{n} times" } # end # - # s.on_stop do |exception, message| + # s.on_fail do |exception, message| # Sentry.capture_exception(exception) # end # end diff --git a/lib/sourced/consumer.rb b/lib/sourced/consumer.rb index 6b4e01d7..03798da4 100644 --- a/lib/sourced/consumer.rb +++ b/lib/sourced/consumer.rb @@ -59,13 +59,13 @@ def consumer(&) # later = 5 + 5 * retry_count # group.retry(later, retry_count: retry_count + 1) # else - # group.stop(exception) + # group.fail(exception:) # end # end # # @param exception [Exception] the exception raised # @param message [Sourced::Message] the event or command being handled - # @param group [#stop, #retry] consumer group object to update state, ie. for retries + # @param group [#stop, #fail, #retry] consumer group object to update state, ie. for retries def on_exception(exception, message, group) Sourced.config.error_strategy.call(exception, message, group) end diff --git a/lib/sourced/error_strategy.rb b/lib/sourced/error_strategy.rb index 2d1ff632..57a4fbb5 100644 --- a/lib/sourced/error_strategy.rb +++ b/lib/sourced/error_strategy.rb @@ -3,9 +3,9 @@ module Sourced # Built-in configurable error strategy # for handling exceptions raised during processing messages (commands or events) - # By default it stops the consumer group immediately. + # By default it marks the consumer group as failed immediately. # It can be configured to retry a number of times with a delay between retries. - # It can also register callbacks to be called on retry and on stop. + # It can also register callbacks to be called on retry and on failure. # # @example retry with exponential back off and callbacks # strategy = Sourced::ErrorStrategy.new do |s| @@ -15,7 +15,7 @@ module Sourced # LOGGER.info("Retrying #{n} times") # end # - # s.on_stop do |exception, message| + # s.on_fail do |exception, _message| # Sentry.capture_exception(exception) # end # end @@ -32,7 +32,7 @@ def initialize(&setup) @retry_after = RETRY_AFTER @backoff = BACKOFF @on_retry = NOOP_CALLBACK - @on_stop = NOOP_CALLBACK + @on_fail = NOOP_CALLBACK yield(self) if block_given? freeze @@ -53,15 +53,15 @@ def on_retry(callable = nil, &blk) @on_retry = callable || blk end - def on_stop(callable = nil, &blk) - @on_stop = callable || blk + def on_fail(callable = nil, &blk) + @on_fail = callable || blk end # The Error Strategy interface # # @param exception [Exception] # @param message [Sourced::Message] - # @param group [#retry, #stop] + # @param group [#retry, #fail] def call(exception, message, group) retry_count = group.error_context[:retry_count] || 1 if retry_count <= max_retries @@ -71,8 +71,8 @@ def call(exception, message, group) retry_count += 1 group.retry(later, retry_count:) else - @on_stop.call(exception, message) - group.stop(exception:, message:) + @on_fail.call(exception, message) + group.fail(exception:) end end diff --git a/spec/consumer_spec.rb b/spec/consumer_spec.rb index d90d34ff..502818bc 100644 --- a/spec/consumer_spec.rb +++ b/spec/consumer_spec.rb @@ -63,12 +63,12 @@ class TestConsumer end describe '.on_exception' do - it 'stops the consumer group by default' do - group = double('group', error_context: {}, stop: true) + it 'fails the consumer group by default' do + group = double('group', error_context: {}, fail: true) exception = StandardError.new('test error') message = { id: 1 } TestConsumer::TestConsumer.on_exception(exception, message, group) - expect(group).to have_received(:stop).with(exception:, message:) + expect(group).to have_received(:fail).with(exception:) end end end diff --git a/spec/error_strategy_spec.rb b/spec/error_strategy_spec.rb index 59d64362..58968c70 100644 --- a/spec/error_strategy_spec.rb +++ b/spec/error_strategy_spec.rb @@ -11,9 +11,18 @@ def retry(later, ctx = {}) self end - def stop(reason = {}) + def stop(message: nil) self.status = :stopped - self.error_context.merge!(reason:) + self.error_context[:message] = message if message + self + end + + def fail(exception: nil) + self.status = :failed + if exception + self.error_context[:exception_class] = exception.class.to_s + self.error_context[:exception_message] = exception.message + end self end end @@ -25,19 +34,20 @@ def stop(reason = {}) before do allow(group).to receive(:retry).and_call_original allow(group).to receive(:stop).and_call_original + allow(group).to receive(:fail).and_call_original end - it 'stops the group immediatly by default' do + it 'fails the group immediatly by default' do strategy = described_class.new strategy.call(exception, message, group) - expect(group).to have_received(:stop).with(exception:, message:) + expect(group).to have_received(:fail).with(exception:) end it 'can be configured with retries' do now = Time.new(2020, 1, 1).utc retries = [] - stop_call = nil + fail_call = nil strategy = described_class.new do |s| s.retry(times: 3, after: 5, backoff: ->(retry_after, retry_count) { retry_after * retry_count }) @@ -46,8 +56,8 @@ def stop(reason = {}) retries << [n, exception, message, later] end - s.on_stop do |exception, message| - stop_call = [exception, message] + s.on_fail do |exception, message| + fail_call = [exception, message] end end @@ -56,7 +66,7 @@ def stop(reason = {}) strategy.call(exception, message, group) strategy.call(exception, message, group) - expect(stop_call).to be(nil) + expect(fail_call).to be(nil) strategy.call(exception, message, group) @@ -66,12 +76,11 @@ def stop(reason = {}) [3, exception, message, now + 15] ]) - expect(stop_call).to eq([exception, message]) + expect(fail_call).to eq([exception, message]) - expect(group).to have_received(:retry).with(now + 5, retry_count: 2).exactly(1).times - expect(group).to have_received(:retry).with(now + 10, retry_count: 3).exactly(1).times - expect(group).to have_received(:retry).with(now + 15, retry_count: 4).exactly(1).times - expect(group).to have_received(:stop).exactly(1).times + expect(group.retry_at).to eq(now + 15) + expect(group.status).to eq(:failed) + expect(group.error_context[:retry_count]).to eq(4) end end end diff --git a/spec/shared_examples/backend_examples.rb b/spec/shared_examples/backend_examples.rb index ea18ba40..96e33517 100644 --- a/spec/shared_examples/backend_examples.rb +++ b/spec/shared_examples/backend_examples.rb @@ -1258,10 +1258,10 @@ def self.handled_messages expect(counts).to eq([nil, 1]) end - specify '#stop(error)' do + specify '#stop(message:)' do backend.register_consumer_group('group1') backend.updating_consumer_group('group1') do |group| - group.stop(StandardError.new('boom')) + group.stop(message: 'operator requested shutdown') end gr = backend.stats.groups.first @@ -1273,6 +1273,22 @@ def self.handled_messages expect(gr[:group_id]).to eq('group1') expect(gr[:status]).to eq('active') end + + specify '#fail(exception:)' do + backend.register_consumer_group('group1') + backend.updating_consumer_group('group1') do |group| + group.fail(exception: StandardError.new('boom')) + end + + gr = backend.stats.groups.first + expect(gr[:group_id]).to eq('group1') + expect(gr[:status]).to eq('failed') + + backend.start_consumer_group('group1') + gr = backend.stats.groups.first + expect(gr[:group_id]).to eq('group1') + expect(gr[:status]).to eq('active') + end end describe '#notifier' do From 189b81a2632260a579ebc006f528a016bea4eec2 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 00:05:55 +0000 Subject: [PATCH 036/115] fail status for CCC consumer groups --- lib/sourced/ccc/README.md | 8 ++++---- lib/sourced/ccc/consumer.rb | 16 ++++++++++++++-- lib/sourced/ccc/store.rb | 12 ++++++++---- spec/sourced/ccc/router_spec.rb | 6 ++++-- spec/sourced/ccc/store_spec.rb | 22 +++++++++++++++++++--- 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 052c5e3a..86aebe3a 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -248,7 +248,7 @@ CCC already supports consumer-group retries on failure. - On reactor errors, `Router#handle_next_for` calls the reactor's `on_exception` hook. - By default, that hook uses `CCC.config.error_strategy`. -- The default `Sourced::ErrorStrategy` stops the consumer group immediately. +- The default `Sourced::ErrorStrategy` marks the consumer group as failed immediately. - If you configure a retrying error strategy, CCC stores the next retry time in the consumer group's `retry_at` column and skips claiming work for that group until that time has passed. So retries are built in already, but they are opt-in via the error strategy configuration. @@ -275,9 +275,9 @@ Sourced::CCC.configure do |c| ) end - s.on_stop do |exception, message| + s.on_fail do |exception, message| LOGGER.error( - "CCC stopping consumer group after retries for #{message.type} (#{message.id}): " \ + "CCC failing consumer group after retries for #{message.type} (#{message.id}): " \ "#{exception.class}: #{exception.message}" ) end @@ -293,7 +293,7 @@ With the configuration above, failures retry after: - retry 4: 16 seconds - retry 5: 32 seconds -After the configured retries are exhausted, the consumer group is stopped. +After the configured retries are exhausted, the consumer group is marked as failed. ## Registering reactors diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index 6b806ad2..7ccc4b19 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -15,11 +15,23 @@ def initialize(group_id, row, logger) @updates = { error_context: @error_context.dup } end - def stop(exception:, message:) - @logger.error "CCC: stopping consumer group #{group_id} message: '#{message&.type}' (#{message&.id}). #{exception&.class}: #{exception&.message}" + def stop(message: nil) + @logger.error "CCC: stopping consumer group #{group_id}" @updates[:status] = Store::STOPPED @updates[:retry_at] = nil @updates[:updated_at] = Time.now.iso8601 + @updates[:error_context][:message] = message if message + end + + def fail(exception: nil) + @logger.error "CCC: failing consumer group #{group_id}. #{exception&.class}: #{exception&.message}" + @updates[:status] = Store::FAILED + @updates[:retry_at] = nil + @updates[:updated_at] = Time.now.iso8601 + if exception + @updates[:error_context][:exception_class] = exception.class.to_s + @updates[:error_context][:exception_message] = exception.message + end end def retry(time, **ctx) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 31d18d2d..a30f6e98 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -39,6 +39,7 @@ def to_ary = [messages, guard] class Store ACTIVE = 'active' STOPPED = 'stopped' + FAILED = 'failed' # @return [Sequel::SQLite::Database] attr_reader :db @@ -373,16 +374,19 @@ def consumer_group_active?(group_id) row[:status] == ACTIVE end - # Stop a consumer group. Stopped groups are skipped by {#claim_next}. + # Stop a consumer group intentionally. Stopped groups are skipped by {#claim_next}. # # @param group_id [String, #group_id] identifier or object responding to +#group_id+ + # @param message [String, nil] optional operator-supplied reason # @return [void] - def stop_consumer_group(group_id) + def stop_consumer_group(group_id, message = nil) group_id = resolve_group_id(group_id) - db[:ccc_consumer_groups].where(group_id: group_id).update(status: STOPPED, updated_at: Time.now.iso8601) + updating_consumer_group(group_id) do |group| + group.stop(message:) + end end - # Re-activate a stopped consumer group, clearing retry state. + # Re-activate a stopped or failed consumer group, clearing retry state. # # @param group_id [String, #group_id] identifier or object responding to +#group_id+ # @return [void] diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 9dbcee2e..203dfde1 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -221,7 +221,7 @@ def self.handle_batch(claim) expect(RouterTestDecider).to have_received(:on_exception) end - it 'on_exception stops consumer group when default strategy' do + it 'on_exception fails consumer group when default strategy' do store.append( CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) ) @@ -233,6 +233,8 @@ def self.handle_batch(claim) router.handle_next_for(RouterTestDecider) expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false + row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:status]).to eq('failed') end it 'on_exception persists error_context in the database' do @@ -249,7 +251,7 @@ def self.handle_batch(claim) row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first expect(row[:error_context]).not_to be_nil - expect(row[:status]).to eq('stopped') + expect(row[:status]).to eq('failed') end it 'on_exception with retry strategy sets retry_at on consumer group' do diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 14d7461b..5ccd7d08 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -669,16 +669,32 @@ module CCCStoreTestMessages group.retry(Time.now + 30, retry_count: 1) end - err = RuntimeError.new('test error') - msg = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) store.updating_consumer_group('my-group') do |group| - group.stop(exception: err, message: msg) + group.stop(message: 'operator requested shutdown') end row = db[:ccc_consumer_groups].where(group_id: 'my-group').first expect(row[:status]).to eq('stopped') expect(row[:retry_at]).to be_nil end + + it 'fail sets status to FAILED and clears retry_at' do + store.updating_consumer_group('my-group') do |group| + group.retry(Time.now + 30, retry_count: 1) + end + + err = RuntimeError.new('test error') + store.updating_consumer_group('my-group') do |group| + group.fail(exception: err) + end + + row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + expect(row[:status]).to eq('failed') + expect(row[:retry_at]).to be_nil + ctx = JSON.parse(row[:error_context], symbolize_names: true) + expect(ctx[:exception_class]).to eq('RuntimeError') + expect(ctx[:exception_message]).to eq('test error') + end end describe '#reset_consumer_group' do From c6c5be251fa009ebeb2cbeb73bf23f490c490bb4 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 10:22:47 +0000 Subject: [PATCH 037/115] Add CCC::Store#stats for monitoring and debugging Returns max_position and per-consumer-group diagnostics (status, retry_at, oldest/newest processed positions, partition count). Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 27 +++++++++++++ spec/sourced/ccc/store_spec.rb | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index a30f6e98..930f8734 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -32,6 +32,8 @@ def instance_of?(klass) = __getobj__.instance_of?(klass) def to_ary = [messages, guard] end + Stats = Data.define(:max_position, :groups) + # SQLite-backed store for CCC's flat, globally-ordered message log. # Provides message storage with automatic key-pair indexing, # consumer group management, and partition-based offset tracking @@ -627,6 +629,31 @@ def advance_offset(group_id, partition:, position:) end end + # System-wide diagnostics for monitoring and debugging. + # + # @return [CCC::Stats] max_position and per-group processing state + def stats + groups = db.fetch(<<~SQL).all + SELECT + cg.group_id, + cg.status, + cg.retry_at, + COALESCE(MIN(CASE WHEN o.last_position > 0 THEN o.last_position END), 0) AS oldest_processed, + COALESCE(MAX(o.last_position), 0) AS newest_processed, + COUNT(o.id) AS partition_count + FROM ccc_consumer_groups cg + LEFT JOIN ccc_offsets o ON o.consumer_group_id = cg.id + GROUP BY cg.id, cg.group_id, cg.status, cg.retry_at + ORDER BY cg.group_id + SQL + + groups.each do |g| + g[:retry_at] = Time.parse(g[:retry_at]) if g[:retry_at] + end + + Stats.new(max_position: latest_position, groups: groups) + end + # Current max position in the message log. # # @return [Integer] max position, or 0 if the store is empty diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 5ccd7d08..3c89bfa7 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1392,6 +1392,80 @@ module CCCStoreTestMessages end end + describe '#stats' do + it 'returns zeroes for an empty store' do + result = store.stats + expect(result).to be_a(Sourced::CCC::Stats) + expect(result.max_position).to eq(0) + expect(result.groups).to eq([]) + end + + it 'returns groups with zeroed stats when no messages processed' do + store.register_consumer_group('group-a') + store.register_consumer_group('group-b') + + result = store.stats + expect(result.max_position).to eq(0) + expect(result.groups.size).to eq(2) + + group_a = result.groups.find { |g| g[:group_id] == 'group-a' } + expect(group_a[:status]).to eq('active') + expect(group_a[:retry_at]).to be_nil + expect(group_a[:oldest_processed]).to eq(0) + expect(group_a[:newest_processed]).to eq(0) + expect(group_a[:partition_count]).to eq(0) + end + + it 'reflects processing state after claim and ack' do + group_id = 'stats-test' + store.register_consumer_group(group_id) + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + + # Claim and ack first partition + r1 = store.claim_next(group_id, partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) + + result = store.stats + expect(result.max_position).to eq(2) + + group = result.groups.first + expect(group[:group_id]).to eq(group_id) + expect(group[:partition_count]).to eq(2) # both partitions bootstrapped + expect(group[:oldest_processed]).to be > 0 + expect(group[:newest_processed]).to be > 0 + end + + it 'reports stopped and failed group statuses' do + store.register_consumer_group('active-group') + store.register_consumer_group('stopped-group') + store.register_consumer_group('failed-group') + + store.stop_consumer_group('stopped-group') + store.updating_consumer_group('failed-group') { |g| g.fail(exception: RuntimeError.new('boom')) } + + result = store.stats + statuses = result.groups.map { |g| [g[:group_id], g[:status]] }.to_h + + expect(statuses['active-group']).to eq('active') + expect(statuses['stopped-group']).to eq('stopped') + expect(statuses['failed-group']).to eq('failed') + end + + it 'groups are ordered by group_id' do + store.register_consumer_group('zebra') + store.register_consumer_group('alpha') + store.register_consumer_group('middle') + + result = store.stats + expect(result.groups.map { |g| g[:group_id] }).to eq(%w[alpha middle zebra]) + end + end + describe '#release' do let(:group_id) { 'release-test' } From d523210fc1660b61268523eca9d354e91a990866 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 10:31:13 +0000 Subject: [PATCH 038/115] Include error_context in CCC::Store#stats and document stats API Add error_context (exception class/message, stop reason) to per-group stats output. Document the Monitoring section in the CCC README with field reference tables and usage examples. Add YARD example to #stats. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/README.md | 44 ++++++++++++++++++++++++++++++++++ lib/sourced/ccc/store.rb | 29 +++++++++++++++++++++- spec/sourced/ccc/store_spec.rb | 18 +++++++++----- 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 86aebe3a..4ed502de 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -365,6 +365,50 @@ store.stop_consumer_group('CourseApp::CourseDecider') When retries are configured via `CCC.config.error_strategy`, failed consumer groups remain active but paused until their `retry_at` time. Once that time passes, they become claimable again automatically. +## Monitoring + +`Store#stats` returns system-wide diagnostics for monitoring and debugging CCC deployments. + +```ruby +stats = store.stats +stats.max_position # => 42 (latest position in the message log) +stats.groups # => array of per-consumer-group hashes +``` + +Each group hash contains: + +| Key | Description | +|----------------------|----------------------------------------------------------------| +| `group_id` | Consumer group identifier (e.g. `"CourseDecider"`) | +| `status` | `"active"`, `"stopped"`, or `"failed"` | +| `retry_at` | `Time` of next retry, or `nil` | +| `error_context` | Hash with error details (`{}` when healthy, see below) | +| `oldest_processed` | `MIN(last_position)` across partitions where processing started | +| `newest_processed` | `MAX(last_position)` across partitions | +| `partition_count` | Number of offset rows (partitions) for this group | + +### `error_context` + +The `error_context` hash is empty (`{}`) for healthy groups. When a group is stopped or has failed, it may contain: + +| Key | Present when | Description | +|----------------------|--------------|--------------------------------------| +| `:message` | Stopped | Operator-supplied reason for stopping | +| `:exception_class` | Failed | Exception class name (e.g. `"RuntimeError"`) | +| `:exception_message` | Failed | Exception message string | + +When retries are configured, `error_context` also accumulates retry state set by `GroupUpdater#retry_later`. + +```ruby +stats = store.stats +stats.groups.each do |g| + puts "#{g[:group_id]}: #{g[:status]} (#{g[:partition_count]} partitions, up to position #{g[:newest_processed]})" + if g[:status] == 'failed' + puts " error: #{g[:error_context][:exception_class]}: #{g[:error_context][:exception_message]}" + end +end +``` + ## Full example See `examples/ccc_app/` for a complete Sinatra application with: diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 930f8734..14c55d29 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -631,6 +631,31 @@ def advance_offset(group_id, partition:, position:) # System-wide diagnostics for monitoring and debugging. # + # @example + # stats = store.stats + # stats.max_position # => 42 + # stats.groups + # # => [ + # # { + # # group_id: "my_decider", + # # status: "active", + # # retry_at: nil, + # # error_context: {}, + # # oldest_processed: 10, + # # newest_processed: 42, + # # partition_count: 3 + # # }, + # # { + # # group_id: "failing_decider", + # # status: "failed", + # # retry_at: nil, + # # error_context: { exception_class: "RuntimeError", exception_message: "boom" }, + # # oldest_processed: 5, + # # newest_processed: 30, + # # partition_count: 2 + # # } + # # ] + # # @return [CCC::Stats] max_position and per-group processing state def stats groups = db.fetch(<<~SQL).all @@ -638,17 +663,19 @@ def stats cg.group_id, cg.status, cg.retry_at, + cg.error_context, COALESCE(MIN(CASE WHEN o.last_position > 0 THEN o.last_position END), 0) AS oldest_processed, COALESCE(MAX(o.last_position), 0) AS newest_processed, COUNT(o.id) AS partition_count FROM ccc_consumer_groups cg LEFT JOIN ccc_offsets o ON o.consumer_group_id = cg.id - GROUP BY cg.id, cg.group_id, cg.status, cg.retry_at + GROUP BY cg.id, cg.group_id, cg.status, cg.retry_at, cg.error_context ORDER BY cg.group_id SQL groups.each do |g| g[:retry_at] = Time.parse(g[:retry_at]) if g[:retry_at] + g[:error_context] = g[:error_context] ? JSON.parse(g[:error_context], symbolize_names: true) : {} end Stats.new(max_position: latest_position, groups: groups) diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 3c89bfa7..7a91ffec 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1440,20 +1440,26 @@ module CCCStoreTestMessages expect(group[:newest_processed]).to be > 0 end - it 'reports stopped and failed group statuses' do + it 'reports stopped and failed group statuses with error context' do store.register_consumer_group('active-group') store.register_consumer_group('stopped-group') store.register_consumer_group('failed-group') - store.stop_consumer_group('stopped-group') + store.stop_consumer_group('stopped-group', 'maintenance') store.updating_consumer_group('failed-group') { |g| g.fail(exception: RuntimeError.new('boom')) } result = store.stats - statuses = result.groups.map { |g| [g[:group_id], g[:status]] }.to_h + by_id = result.groups.map { |g| [g[:group_id], g] }.to_h - expect(statuses['active-group']).to eq('active') - expect(statuses['stopped-group']).to eq('stopped') - expect(statuses['failed-group']).to eq('failed') + expect(by_id['active-group'][:status]).to eq('active') + expect(by_id['active-group'][:error_context]).to eq({}) + + expect(by_id['stopped-group'][:status]).to eq('stopped') + expect(by_id['stopped-group'][:error_context][:message]).to eq('maintenance') + + expect(by_id['failed-group'][:status]).to eq('failed') + expect(by_id['failed-group'][:error_context][:exception_class]).to eq('RuntimeError') + expect(by_id['failed-group'][:error_context][:exception_message]).to eq('boom') end it 'groups are ordered by group_id' do From 949bb513f74b781066f3ff0fa79e8f88c2b56719 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 11:05:30 +0000 Subject: [PATCH 039/115] Add CCC::Store#read_correlation_batch for tracing causal chains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Looks up all messages sharing the same correlation_id as a given message, enabling command → event → reaction chain inspection. Adds a correlation_id index for efficient lookups. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 18 +++++++++++++ spec/sourced/ccc/store_spec.rb | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 14c55d29..6b4f39a4 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -93,6 +93,7 @@ def install! ) SQL db.run('CREATE INDEX IF NOT EXISTS idx_ccc_message_type ON ccc_messages(message_type)') + db.run('CREATE INDEX IF NOT EXISTS idx_ccc_correlation_id ON ccc_messages(correlation_id)') db.run(<<~SQL) CREATE TABLE IF NOT EXISTS ccc_key_pairs ( @@ -681,6 +682,23 @@ def stats Stats.new(max_position: latest_position, groups: groups) end + # Fetch all messages sharing the same correlation_id as the given message. + # Useful for tracing causal chains (command -> events -> reactions). + # + # @param message_id [String] UUID of any message in the correlation chain + # @return [Array] correlated messages ordered by position, or [] if not found + def read_correlation_batch(message_id) + correlation_id = db[:ccc_messages] + .where(message_id: message_id) + .get(:correlation_id) + return [] unless correlation_id + + db[:ccc_messages] + .where(correlation_id: correlation_id) + .order(:position) + .map { |row| deserialize(row) } + end + # Current max position in the message log. # # @return [Integer] max position, or 0 if the store is empty diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 7a91ffec..3d6a9c9a 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1472,6 +1472,55 @@ module CCCStoreTestMessages end end + describe '#read_correlation_batch' do + it 'returns all messages sharing the same correlation_id, ordered by position' do + # Create a command (source of the correlation chain) + cmd = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor' }) + + # Create correlated events + evt1 = cmd.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) + evt2 = cmd.correlate(CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'Asset A' })) + + store.append([cmd, evt1, evt2]) + + results = store.read_correlation_batch(cmd.id) + expect(results.size).to eq(3) + expect(results.map(&:id)).to eq([cmd.id, evt1.id, evt2.id]) + expect(results.map(&:position)).to eq(results.map(&:position).sort) + end + + it 'returns [] for an unknown message_id' do + expect(store.read_correlation_batch(SecureRandom.uuid)).to eq([]) + end + + it 'excludes messages with a different correlation_id' do + cmd1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + evt1 = cmd1.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) + + # Unrelated chain + cmd2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + evt2 = cmd2.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-2', asset_id: 'a-2' })) + + store.append([cmd1, evt1, cmd2, evt2]) + + results = store.read_correlation_batch(cmd1.id) + expect(results.size).to eq(2) + expect(results.map(&:id)).to eq([cmd1.id, evt1.id]) + end + + it 'can be queried from any message in the chain' do + cmd = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + evt = cmd.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) + + store.append([cmd, evt]) + + # Query from the event, not the command + results = store.read_correlation_batch(evt.id) + expect(results.size).to eq(2) + expect(results.map(&:id)).to eq([cmd.id, evt.id]) + end + end + describe '#release' do let(:group_id) { 'release-test' } From 5c21da5c4de4191381f02880e48806ba4cc6095f Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 11:08:40 +0000 Subject: [PATCH 040/115] Add YARD examples to CCC::Store#read_partition Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 6b4f39a4..17d31203 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -307,6 +307,40 @@ def read(conditions, from_position: nil, limit: nil) # matches the given value. Messages that don't declare a partition # attribute pass through (same logic as {#claim_next}). # + # @example Single partition attribute + # result = store.read_partition( + # { device_id: 'dev-1' }, + # handled_types: ['device.registered', 'device.bound'] + # ) + # result.messages # => [#, ...] + # result.guard # => # + # + # @example Composite partition (AND semantics — messages must match all attributes they declare) + # result = store.read_partition( + # { course_name: 'Algebra', user_id: 'joe' }, + # handled_types: ['course.created', 'user.joined_course'] + # ) + # # Returns CourseCreated(course_name: 'Algebra') — matches on its only attribute + # # Returns UserJoinedCourse(course_name: 'Algebra', user_id: 'joe') — matches both + # # Excludes UserJoinedCourse(course_name: 'Algebra', user_id: 'jane') — user_id mismatch + # + # @example Resuming from a position (e.g. after processing a batch) + # result = store.read_partition( + # { device_id: 'dev-1' }, + # handled_types: ['device.registered'], + # from_position: 42 + # ) + # # Only returns messages with position > 42 + # + # @example Using the guard for optimistic concurrency on append + # result = store.read_partition( + # { device_id: 'dev-1' }, + # handled_types: ['device.registered', 'device.bound'] + # ) + # # ... build new events from result.messages ... + # store.append(new_events, guard: result.guard) + # # Raises Sourced::ConcurrentAppendError if conflicting writes occurred + # # @param partition_attrs [Hash{Symbol|String => String}] partition attribute values # @param handled_types [Array] message type strings to include # @param from_position [Integer] fetch messages after this position (default 0) From d6dc34ef5c580d7cdf163e4e5fd18d3ec9d0526e Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 11:18:14 +0000 Subject: [PATCH 041/115] Add CCC::Store#read_all for paginating the global event log Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/README.md | 14 ++++++++++++ lib/sourced/ccc/store.rb | 19 ++++++++++++++++ spec/sourced/ccc/store_spec.rb | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 4ed502de..c1782cd8 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -92,6 +92,20 @@ result = store.read_partition( ) ``` +### Browsing the global log + +`read_all` paginates the entire message log in position order, without requiring query conditions or partition attributes. + +```ruby +# First page (default limit: 50) +messages = store.read_all(limit: 20) + +# Next page — pass the last position from the previous page +messages = store.read_all(from_position: messages.last.position, limit: 20) +``` + +Returns an array of `PositionedMessage` instances ordered by position, or `[]` if the store is empty or there are no more messages after `from_position`. + ## Deciders Deciders handle commands, enforce invariants, and produce events. They rebuild state from event history before each decision. diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 17d31203..a908ebfb 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -282,6 +282,25 @@ def update_schedule! end end + # Paginate the global event log in position order. + # + # @example First page + # messages = store.read_all(limit: 20) + # + # @example Next page (using the last position from the previous page) + # messages = store.read_all(from_position: 20, limit: 20) + # + # @param from_position [Integer] return messages after this position (default 0) + # @param limit [Integer] max number of messages to return (default 50) + # @return [Array] messages ordered by position + def read_all(from_position: 0, limit: 50) + db[:ccc_messages] + .where { position > from_position } + .order(:position) + .limit(limit) + .map { |row| deserialize(row) } + end + # Query messages by conditions. Each condition matches on # (message_type AND key_name AND key_value). Multiple conditions are OR'd. # diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 3d6a9c9a..bf3bfa04 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -289,6 +289,47 @@ module CCCStoreTestMessages end end + describe '#read_all' do + it 'returns messages in position order' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' }) + ]) + + messages = store.read_all + expect(messages.size).to eq(3) + expect(messages.map(&:position)).to eq([1, 2, 3]) + end + + it 'paginates with from_position and limit' do + 5.times do |i| + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) + end + + page1 = store.read_all(limit: 2) + expect(page1.map(&:position)).to eq([1, 2]) + + page2 = store.read_all(from_position: page1.last.position, limit: 2) + expect(page2.map(&:position)).to eq([3, 4]) + + page3 = store.read_all(from_position: page2.last.position, limit: 2) + expect(page3.map(&:position)).to eq([5]) + end + + it 'returns [] for an empty store' do + expect(store.read_all).to eq([]) + end + + it 'returns PositionedMessage instances' do + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + messages = store.read_all + expect(messages.first).to be_a(Sourced::CCC::PositionedMessage) + expect(messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) + end + end + describe '#read' do before do store.append([ From 52450ef8e435abf53e8fd3a0b36c41bfffa440c2 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 14:54:39 +0000 Subject: [PATCH 042/115] CCC::Topology --- lib/sourced/ccc/topology.rb | 437 ++++++++++++++++++++++++++++++ spec/sourced/ccc/topology_spec.rb | 432 +++++++++++++++++++++++++++++ 2 files changed, 869 insertions(+) create mode 100644 lib/sourced/ccc/topology.rb create mode 100644 spec/sourced/ccc/topology_spec.rb diff --git a/lib/sourced/ccc/topology.rb b/lib/sourced/ccc/topology.rb new file mode 100644 index 00000000..ee7fab63 --- /dev/null +++ b/lib/sourced/ccc/topology.rb @@ -0,0 +1,437 @@ +# frozen_string_literal: true + +require 'sourced/topology' + +module Sourced + module CCC + # Analyzes registered CCC reactors (Deciders and Projectors) and builds a + # flat array of node structs describing the message flow graph. This enables + # visualization, introspection, and Event Modeling diagram generation for + # CCC-based systems. + # + # Works like {Sourced::Topology} but adapted for CCC's stream-less messages, + # CCC-specific handler prefixes (+ccc_decide+, +ccc_reaction+), and + # {CCC::Message} registry. + # + # @example Build topology from registered reactors + # nodes = Sourced::CCC::Topology.build([MyDecider, MyProjector]) + # nodes.each do |node| + # puts "#{node.type}: #{node.name} (#{node.id})" + # end + # + # @example Access topology via the CCC module (uses global router) + # Sourced::CCC.register(MyDecider) + # Sourced::CCC.register(MyProjector) + # Sourced::CCC.topology.each { |node| puts node.id } + # + # @example Filter by node type + # commands = Sourced::CCC::Topology.build(reactors).select { |n| n.type == 'command' } + # commands.each { |cmd| puts "#{cmd.name} => #{cmd.produces.inspect}" } + # + # @see Sourced::Topology + module Topology + # @!parse + # # A command node in the topology graph. + # # @attr type [String] always +'command'+ + # # @attr id [String] message type string (e.g. +'orders.create_order'+) + # # @attr name [String] Ruby class name (e.g. +'Orders::CreateOrder'+) + # # @attr group_id [String] reactor group that handles this command + # # @attr produces [Array] event type strings produced by this command handler + # # @attr schema [Hash] JSON Schema of the command payload, or +{}+ + # CommandNode = Struct.new(:type, :id, :name, :group_id, :produces, :schema) + # + # # An event node in the topology graph. + # # @attr type [String] always +'event'+ + # # @attr id [String] message type string + # # @attr name [String] Ruby class name + # # @attr group_id [String] reactor group where this event was first seen + # # @attr produces [Array] always +[]+ + # # @attr schema [Hash] JSON Schema of the event payload, or +{}+ + # EventNode = Struct.new(:type, :id, :name, :group_id, :produces, :schema) + # + # # An automation (reaction) node in the topology graph. + # # @attr type [String] always +'automation'+ + # # @attr id [String] composite ID (e.g. +'evt.type-GroupId-aut'+) + # # @attr name [String] human-readable label (e.g. +'reaction(WidgetCreated)'+) + # # @attr group_id [String] reactor group that owns this reaction + # # @attr consumes [Array] event type strings or readmodel IDs consumed + # # @attr produces [Array] command type strings dispatched + # AutomationNode = Struct.new(:type, :id, :name, :group_id, :consumes, :produces) + # + # # A read model node in the topology graph (projectors only). + # # @attr type [String] always +'readmodel'+ + # # @attr id [String] composite ID (e.g. +'my_app.widget_projector-rm'+) + # # @attr name [String] projector group_id + # # @attr group_id [String] projector group_id + # # @attr consumes [Array] event type strings that feed this read model + # # @attr produces [Array] automation node IDs triggered by this read model + # # @attr schema [Hash] always +{}+ + # ReadModelNode = Struct.new(:type, :id, :name, :group_id, :consumes, :produces, :schema) + + CommandNode = Sourced::Topology::CommandNode + EventNode = Sourced::Topology::EventNode + AutomationNode = Sourced::Topology::AutomationNode + ReadModelNode = Sourced::Topology::ReadModelNode + + # Prism-based source analyzer adapted for CCC handler prefixes. + # + # Overrides {Sourced::Topology::SourceAnalyzer#events_produced_by} and + # {Sourced::Topology::SourceAnalyzer#commands_dispatched_by} to use the + # +ccc_decide+ and +ccc_reaction+ method name prefixes instead of the + # main Sourced +decide+ and +reaction+ prefixes. + # + # @example + # analyzer = CCC::Topology::Analyzer.new + # refs = analyzer.events_produced_by(MyDecider, CreateWidget) + # # => [[:const, "WidgetCreated"]] + # + # @see Sourced::Topology::SourceAnalyzer + class Analyzer < Sourced::Topology::SourceAnalyzer + # Extract event references from a CCC command handler block. + # + # Looks up the handler method generated by +Decider.command(CmdClass)+ + # and uses Prism AST analysis to find +event(...)+ calls within it. + # + # @param reactor [Class] a {CCC::Decider} subclass + # @param cmd_class [Class] a {CCC::Command} subclass handled by the reactor + # @return [Array] AST references, e.g. +[[:const, "WidgetCreated"]]+ + # + # @example + # analyzer.events_produced_by(WidgetDecider, CreateWidget) + # # => [[:const, "WidgetCreated"]] + def events_produced_by(reactor, cmd_class) + return [] unless @prism_available + + method_name = Sourced.message_method_name('ccc_decide', cmd_class.name) + extract_calls_from_handler(reactor, method_name, :event) + end + + # Extract dispatch references from a CCC reaction handler block. + # + # Looks up the handler method generated by +reaction(EvtClass)+ + # and uses Prism AST analysis to find +dispatch(...)+ calls within it. + # Follows chained calls like +dispatch(Cmd).at(time)+. + # + # @param reactor [Class] a CCC reactor class (Decider or Projector subclass) + # @param evt_class [Class] the event class whose reaction handler to analyze + # @return [Array] AST references, e.g. +[[:const, "NotifyWidget"]]+ + # + # @example + # analyzer.commands_dispatched_by(WidgetDecider, WidgetCreated) + # # => [[:const, "NotifyWidget"]] + def commands_dispatched_by(reactor, evt_class) + return [] unless @prism_available + + method_name = Sourced.message_method_name(CCC::React::PREFIX, evt_class.name) + extract_calls_from_handler(reactor, method_name, :dispatch) + end + end + + # Analyze registered CCC reactors and build the topology graph. + # + # Iterates each reactor class and extracts: + # - {CommandNode}s from Decider +handled_commands+, with Prism-extracted +produces+ + # - {EventNode}s from command produces and evolve handlers, deduplicated by type string + # - {AutomationNode}s from specific and catch-all reactions, with Prism-extracted +produces+ + # - {ReadModelNode}s for Projector subclasses, linking consumed events to automation outputs + # + # Nodes are deduplicated across reactors: the first reactor to declare a command or + # event "owns" it (gets its +group_id+). + # + # @param reactors [Enumerable] reactor classes ({CCC::Decider} and/or + # {CCC::Projector} subclasses) + # @return [Array] + # flat array of topology nodes + # + # @example Build from explicit reactor list + # nodes = CCC::Topology.build([OrderDecider, OrderProjector]) + # + # nodes.select { |n| n.type == 'command' }.each do |cmd| + # puts "#{cmd.name} produces: #{cmd.produces}" + # end + # # OrderDecider::PlaceOrder produces: ["orders.order_placed"] + # + # @example Inspect automation (reaction) wiring + # nodes = CCC::Topology.build([OrderDecider, NotificationDecider]) + # + # nodes.select { |n| n.type == 'automation' }.each do |aut| + # puts "#{aut.name}: consumes #{aut.consumes} => dispatches #{aut.produces}" + # end + # # reaction(OrderPlaced): consumes ["orders.order_placed"] => dispatches ["notifications.send_receipt"] + # + # @example Projector readmodel wiring + # nodes = CCC::Topology.build([OrderSummaryProjector]) + # + # rm = nodes.find { |n| n.type == 'readmodel' } + # puts "#{rm.name} consumes #{rm.consumes}, triggers #{rm.produces}" + # # OrderSummaryProjector consumes ["orders.order_placed"], triggers ["orders.order_placed-OrderSummaryProjector-aut"] + def self.build(reactors) + analyzer = Analyzer.new + nodes = [] + command_ids = {} + event_nodes = {} + + reactors.each do |reactor| + group_id = reactor.group_id + + # Command nodes (deciders only) + if reactor.respond_to?(:handled_commands) + reactor.handled_commands.each do |cmd_class| + next if command_ids.key?(cmd_class.type) + + produced_refs = analyzer.events_produced_by(reactor, cmd_class) + produced_types = resolve_refs(produced_refs, reactor) + + schema = extract_schema(cmd_class) + cmd_node = CommandNode.new( + type: 'command', + id: cmd_class.type, + name: cmd_class.name, + group_id: group_id, + produces: produced_types, + schema: schema + ) + nodes << cmd_node + command_ids[cmd_class.type] = cmd_node + + # Register event nodes discovered via Prism from this command's handler + produced_types.each do |evt_type| + next if event_nodes.key?(evt_type) + next if command_ids.key?(evt_type) + + evt_class = find_event_class(evt_type) + next unless evt_class + + event_nodes[evt_type] = EventNode.new( + type: 'event', + id: evt_type, + name: evt_class.name, + group_id: group_id, + produces: [], + schema: extract_schema(evt_class) + ) + end + end + end + + # Event nodes from evolve handlers (covers projectors and events not yet seen) + if reactor.respond_to?(:handled_messages_for_evolve) + reactor.handled_messages_for_evolve.each do |evt_class| + # Skip command classes that ended up in evolve handlers + next if evt_class < CCC::Command + + evt_type = evt_class.type + next if event_nodes.key?(evt_type) + next if command_ids.key?(evt_type) + + event_nodes[evt_type] = EventNode.new( + type: 'event', + id: evt_type, + name: evt_class.name, + group_id: group_id, + produces: [], + schema: extract_schema(evt_class) + ) + end + end + + is_projector = reactor < CCC::Projector + rm_id = is_projector ? "#{Sourced::Types::ModuleToMessageType.parse(group_id)}-rm" : nil + projector_aut_ids = [] + + # Automation nodes from reactions + if reactor.respond_to?(:handled_messages_for_react) + catch_all_events = reactor.respond_to?(:catch_all_react_events) ? reactor.catch_all_react_events : Set.new + specific_events = reactor.handled_messages_for_react.reject { |e| catch_all_events.include?(e) } + + # Specific reactions: one automation node per reacted event + specific_events.each do |evt_class| + produced_refs = analyzer.commands_dispatched_by(reactor, evt_class) + produced_types = resolve_refs(produced_refs, reactor) + + aut_id = "#{evt_class.type}-#{group_id}-aut" + projector_aut_ids << aut_id if is_projector + + nodes << AutomationNode.new( + type: 'automation', + id: aut_id, + name: "reaction(#{evt_class.name})", + group_id: group_id, + consumes: is_projector ? [rm_id] : [evt_class.type], + produces: produced_types + ) + end + + # Catch-all reaction: single automation node for all catch-all events + if catch_all_events.any? + produced_refs = analyzer.commands_dispatched_by(reactor, catch_all_events.first) + produced_types = resolve_refs(produced_refs, reactor) + + group_type_id = Sourced::Types::ModuleToMessageType.parse(group_id) + aut_id = "#{group_type_id}-aut" + projector_aut_ids << aut_id if is_projector + + consumes = if is_projector + [rm_id] + else + catch_all_events.map(&:type) + end + + nodes << AutomationNode.new( + type: 'automation', + id: aut_id, + name: "reaction(#{group_id})", + group_id: group_id, + consumes: consumes, + produces: produced_types + ) + end + end + + # ReadModel nodes (projectors only) + if is_projector + consumes = reactor.handled_messages_for_evolve + .reject { |c| c < CCC::Command } + .map(&:type) + + nodes << ReadModelNode.new( + type: 'readmodel', + id: rm_id, + name: group_id, + group_id: group_id, + consumes: consumes, + produces: projector_aut_ids, + schema: {} + ) + end + end + + nodes + event_nodes.values + end + + # Resolve AST references to CCC message type strings. + # + # Each reference is a two-element array from the Prism analyzer: + # - +[:const, "WidgetCreated"]+ — constant reference + # - +[:symbol, "widget_created"]+ — symbol reference resolved via +reactor[]+ + # - +[:const_path, "MyApp::WidgetCreated"]+ — fully qualified constant + # - +[:const_index, {receiver: "Reactor", index: "cmd"}]+ — bracket accessor + # + # @param refs [Array] AST reference pairs + # @param reactor [Class] reactor class for namespace resolution + # @return [Array] resolved, unique message type strings + def self.resolve_refs(refs, reactor) + refs.filter_map do |ref_type, ref_value| + resolve_ref(ref_type, ref_value, reactor) + end.uniq + end + + # Resolve a single AST reference to a message type string. + # + # @param ref_type [Symbol] one of +:symbol+, +:const+, +:const_path+, +:const_index+ + # @param ref_value [String, Hash] the reference value from Prism analysis + # @param reactor [Class] reactor class for namespace resolution + # @return [String, nil] resolved type string, or +nil+ if unresolvable + def self.resolve_ref(ref_type, ref_value, reactor) + case ref_type + when :symbol + resolve_symbol_ref(ref_value, reactor) + when :const + resolve_const_ref(ref_value, reactor) + when :const_path + resolve_const_path_ref(ref_value) + when :const_index + resolve_const_index_ref(ref_value, reactor) + end + end + + # Resolve a symbol reference (e.g. +:widget_created+) via the reactor's + # bracket accessor (+reactor[:widget_created]+). + # + # @param symbol_name [String] symbol name without colon + # @param reactor [Class] reactor class responding to +[]+ + # @return [String, nil] type string or +nil+ + def self.resolve_symbol_ref(symbol_name, reactor) + sym = symbol_name.to_sym + if reactor.respond_to?(:[]) + begin + reactor[sym]&.type + rescue StandardError + nil + end + end + end + + # Resolve an unqualified constant name by searching the reactor's module + # hierarchy, then top-level. + # + # @param const_name [String] unqualified constant name (e.g. +"WidgetCreated"+) + # @param reactor [Class] reactor class for namespace context + # @return [String, nil] type string or +nil+ + def self.resolve_const_ref(const_name, reactor) + klass = Sourced::Topology.send(:resolve_constant_in_context, const_name, reactor) + klass&.respond_to?(:type) ? klass.type : nil + end + + # Resolve a fully qualified constant path (e.g. +"MyApp::WidgetCreated"+). + # + # @param path [String] fully qualified constant path + # @return [String, nil] type string or +nil+ + def self.resolve_const_path_ref(path) + klass = Object.const_get(path) + klass.type + rescue NameError + nil + end + + # Resolve a +Reactor[:symbol]+ bracket-accessor reference. + # + # @param ref [Hash] with +:receiver+ (constant name) and +:index+ (symbol name) + # @param reactor [Class] reactor class for namespace resolution + # @return [String, nil] type string or +nil+ + def self.resolve_const_index_ref(ref, reactor) + receiver_name = ref[:receiver] + klass = if receiver_name.include?('::') + Object.const_get(receiver_name) + else + Sourced::Topology.send(:resolve_constant_in_context, receiver_name, reactor) + end + return nil unless klass && klass.respond_to?(:[]) + + klass[ref[:index].to_sym].type + rescue StandardError + nil + end + + # Look up a CCC event class by type string from {CCC::Message.registry}. + # + # @param type_string [String] message type (e.g. +"orders.order_placed"+) + # @return [Class, nil] the event class or +nil+ + def self.find_event_class(type_string) + CCC::Message.registry[type_string] + end + + # Extract JSON Schema from a message class's +Payload+ constant. + # + # @param msg_class [Class] a {CCC::Message} subclass + # @return [Hash] JSON Schema hash, or +{}+ if no schema is available + def self.extract_schema(msg_class) + return {} unless msg_class.const_defined?(:Payload, false) + + payload_class = msg_class::Payload + if payload_class.respond_to?(:to_json_schema) + payload_class.to_json_schema + else + {} + end + rescue StandardError + {} + end + + private_class_method :resolve_refs, :resolve_ref, :resolve_symbol_ref, + :resolve_const_ref, :resolve_const_path_ref, + :resolve_const_index_ref, + :find_event_class, :extract_schema + end + end +end diff --git a/spec/sourced/ccc/topology_spec.rb b/spec/sourced/ccc/topology_spec.rb new file mode 100644 index 00000000..71ba6e91 --- /dev/null +++ b/spec/sourced/ccc/topology_spec.rb @@ -0,0 +1,432 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +module CCCTopologyTest + # --- Messages --- + CreateWidget = Sourced::CCC::Command.define('ccc_topo.create_widget') do + attribute :widget_id, String + attribute :name, String + end + + WidgetCreated = Sourced::CCC::Event.define('ccc_topo.widget_created') do + attribute :widget_id, String + attribute :name, String + end + + NotifyWidget = Sourced::CCC::Command.define('ccc_topo.notify_widget') do + attribute :widget_id, String + end + + WidgetNotified = Sourced::CCC::Event.define('ccc_topo.widget_notified') do + attribute :widget_id, String + end + + ArchiveWidget = Sourced::CCC::Command.define('ccc_topo.archive_widget') do + attribute :widget_id, String + end + + WidgetArchived = Sourced::CCC::Event.define('ccc_topo.widget_archived') do + attribute :widget_id, String + end + + DelayedCmd = Sourced::CCC::Command.define('ccc_topo.delayed_cmd') do + attribute :widget_id, String + end + + ScheduleEvent = Sourced::CCC::Event.define('ccc_topo.schedule_event') do + attribute :widget_id, String + end + + # --- Decider --- + class WidgetDecider < Sourced::CCC::Decider + partition_by :widget_id + + state { |_| { exists: false } } + + evolve WidgetCreated do |state, _evt| + state[:exists] = true + end + + evolve WidgetArchived do |state, _evt| + state[:exists] = false + end + + command CreateWidget do |_state, cmd| + event WidgetCreated, widget_id: cmd.payload.widget_id, name: cmd.payload.name + end + + command ArchiveWidget do |_state, _cmd| + event WidgetArchived, widget_id: 'x' + end + + reaction WidgetCreated do |_state, evt| + dispatch NotifyWidget, widget_id: evt.payload.widget_id + end + end + + # --- Notifier Decider --- + class NotifierDecider < Sourced::CCC::Decider + partition_by :widget_id + + state { |_| {} } + + evolve WidgetNotified do |state, _evt| + state[:notified] = true + end + + command NotifyWidget do |_state, cmd| + event WidgetNotified, widget_id: cmd.payload.widget_id + end + end + + # --- Projector (StateStored, no reactions) --- + class WidgetListProjector < Sourced::CCC::Projector::StateStored + partition_by :widget_id + consumer_group 'CCCTopologyTest::WidgetListProjector' + + state { |_| { items: [] } } + + evolve WidgetCreated do |state, msg| + state[:items] << msg.payload.name + end + end + + # --- Projector (EventSourced, with catch-all reaction) --- + class ReactingProjector < Sourced::CCC::Projector::EventSourced + partition_by :widget_id + consumer_group 'CCCTopologyTest::ReactingProjector' + + state { |_| { items: [] } } + + evolve WidgetCreated do |state, msg| + state[:items] << msg.payload.name + end + + reaction do |_state, evt| + dispatch NotifyWidget, widget_id: 'x' + end + end + + # --- Projector with specific + catch-all reactions --- + class MixedReactingProjector < Sourced::CCC::Projector::EventSourced + partition_by :widget_id + consumer_group 'CCCTopologyTest::MixedReactingProjector' + + state { |_| { items: [] } } + + evolve WidgetCreated do |state, msg| + state[:items] << msg.payload.name + end + + evolve WidgetArchived do |_state, _msg| + end + + reaction WidgetCreated do |_state, evt| + dispatch NotifyWidget, widget_id: evt.payload.widget_id + end + + reaction do |_state, _evt| + dispatch ArchiveWidget, widget_id: 'x' + end + end + + # --- Decider with chained dispatch (.at) --- + class SchedulingDecider < Sourced::CCC::Decider + partition_by :widget_id + + state { |_| {} } + + evolve ScheduleEvent do |state, _evt| + state[:scheduled] = true + end + + command CreateWidget do |_state, cmd| + event ScheduleEvent, widget_id: cmd.payload.widget_id + end + + reaction ScheduleEvent do |_state, evt| + dispatch(DelayedCmd, widget_id: evt.payload.widget_id).at(evt.created_at + 300) + end + end +end + +RSpec.describe Sourced::CCC::Topology do + let(:nodes) { described_class.build(reactors) } + + def find_node(id) + nodes.find { |n| n.id == id } + end + + def find_nodes_by_type(type) + nodes.select { |n| n.type == type } + end + + context 'with WidgetDecider and NotifierDecider' do + let(:reactors) { [CCCTopologyTest::WidgetDecider, CCCTopologyTest::NotifierDecider] } + + it 'builds command nodes for handled commands' do + cmd_nodes = find_nodes_by_type('command') + expect(cmd_nodes.map(&:id)).to contain_exactly( + 'ccc_topo.create_widget', + 'ccc_topo.archive_widget', + 'ccc_topo.notify_widget' + ) + end + + it 'sets correct group_id on command nodes' do + node = find_node('ccc_topo.create_widget') + expect(node.group_id).to eq('CCCTopologyTest::WidgetDecider') + end + + it 'extracts produced events via Prism' do + node = find_node('ccc_topo.create_widget') + expect(node.produces).to eq(['ccc_topo.widget_created']) + end + + it 'extracts produced events for NotifierDecider' do + node = find_node('ccc_topo.notify_widget') + expect(node.produces).to eq(['ccc_topo.widget_notified']) + end + + it 'sets command name from message class' do + node = find_node('ccc_topo.create_widget') + expect(node.name).to eq('CCCTopologyTest::CreateWidget') + end + + it 'extracts schema from command payload' do + node = find_node('ccc_topo.create_widget') + expect(node.schema).to include('type' => 'object') + expect(node.schema['properties']).to include('widget_id', 'name') + end + + it 'builds event nodes deduplicated by type' do + evt_nodes = find_nodes_by_type('event') + evt_types = evt_nodes.map(&:id) + expect(evt_types).to contain_exactly( + 'ccc_topo.widget_created', + 'ccc_topo.widget_archived', + 'ccc_topo.widget_notified' + ) + end + + it 'assigns first-seen group_id to event nodes' do + node = find_node('ccc_topo.widget_created') + expect(node.group_id).to eq('CCCTopologyTest::WidgetDecider') + end + + it 'event nodes have empty produces' do + find_nodes_by_type('event').each { |n| expect(n.produces).to eq([]) } + end + + it 'extracts schema from event payload' do + node = find_node('ccc_topo.widget_created') + expect(node.schema).to include('type' => 'object') + expect(node.schema['properties']).to include('widget_id', 'name') + end + + it 'builds automation nodes for reactions' do + aut_nodes = find_nodes_by_type('automation') + expect(aut_nodes.map(&:id)).to include( + 'ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut' + ) + end + + it 'sets correct consumes on automation nodes' do + node = find_node('ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut') + expect(node.consumes).to eq(['ccc_topo.widget_created']) + end + + it 'extracts dispatched commands from reactions via Prism' do + node = find_node('ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut') + expect(node.produces).to eq(['ccc_topo.notify_widget']) + end + + it 'sets automation name from event class' do + node = find_node('ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut') + expect(node.name).to eq('reaction(CCCTopologyTest::WidgetCreated)') + end + end + + context 'with WidgetListProjector (no reactions)' do + let(:reactors) { [CCCTopologyTest::WidgetListProjector] } + + it 'does not build command nodes' do + expect(find_nodes_by_type('command')).to be_empty + end + + it 'does not build automation nodes' do + expect(find_nodes_by_type('automation')).to be_empty + end + + it 'builds event nodes from evolve handlers' do + evt_nodes = find_nodes_by_type('event') + expect(evt_nodes.map(&:id)).to eq(['ccc_topo.widget_created']) + end + + it 'builds a readmodel node' do + node = find_node('ccc_topology_test.widget_list_projector-rm') + expect(node).not_to be_nil + expect(node.type).to eq('readmodel') + end + + it 'sets correct name and group_id on readmodel node' do + node = find_node('ccc_topology_test.widget_list_projector-rm') + expect(node.name).to eq('CCCTopologyTest::WidgetListProjector') + expect(node.group_id).to eq('CCCTopologyTest::WidgetListProjector') + end + + it 'readmodel consumes the evolved event types' do + node = find_node('ccc_topology_test.widget_list_projector-rm') + expect(node.consumes).to eq(['ccc_topo.widget_created']) + end + + it 'readmodel produces nothing when there are no reactions' do + node = find_node('ccc_topology_test.widget_list_projector-rm') + expect(node.produces).to eq([]) + end + end + + context 'with ReactingProjector (catch-all reaction)' do + let(:reactors) { [CCCTopologyTest::ReactingProjector] } + let(:rm_id) { 'ccc_topology_test.reacting_projector-rm' } + let(:aut_id) { 'ccc_topology_test.reacting_projector-aut' } + + it 'builds a readmodel node' do + node = find_node(rm_id) + expect(node).not_to be_nil + expect(node.type).to eq('readmodel') + end + + it 'readmodel consumes the evolved event types' do + node = find_node(rm_id) + expect(node.consumes).to eq(['ccc_topo.widget_created']) + end + + it 'readmodel produces a single automation node' do + node = find_node(rm_id) + expect(node.produces).to eq([aut_id]) + end + + it 'builds a single catch-all automation node' do + aut_nodes = find_nodes_by_type('automation') + expect(aut_nodes.size).to eq(1) + node = aut_nodes.first + expect(node.id).to eq(aut_id) + expect(node.name).to eq('reaction(CCCTopologyTest::ReactingProjector)') + end + + it 'automation node consumes the readmodel' do + node = find_node(aut_id) + expect(node.consumes).to eq([rm_id]) + end + + it 'automation node produces dispatched commands' do + node = find_node(aut_id) + expect(node.produces).to eq(['ccc_topo.notify_widget']) + end + end + + context 'with MixedReactingProjector (specific + catch-all reactions)' do + let(:reactors) { [CCCTopologyTest::MixedReactingProjector] } + let(:rm_id) { 'ccc_topology_test.mixed_reacting_projector-rm' } + let(:specific_aut_id) { 'ccc_topo.widget_created-CCCTopologyTest::MixedReactingProjector-aut' } + let(:catchall_aut_id) { 'ccc_topology_test.mixed_reacting_projector-aut' } + + it 'builds two automation nodes: one specific and one catch-all' do + aut_nodes = find_nodes_by_type('automation') + expect(aut_nodes.map(&:id)).to contain_exactly(specific_aut_id, catchall_aut_id) + end + + it 'specific automation is named after the event' do + node = find_node(specific_aut_id) + expect(node.name).to eq('reaction(CCCTopologyTest::WidgetCreated)') + end + + it 'catch-all automation is named after the reactor' do + node = find_node(catchall_aut_id) + expect(node.name).to eq('reaction(CCCTopologyTest::MixedReactingProjector)') + end + + it 'both automation nodes consume the readmodel' do + [specific_aut_id, catchall_aut_id].each do |id| + node = find_node(id) + expect(node.consumes).to eq([rm_id]) + end + end + + it 'readmodel produces both automation node IDs' do + node = find_node(rm_id) + expect(node.produces).to contain_exactly(specific_aut_id, catchall_aut_id) + end + end + + context 'with SchedulingDecider (chained dispatch)' do + let(:reactors) { [CCCTopologyTest::SchedulingDecider] } + + it 'detects dispatch through .at() chain' do + node = find_node('ccc_topo.schedule_event-CCCTopologyTest::SchedulingDecider-aut') + expect(node).not_to be_nil + expect(node.produces).to eq(['ccc_topo.delayed_cmd']) + end + end + + context 'event deduplication across reactors' do + let(:reactors) { [CCCTopologyTest::WidgetDecider, CCCTopologyTest::WidgetListProjector] } + + it 'deduplicates event nodes by type string' do + evt_nodes = find_nodes_by_type('event').select { |n| n.id == 'ccc_topo.widget_created' } + expect(evt_nodes.size).to eq(1) + end + + it 'uses first reactor as group_id owner' do + node = find_node('ccc_topo.widget_created') + expect(node.group_id).to eq('CCCTopologyTest::WidgetDecider') + end + end + + context 'command deduplication across reactors' do + let(:reactors) { [CCCTopologyTest::WidgetDecider, CCCTopologyTest::SchedulingDecider] } + + it 'deduplicates command nodes by type string' do + cmd_nodes = find_nodes_by_type('command').select { |n| n.id == 'ccc_topo.create_widget' } + expect(cmd_nodes.size).to eq(1) + end + + it 'produces no duplicate IDs' do + ids = nodes.map(&:id) + expect(ids).to eq(ids.uniq) + end + end + + context 'with all test reactors' do + let(:reactors) do + [ + CCCTopologyTest::WidgetDecider, + CCCTopologyTest::NotifierDecider, + CCCTopologyTest::WidgetListProjector, + CCCTopologyTest::ReactingProjector, + CCCTopologyTest::SchedulingDecider + ] + end + + it 'returns flat array of node structs' do + nodes.each do |n| + expect(n).to be_a(Struct) + expect(%w[command event automation readmodel]).to include(n.type) + end + end + + it 'all command nodes have produces arrays' do + find_nodes_by_type('command').each { |n| expect(n.produces).to be_an(Array) } + end + + it 'all automation nodes have consumes and produces arrays' do + find_nodes_by_type('automation').each do |n| + expect(n.consumes).to be_an(Array) + expect(n.produces).to be_an(Array) + end + end + end +end From 039db1f27362af88109419a2d61a1dfa40c65cda Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 14:55:29 +0000 Subject: [PATCH 043/115] Falcon integration for Sourced::CCC with deferred configuration (to run after fork) --- lib/sourced/ccc.rb | 57 ++++++++++++-- lib/sourced/ccc/configuration.rb | 8 ++ lib/sourced/ccc/falcon.rb | 5 ++ spec/sourced/ccc/configuration_spec.rb | 100 +++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 lib/sourced/ccc/falcon.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 96b7ebe0..fba02d82 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -9,12 +9,22 @@ def self.config @config ||= Configuration.new end - # Configure the CCC module. Calls setup! after yielding. + # Configure the CCC module. Stores the block for re-running after fork + # (see {.setup!}), then runs it immediately. # @yieldparam config [Configuration] - def self.configure(&) - yield config if block_given? - config.setup! - config.freeze + def self.configure(&block) + @configure_block = block + setup! + end + + # Run (or re-run) the configure block on a fresh Configuration. + # Safe to call after a process fork to re-establish database connections. + # @return [void] + def self.setup! + @config = Configuration.new + @configure_block&.call(@config) + @config.setup! + @config.freeze end # Register a reactor class with the global router. @@ -40,6 +50,42 @@ def self.router # Reset the global configuration. For test teardown. def self.reset! @config = nil + @configure_block = nil + @topology = nil + end + + # Build and cache the topology graph from all reactors registered with + # the global {.router}. The result is memoized; call {.reset_topology} + # to force a rebuild after registering new reactors. + # + # @return [Array] + # flat array of topology node structs + # + # @example Inspect the global topology + # Sourced::CCC.register(MyDecider) + # Sourced::CCC.register(MyProjector) + # + # Sourced::CCC.topology.each do |node| + # puts "#{node.type}: #{node.name} (#{node.id})" + # end + # + # @see CCC::Topology.build + def self.topology + @topology ||= CCC::Topology.build(router.reactors) + end + + # Clear the cached topology so it is rebuilt on next access to {.topology}. + # Useful after registering additional reactors at runtime. + # + # @return [nil] + # + # @example + # Sourced::CCC.register(LateAddedDecider) + # Sourced::CCC.reset_topology + # Sourced::CCC.topology # now includes LateAddedDecider + def self.reset_topology + @topology = nil end # Returned by {.handle!} with command, reactor instance, and new events. @@ -189,4 +235,5 @@ def self.load(reactor_class, store: nil, **partition_attrs) require 'sourced/ccc/worker' require 'sourced/ccc/stale_claim_reaper' require 'sourced/ccc/dispatcher' +require 'sourced/ccc/topology' require 'sourced/ccc/supervisor' diff --git a/lib/sourced/ccc/configuration.rb b/lib/sourced/ccc/configuration.rb index fbe5ea34..d76a8c78 100644 --- a/lib/sourced/ccc/configuration.rb +++ b/lib/sourced/ccc/configuration.rb @@ -35,6 +35,7 @@ def initialize @store = nil @router = nil @error_strategy = nil + @reactors = [] @setup = false end @@ -60,12 +61,19 @@ def error_strategy @error_strategy || Sourced.config.error_strategy end + # Buffer a reactor class for registration during {#setup!}. + # @param reactor [Class] a CCC reactor class + def register(reactor) + @reactors << reactor + end + def setup! return if @setup @store ||= Store.new(Sequel.sqlite) @store.install! @router ||= Router.new(store: @store) + @reactors.each { |r| @router.register(r) } @setup = true end end diff --git a/lib/sourced/ccc/falcon.rb b/lib/sourced/ccc/falcon.rb new file mode 100644 index 00000000..3154e735 --- /dev/null +++ b/lib/sourced/ccc/falcon.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require 'falcon' +require_relative 'falcon/environment' +require_relative 'falcon/service' diff --git a/spec/sourced/ccc/configuration_spec.rb b/spec/sourced/ccc/configuration_spec.rb index eeb0fd30..f576cec8 100644 --- a/spec/sourced/ccc/configuration_spec.rb +++ b/spec/sourced/ccc/configuration_spec.rb @@ -80,12 +80,84 @@ def self.name = 'TestConfigReactor' end end + describe 'CCC.setup!' do + let(:reactor_class) do + Class.new(Sourced::CCC::Projector::StateStored) do + def self.name = 'SetupTestReactor' + + consumer_group 'setup-test-reactor' + partition_by :thing_id + + state { |_| {} } + end + end + + it 'replays the configure block on a fresh Configuration' do + call_count = 0 + Sourced::CCC.configure do |c| + call_count += 1 + c.worker_count = 8 + end + + expect(call_count).to eq(1) + original_config = Sourced::CCC.config + + Sourced::CCC.setup! + + expect(call_count).to eq(2) + expect(Sourced::CCC.config).not_to be(original_config) + expect(Sourced::CCC.config.worker_count).to eq(8) + expect(Sourced::CCC.config).to be_frozen + end + + it 'creates a new store connection on each call' do + Sourced::CCC.configure {} + store1 = Sourced::CCC.config.store + + Sourced::CCC.setup! + store2 = Sourced::CCC.config.store + + expect(store2).not_to be(store1) + end + + it 'replays registrations from the configure block' do + Sourced::CCC.configure do |c| + c.register(reactor_class) + end + + expect(Sourced::CCC.router.reactors).to include(reactor_class) + + Sourced::CCC.setup! + + expect(Sourced::CCC.router.reactors).to include(reactor_class) + end + + it 'works without a configure block' do + Sourced::CCC.setup! + + expect(Sourced::CCC.config.store).to be_a(Sourced::CCC::Store) + expect(Sourced::CCC.config.router).to be_a(Sourced::CCC::Router) + expect(Sourced::CCC.config).to be_frozen + end + end + describe 'CCC.reset!' do it 'clears the singleton config' do original = Sourced::CCC.config Sourced::CCC.reset! expect(Sourced::CCC.config).not_to be(original) end + + it 'clears the stored configure block' do + Sourced::CCC.configure do |c| + c.worker_count = 8 + end + + Sourced::CCC.reset! + Sourced::CCC.setup! + + expect(Sourced::CCC.config.worker_count).to eq(2) + end end describe '#store=' do @@ -145,6 +217,34 @@ def self.name = 'TestConfigReactor' end end + describe '#register' do + let(:reactor_class) do + Class.new(Sourced::CCC::Projector::StateStored) do + def self.name = 'ConfigRegisterReactor' + + consumer_group 'config-register-reactor' + partition_by :thing_id + + state { |_| {} } + end + end + + it 'buffers reactors and registers them during setup!' do + config = described_class.new + config.register(reactor_class) + config.setup! + + expect(config.router.reactors).to include(reactor_class) + end + + it 'does not register before setup! is called' do + config = described_class.new + config.register(reactor_class) + + expect(config.router).to be_nil + end + end + describe '#setup!' do it 'is idempotent' do config = described_class.new From 59c3cebd901229bc53d6fedd650967d2c071387c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 14:59:41 +0000 Subject: [PATCH 044/115] Add CCC::Falcon::Environment and Service with post-fork setup, update docs - CCC::Falcon::Environment mixin and CCC::Falcon::Service for running web server + CCC workers as sibling Falcon fibers - Service calls CCC.setup! on run to replay the configure block with fresh SQLite connections after Falcon forks - Simplify examples/ccc_app/falcon.rb to use the new library integration - Document Falcon usage, CCC.setup!, and c.register in CCC README Co-Authored-By: Claude Opus 4.6 --- examples/ccc_app/domain.rb | 149 ++++++++++++++++++++++++++ examples/ccc_app/falcon.rb | 14 +++ lib/sourced/ccc/README.md | 66 +++++++++++- lib/sourced/ccc/falcon/environment.rb | 34 ++++++ lib/sourced/ccc/falcon/service.rb | 50 +++++++++ 5 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 examples/ccc_app/domain.rb create mode 100644 examples/ccc_app/falcon.rb create mode 100644 lib/sourced/ccc/falcon/environment.rb create mode 100644 lib/sourced/ccc/falcon/service.rb diff --git a/examples/ccc_app/domain.rb b/examples/ccc_app/domain.rb new file mode 100644 index 00000000..4bf24315 --- /dev/null +++ b/examples/ccc_app/domain.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'bundler/setup' +require 'sourced' +require 'sourced/ccc' +require 'sequel' +require 'fileutils' +require 'json' + +CCC = Sourced::CCC + +module CourseApp + # --- Messages --- + + class Event < CCC::Message; end + class Command < CCC::Message; end + + CreateCourse = Command.define('courses.create') { attribute :course_id, String; attribute :course_name, String } + CourseCreated = Event.define('courses.created') { attribute :course_id, String; attribute :course_name, String } + EnrolStudent = Command.define('courses.enrol') { attribute :course_id, String; attribute :student_id, String } + StudentEnrolled = Event.define('courses.enrolled') { attribute :course_id, String; attribute :student_id, String } + + # --- Deciders --- + + # Enforces course name uniqueness. + # Partition by :course_name so CCC.load reads all CourseCreated with that name. + class CourseDecider < CCC::Decider + partition_by :course_name + + state do |_partition_values| + { name_taken: false } + end + + evolve CourseCreated do |state, _event| + state[:name_taken] = true + end + + command CreateCourse do |state, cmd| + raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] + + event CourseCreated, course_id: cmd.payload.course_id, course_name: cmd.payload.course_name + end + end + + # Enforces enrolment rules: course must exist, no duplicates, max 20 students. + # Partition by :course_id so CCC.load reads CourseCreated + StudentEnrolled for that course. + class EnrolmentDecider < CCC::Decider + partition_by :course_id + + state do |_partition_values| + { course_exists: false, student_ids: [], student_count: 0 } + end + + evolve CourseCreated do |state, _event| + state[:course_exists] = true + end + + evolve StudentEnrolled do |state, event| + state[:student_ids] << event.payload.student_id + state[:student_count] += 1 + end + + command EnrolStudent do |state, cmd| + raise "Course '#{cmd.payload.course_id}' does not exist" unless state[:course_exists] + raise "Student '#{cmd.payload.student_id}' is already enrolled" if state[:student_ids].include?(cmd.payload.student_id) + raise "Course is full (max 20 students). Has #{state[:student_count]}" if state[:student_count] >= 20 + + event StudentEnrolled, + course_id: cmd.payload.course_id, + student_id: cmd.payload.student_id + end + end + + # --- Projector (async read model) --- + + # Builds a file-backed course catalog from events. + # Each course is written to a JSON file in storage/projections/. + # Registered with CCC for background processing by workers. + class CourseCatalogProjector < CCC::Projector::EventSourced + partition_by :course_id + + PROJECTIONS_DIR = File.join(__dir__, 'storage', 'projections') + + class << self + def projection_path(course_id) + File.join(PROJECTIONS_DIR, "#{course_id}.json") + end + + def read_course(course_id) + path = projection_path(course_id) + return nil unless File.exist?(path) + + JSON.parse(File.read(path), symbolize_names: true) + end + + def all_courses + Dir.glob(File.join(PROJECTIONS_DIR, '*.json')).filter_map do |path| + JSON.parse(File.read(path), symbolize_names: true) + rescue JSON::ParserError + nil + end + end + end + + state do |partition_values| + { course_id: nil, course_name: nil, students: [] } + # course_id = partition_values&.first + # existing = self.class.read_course(course_id) if course_id + # if existing + # { course_id: existing[:course_id], course_name: existing[:course_name], students: Array(existing[:students]).dup } + # else + # { course_id: nil, course_name: nil, students: [] } + # end + end + + evolve CourseCreated do |state, event| + state[:course_id] = event.payload.course_id + state[:course_name] = event.payload.course_name + end + + evolve StudentEnrolled do |state, event| + state[:students] << event.payload.student_id + end + + sync do |state:, messages:, **| + next unless state[:course_id] + + FileUtils.mkdir_p(PROJECTIONS_DIR) + data = { + course_id: state[:course_id], + course_name: state[:course_name], + students: state[:students], + student_count: state[:students].size + } + File.write(self.class.projection_path(state[:course_id]), JSON.pretty_generate(data)) + end + end + + # --- Configuration --- + + DB_PATH = File.join(__dir__, 'storage', 'ccc_app.db') + + CCC.configure do |c| + c.store = Sequel.sqlite(DB_PATH) + c.register(CourseDecider) + c.register(EnrolmentDecider) + c.register(CourseCatalogProjector) + end +end diff --git a/examples/ccc_app/falcon.rb b/examples/ccc_app/falcon.rb new file mode 100644 index 00000000..e7738a58 --- /dev/null +++ b/examples/ccc_app/falcon.rb @@ -0,0 +1,14 @@ +#!/usr/bin/env falcon-host +# frozen_string_literal: true + +require_relative 'domain' +require_relative 'app' +require 'sourced/ccc/falcon' + +service "ccc-app" do + include Sourced::CCC::Falcon::Environment + include Falcon::Environment::Rackup + + # url "http://localhost:9292" + count 1 +end diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index c1782cd8..0dc2518a 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -246,6 +246,10 @@ Sourced::CCC.configure do |c| # Pass a Sequel SQLite connection or a CCC::Store instance c.store = Sequel.sqlite('my_app.db') + # Register reactors (recommended when using Falcon — see below) + c.register(CourseDecider) + c.register(CourseCatalogProjector) + # Optional settings c.worker_count = 4 # background worker fibers (default: 2) c.batch_size = 50 # messages per claim (default: 50) @@ -256,6 +260,17 @@ Sourced::CCC.configure do |c| end ``` +### `CCC.setup!` + +`CCC.configure` stores its block and runs it immediately. You can re-run it later with `CCC.setup!` to create a fresh `Configuration` with new database connections. This is useful after process forks (e.g. Falcon), where SQLite connections become invalid. + +```ruby +# Re-run the configure block (creates fresh store, router, and registrations) +Sourced::CCC.setup! +``` + +`CCC::Falcon::Service` calls `setup!` automatically — you don't need to call it yourself when using the Falcon integration. + ## Failure handling and retries CCC already supports consumer-group retries on failure. @@ -311,7 +326,18 @@ After the configured retries are exhausted, the consumer group is marked as fail ## Registering reactors +Reactors can be registered inside the `configure` block or separately via `CCC.register`. + ```ruby +# Inside the configure block (recommended for Falcon — survives fork replay) +Sourced::CCC.configure do |c| + c.store = Sequel.sqlite('my_app.db') + c.register(CourseDecider) + c.register(EnrolmentDecider) + c.register(CourseCatalogProjector) +end + +# Or separately (fine for scripts and single-process apps) Sourced::CCC.register(CourseDecider) Sourced::CCC.register(EnrolmentDecider) Sourced::CCC.register(CourseCatalogProjector) @@ -321,7 +347,45 @@ This registers the reactor's consumer group with the store and adds it to the gl ## Background processing -The supervisor starts workers that claim partitions, process messages, and ack offsets. +### Falcon (recommended) + +`CCC::Falcon` provides a ready-made Falcon service that runs both the web server and CCC background workers as sibling fibers. No separate worker process needed. + +```ruby +# falcon.rb +#!/usr/bin/env falcon-host +require_relative 'domain' +require_relative 'app' +require 'sourced/ccc/falcon' + +service "my-app" do + include Sourced::CCC::Falcon::Environment + include Falcon::Environment::Rackup + + url "http://localhost:9292" + count 1 +end +``` + +Start with: + +```bash +bundle exec falcon host +``` + +The service automatically calls `CCC.setup!` in each forked process, which replays the `CCC.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe — unlike Postgres, where Sequel reconnects lazily after fork. + +This is why registering reactors inside the `configure` block is recommended when using Falcon: registrations made via `CCC.register` outside the block won't be replayed after fork. + +#### How it works + +- `CCC::Falcon::Environment` — mixin that sets the `service_class` to `CCC::Falcon::Service`. Include it in your Falcon service definition alongside `Falcon::Environment::Rackup`. +- `CCC::Falcon::Service` — extends `Falcon::Service::Server`. On `run`, it calls `CCC.setup!`, starts the web server, and spawns a `CCC::Dispatcher` with all settings from `CCC.config`. On `stop`, it shuts down the dispatcher before the server. +- No separate HouseKeeper fibers are needed — the `StaleClaimReaper` is embedded in the CCC Dispatcher. + +### Supervisor (standalone) + +For running workers without a web server, the supervisor starts workers that claim partitions, process messages, and ack offsets. ```ruby # Start blocking (handles INT/TERM signals for graceful shutdown) diff --git a/lib/sourced/ccc/falcon/environment.rb b/lib/sourced/ccc/falcon/environment.rb new file mode 100644 index 00000000..e22f7f40 --- /dev/null +++ b/lib/sourced/ccc/falcon/environment.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sourced + module CCC + module Falcon + # Environment mixin for configuring a combined Falcon web server + CCC workers service. + # + # Include this module in a Falcon service definition to get CCC worker defaults + # alongside the standard Falcon server environment. All settings are read from + # {CCC.config} — no per-service config methods needed. + # + # The Service automatically calls {CCC.setup!} at the start of +run+ to + # re-establish database connections after Falcon forks (SQLite connections + # are not fork-safe). This replays the block passed to {CCC.configure}. + # + # @example falcon.rb + # #!/usr/bin/env falcon-host + # require 'sourced/ccc/falcon' + # require_relative 'config/environment' + # + # service "my-app" do + # include Sourced::CCC::Falcon::Environment + # include Falcon::Environment::Rackup + # + # url "http://[::]:9292" + # end + module Environment + include ::Falcon::Environment::Server + + def service_class = CCC::Falcon::Service + end + end + end +end diff --git a/lib/sourced/ccc/falcon/service.rb b/lib/sourced/ccc/falcon/service.rb new file mode 100644 index 00000000..d2116082 --- /dev/null +++ b/lib/sourced/ccc/falcon/service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'sourced/ccc/dispatcher' + +module Sourced + module CCC + module Falcon + # A Falcon service that runs both the web server and CCC background workers + # as sibling fibers within the same Async reactor. + # + # Uses a CCC::Dispatcher for signal-driven worker dispatch. The Dispatcher + # already embeds the StaleClaimReaper, so no separate HouseKeeper is needed + # (unlike {Sourced::Falcon::Service}). + # + # All configuration is read from {CCC.config}. + class Service < ::Falcon::Service::Server + def run(instance, evaluator) + CCC.setup! + + server = evaluator.make_server(@bound_endpoint) + + config = CCC.config + @dispatcher = CCC::Dispatcher.new( + router: CCC.router, + worker_count: config.worker_count, + batch_size: config.batch_size, + max_drain_rounds: config.max_drain_rounds, + catchup_interval: config.catchup_interval, + housekeeping_interval: config.housekeeping_interval, + claim_ttl_seconds: config.claim_ttl_seconds, + logger: config.logger + ) + + Async do |task| + server.run + @dispatcher.spawn_into(task) + task.children.each(&:wait) + end + + server + end + + def stop(...) + @dispatcher&.stop + super + end + end + end + end +end From 822d9324bf4321fd08d1a4c46f73f1a8bec5d427 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 15:00:53 +0000 Subject: [PATCH 045/115] Revert "Add CCC::Falcon::Environment and Service with post-fork setup, update docs" This reverts commit 59c3cebd901229bc53d6fedd650967d2c071387c. --- examples/ccc_app/domain.rb | 149 -------------------------- examples/ccc_app/falcon.rb | 14 --- lib/sourced/ccc/README.md | 66 +----------- lib/sourced/ccc/falcon/environment.rb | 34 ------ lib/sourced/ccc/falcon/service.rb | 50 --------- 5 files changed, 1 insertion(+), 312 deletions(-) delete mode 100644 examples/ccc_app/domain.rb delete mode 100644 examples/ccc_app/falcon.rb delete mode 100644 lib/sourced/ccc/falcon/environment.rb delete mode 100644 lib/sourced/ccc/falcon/service.rb diff --git a/examples/ccc_app/domain.rb b/examples/ccc_app/domain.rb deleted file mode 100644 index 4bf24315..00000000 --- a/examples/ccc_app/domain.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true - -require 'bundler/setup' -require 'sourced' -require 'sourced/ccc' -require 'sequel' -require 'fileutils' -require 'json' - -CCC = Sourced::CCC - -module CourseApp - # --- Messages --- - - class Event < CCC::Message; end - class Command < CCC::Message; end - - CreateCourse = Command.define('courses.create') { attribute :course_id, String; attribute :course_name, String } - CourseCreated = Event.define('courses.created') { attribute :course_id, String; attribute :course_name, String } - EnrolStudent = Command.define('courses.enrol') { attribute :course_id, String; attribute :student_id, String } - StudentEnrolled = Event.define('courses.enrolled') { attribute :course_id, String; attribute :student_id, String } - - # --- Deciders --- - - # Enforces course name uniqueness. - # Partition by :course_name so CCC.load reads all CourseCreated with that name. - class CourseDecider < CCC::Decider - partition_by :course_name - - state do |_partition_values| - { name_taken: false } - end - - evolve CourseCreated do |state, _event| - state[:name_taken] = true - end - - command CreateCourse do |state, cmd| - raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] - - event CourseCreated, course_id: cmd.payload.course_id, course_name: cmd.payload.course_name - end - end - - # Enforces enrolment rules: course must exist, no duplicates, max 20 students. - # Partition by :course_id so CCC.load reads CourseCreated + StudentEnrolled for that course. - class EnrolmentDecider < CCC::Decider - partition_by :course_id - - state do |_partition_values| - { course_exists: false, student_ids: [], student_count: 0 } - end - - evolve CourseCreated do |state, _event| - state[:course_exists] = true - end - - evolve StudentEnrolled do |state, event| - state[:student_ids] << event.payload.student_id - state[:student_count] += 1 - end - - command EnrolStudent do |state, cmd| - raise "Course '#{cmd.payload.course_id}' does not exist" unless state[:course_exists] - raise "Student '#{cmd.payload.student_id}' is already enrolled" if state[:student_ids].include?(cmd.payload.student_id) - raise "Course is full (max 20 students). Has #{state[:student_count]}" if state[:student_count] >= 20 - - event StudentEnrolled, - course_id: cmd.payload.course_id, - student_id: cmd.payload.student_id - end - end - - # --- Projector (async read model) --- - - # Builds a file-backed course catalog from events. - # Each course is written to a JSON file in storage/projections/. - # Registered with CCC for background processing by workers. - class CourseCatalogProjector < CCC::Projector::EventSourced - partition_by :course_id - - PROJECTIONS_DIR = File.join(__dir__, 'storage', 'projections') - - class << self - def projection_path(course_id) - File.join(PROJECTIONS_DIR, "#{course_id}.json") - end - - def read_course(course_id) - path = projection_path(course_id) - return nil unless File.exist?(path) - - JSON.parse(File.read(path), symbolize_names: true) - end - - def all_courses - Dir.glob(File.join(PROJECTIONS_DIR, '*.json')).filter_map do |path| - JSON.parse(File.read(path), symbolize_names: true) - rescue JSON::ParserError - nil - end - end - end - - state do |partition_values| - { course_id: nil, course_name: nil, students: [] } - # course_id = partition_values&.first - # existing = self.class.read_course(course_id) if course_id - # if existing - # { course_id: existing[:course_id], course_name: existing[:course_name], students: Array(existing[:students]).dup } - # else - # { course_id: nil, course_name: nil, students: [] } - # end - end - - evolve CourseCreated do |state, event| - state[:course_id] = event.payload.course_id - state[:course_name] = event.payload.course_name - end - - evolve StudentEnrolled do |state, event| - state[:students] << event.payload.student_id - end - - sync do |state:, messages:, **| - next unless state[:course_id] - - FileUtils.mkdir_p(PROJECTIONS_DIR) - data = { - course_id: state[:course_id], - course_name: state[:course_name], - students: state[:students], - student_count: state[:students].size - } - File.write(self.class.projection_path(state[:course_id]), JSON.pretty_generate(data)) - end - end - - # --- Configuration --- - - DB_PATH = File.join(__dir__, 'storage', 'ccc_app.db') - - CCC.configure do |c| - c.store = Sequel.sqlite(DB_PATH) - c.register(CourseDecider) - c.register(EnrolmentDecider) - c.register(CourseCatalogProjector) - end -end diff --git a/examples/ccc_app/falcon.rb b/examples/ccc_app/falcon.rb deleted file mode 100644 index e7738a58..00000000 --- a/examples/ccc_app/falcon.rb +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env falcon-host -# frozen_string_literal: true - -require_relative 'domain' -require_relative 'app' -require 'sourced/ccc/falcon' - -service "ccc-app" do - include Sourced::CCC::Falcon::Environment - include Falcon::Environment::Rackup - - # url "http://localhost:9292" - count 1 -end diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 0dc2518a..c1782cd8 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -246,10 +246,6 @@ Sourced::CCC.configure do |c| # Pass a Sequel SQLite connection or a CCC::Store instance c.store = Sequel.sqlite('my_app.db') - # Register reactors (recommended when using Falcon — see below) - c.register(CourseDecider) - c.register(CourseCatalogProjector) - # Optional settings c.worker_count = 4 # background worker fibers (default: 2) c.batch_size = 50 # messages per claim (default: 50) @@ -260,17 +256,6 @@ Sourced::CCC.configure do |c| end ``` -### `CCC.setup!` - -`CCC.configure` stores its block and runs it immediately. You can re-run it later with `CCC.setup!` to create a fresh `Configuration` with new database connections. This is useful after process forks (e.g. Falcon), where SQLite connections become invalid. - -```ruby -# Re-run the configure block (creates fresh store, router, and registrations) -Sourced::CCC.setup! -``` - -`CCC::Falcon::Service` calls `setup!` automatically — you don't need to call it yourself when using the Falcon integration. - ## Failure handling and retries CCC already supports consumer-group retries on failure. @@ -326,18 +311,7 @@ After the configured retries are exhausted, the consumer group is marked as fail ## Registering reactors -Reactors can be registered inside the `configure` block or separately via `CCC.register`. - ```ruby -# Inside the configure block (recommended for Falcon — survives fork replay) -Sourced::CCC.configure do |c| - c.store = Sequel.sqlite('my_app.db') - c.register(CourseDecider) - c.register(EnrolmentDecider) - c.register(CourseCatalogProjector) -end - -# Or separately (fine for scripts and single-process apps) Sourced::CCC.register(CourseDecider) Sourced::CCC.register(EnrolmentDecider) Sourced::CCC.register(CourseCatalogProjector) @@ -347,45 +321,7 @@ This registers the reactor's consumer group with the store and adds it to the gl ## Background processing -### Falcon (recommended) - -`CCC::Falcon` provides a ready-made Falcon service that runs both the web server and CCC background workers as sibling fibers. No separate worker process needed. - -```ruby -# falcon.rb -#!/usr/bin/env falcon-host -require_relative 'domain' -require_relative 'app' -require 'sourced/ccc/falcon' - -service "my-app" do - include Sourced::CCC::Falcon::Environment - include Falcon::Environment::Rackup - - url "http://localhost:9292" - count 1 -end -``` - -Start with: - -```bash -bundle exec falcon host -``` - -The service automatically calls `CCC.setup!` in each forked process, which replays the `CCC.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe — unlike Postgres, where Sequel reconnects lazily after fork. - -This is why registering reactors inside the `configure` block is recommended when using Falcon: registrations made via `CCC.register` outside the block won't be replayed after fork. - -#### How it works - -- `CCC::Falcon::Environment` — mixin that sets the `service_class` to `CCC::Falcon::Service`. Include it in your Falcon service definition alongside `Falcon::Environment::Rackup`. -- `CCC::Falcon::Service` — extends `Falcon::Service::Server`. On `run`, it calls `CCC.setup!`, starts the web server, and spawns a `CCC::Dispatcher` with all settings from `CCC.config`. On `stop`, it shuts down the dispatcher before the server. -- No separate HouseKeeper fibers are needed — the `StaleClaimReaper` is embedded in the CCC Dispatcher. - -### Supervisor (standalone) - -For running workers without a web server, the supervisor starts workers that claim partitions, process messages, and ack offsets. +The supervisor starts workers that claim partitions, process messages, and ack offsets. ```ruby # Start blocking (handles INT/TERM signals for graceful shutdown) diff --git a/lib/sourced/ccc/falcon/environment.rb b/lib/sourced/ccc/falcon/environment.rb deleted file mode 100644 index e22f7f40..00000000 --- a/lib/sourced/ccc/falcon/environment.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - module Falcon - # Environment mixin for configuring a combined Falcon web server + CCC workers service. - # - # Include this module in a Falcon service definition to get CCC worker defaults - # alongside the standard Falcon server environment. All settings are read from - # {CCC.config} — no per-service config methods needed. - # - # The Service automatically calls {CCC.setup!} at the start of +run+ to - # re-establish database connections after Falcon forks (SQLite connections - # are not fork-safe). This replays the block passed to {CCC.configure}. - # - # @example falcon.rb - # #!/usr/bin/env falcon-host - # require 'sourced/ccc/falcon' - # require_relative 'config/environment' - # - # service "my-app" do - # include Sourced::CCC::Falcon::Environment - # include Falcon::Environment::Rackup - # - # url "http://[::]:9292" - # end - module Environment - include ::Falcon::Environment::Server - - def service_class = CCC::Falcon::Service - end - end - end -end diff --git a/lib/sourced/ccc/falcon/service.rb b/lib/sourced/ccc/falcon/service.rb deleted file mode 100644 index d2116082..00000000 --- a/lib/sourced/ccc/falcon/service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/ccc/dispatcher' - -module Sourced - module CCC - module Falcon - # A Falcon service that runs both the web server and CCC background workers - # as sibling fibers within the same Async reactor. - # - # Uses a CCC::Dispatcher for signal-driven worker dispatch. The Dispatcher - # already embeds the StaleClaimReaper, so no separate HouseKeeper is needed - # (unlike {Sourced::Falcon::Service}). - # - # All configuration is read from {CCC.config}. - class Service < ::Falcon::Service::Server - def run(instance, evaluator) - CCC.setup! - - server = evaluator.make_server(@bound_endpoint) - - config = CCC.config - @dispatcher = CCC::Dispatcher.new( - router: CCC.router, - worker_count: config.worker_count, - batch_size: config.batch_size, - max_drain_rounds: config.max_drain_rounds, - catchup_interval: config.catchup_interval, - housekeeping_interval: config.housekeeping_interval, - claim_ttl_seconds: config.claim_ttl_seconds, - logger: config.logger - ) - - Async do |task| - server.run - @dispatcher.spawn_into(task) - task.children.each(&:wait) - end - - server - end - - def stop(...) - @dispatcher&.stop - super - end - end - end - end -end From 55be5e092acc6ea82cb215c38688928b097f7558 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 15:06:21 +0000 Subject: [PATCH 046/115] Falcon support --- lib/sourced/ccc/README.md | 66 ++++++++++++++++++++++++++- lib/sourced/ccc/falcon/environment.rb | 34 ++++++++++++++ lib/sourced/ccc/falcon/service.rb | 50 ++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 lib/sourced/ccc/falcon/environment.rb create mode 100644 lib/sourced/ccc/falcon/service.rb diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index c1782cd8..0dc2518a 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -246,6 +246,10 @@ Sourced::CCC.configure do |c| # Pass a Sequel SQLite connection or a CCC::Store instance c.store = Sequel.sqlite('my_app.db') + # Register reactors (recommended when using Falcon — see below) + c.register(CourseDecider) + c.register(CourseCatalogProjector) + # Optional settings c.worker_count = 4 # background worker fibers (default: 2) c.batch_size = 50 # messages per claim (default: 50) @@ -256,6 +260,17 @@ Sourced::CCC.configure do |c| end ``` +### `CCC.setup!` + +`CCC.configure` stores its block and runs it immediately. You can re-run it later with `CCC.setup!` to create a fresh `Configuration` with new database connections. This is useful after process forks (e.g. Falcon), where SQLite connections become invalid. + +```ruby +# Re-run the configure block (creates fresh store, router, and registrations) +Sourced::CCC.setup! +``` + +`CCC::Falcon::Service` calls `setup!` automatically — you don't need to call it yourself when using the Falcon integration. + ## Failure handling and retries CCC already supports consumer-group retries on failure. @@ -311,7 +326,18 @@ After the configured retries are exhausted, the consumer group is marked as fail ## Registering reactors +Reactors can be registered inside the `configure` block or separately via `CCC.register`. + ```ruby +# Inside the configure block (recommended for Falcon — survives fork replay) +Sourced::CCC.configure do |c| + c.store = Sequel.sqlite('my_app.db') + c.register(CourseDecider) + c.register(EnrolmentDecider) + c.register(CourseCatalogProjector) +end + +# Or separately (fine for scripts and single-process apps) Sourced::CCC.register(CourseDecider) Sourced::CCC.register(EnrolmentDecider) Sourced::CCC.register(CourseCatalogProjector) @@ -321,7 +347,45 @@ This registers the reactor's consumer group with the store and adds it to the gl ## Background processing -The supervisor starts workers that claim partitions, process messages, and ack offsets. +### Falcon (recommended) + +`CCC::Falcon` provides a ready-made Falcon service that runs both the web server and CCC background workers as sibling fibers. No separate worker process needed. + +```ruby +# falcon.rb +#!/usr/bin/env falcon-host +require_relative 'domain' +require_relative 'app' +require 'sourced/ccc/falcon' + +service "my-app" do + include Sourced::CCC::Falcon::Environment + include Falcon::Environment::Rackup + + url "http://localhost:9292" + count 1 +end +``` + +Start with: + +```bash +bundle exec falcon host +``` + +The service automatically calls `CCC.setup!` in each forked process, which replays the `CCC.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe — unlike Postgres, where Sequel reconnects lazily after fork. + +This is why registering reactors inside the `configure` block is recommended when using Falcon: registrations made via `CCC.register` outside the block won't be replayed after fork. + +#### How it works + +- `CCC::Falcon::Environment` — mixin that sets the `service_class` to `CCC::Falcon::Service`. Include it in your Falcon service definition alongside `Falcon::Environment::Rackup`. +- `CCC::Falcon::Service` — extends `Falcon::Service::Server`. On `run`, it calls `CCC.setup!`, starts the web server, and spawns a `CCC::Dispatcher` with all settings from `CCC.config`. On `stop`, it shuts down the dispatcher before the server. +- No separate HouseKeeper fibers are needed — the `StaleClaimReaper` is embedded in the CCC Dispatcher. + +### Supervisor (standalone) + +For running workers without a web server, the supervisor starts workers that claim partitions, process messages, and ack offsets. ```ruby # Start blocking (handles INT/TERM signals for graceful shutdown) diff --git a/lib/sourced/ccc/falcon/environment.rb b/lib/sourced/ccc/falcon/environment.rb new file mode 100644 index 00000000..e22f7f40 --- /dev/null +++ b/lib/sourced/ccc/falcon/environment.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sourced + module CCC + module Falcon + # Environment mixin for configuring a combined Falcon web server + CCC workers service. + # + # Include this module in a Falcon service definition to get CCC worker defaults + # alongside the standard Falcon server environment. All settings are read from + # {CCC.config} — no per-service config methods needed. + # + # The Service automatically calls {CCC.setup!} at the start of +run+ to + # re-establish database connections after Falcon forks (SQLite connections + # are not fork-safe). This replays the block passed to {CCC.configure}. + # + # @example falcon.rb + # #!/usr/bin/env falcon-host + # require 'sourced/ccc/falcon' + # require_relative 'config/environment' + # + # service "my-app" do + # include Sourced::CCC::Falcon::Environment + # include Falcon::Environment::Rackup + # + # url "http://[::]:9292" + # end + module Environment + include ::Falcon::Environment::Server + + def service_class = CCC::Falcon::Service + end + end + end +end diff --git a/lib/sourced/ccc/falcon/service.rb b/lib/sourced/ccc/falcon/service.rb new file mode 100644 index 00000000..d2116082 --- /dev/null +++ b/lib/sourced/ccc/falcon/service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'sourced/ccc/dispatcher' + +module Sourced + module CCC + module Falcon + # A Falcon service that runs both the web server and CCC background workers + # as sibling fibers within the same Async reactor. + # + # Uses a CCC::Dispatcher for signal-driven worker dispatch. The Dispatcher + # already embeds the StaleClaimReaper, so no separate HouseKeeper is needed + # (unlike {Sourced::Falcon::Service}). + # + # All configuration is read from {CCC.config}. + class Service < ::Falcon::Service::Server + def run(instance, evaluator) + CCC.setup! + + server = evaluator.make_server(@bound_endpoint) + + config = CCC.config + @dispatcher = CCC::Dispatcher.new( + router: CCC.router, + worker_count: config.worker_count, + batch_size: config.batch_size, + max_drain_rounds: config.max_drain_rounds, + catchup_interval: config.catchup_interval, + housekeeping_interval: config.housekeeping_interval, + claim_ttl_seconds: config.claim_ttl_seconds, + logger: config.logger + ) + + Async do |task| + server.run + @dispatcher.spawn_into(task) + task.children.each(&:wait) + end + + server + end + + def stop(...) + @dispatcher&.stop + super + end + end + end + end +end From 0f499d17646adc17daad60b684aeef9c83a32193 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 15:15:16 +0000 Subject: [PATCH 047/115] Remove CCC::Configuration#register --- lib/sourced/ccc/README.md | 30 +------------------ lib/sourced/ccc/configuration.rb | 8 ------ spec/sourced/ccc/configuration_spec.rb | 40 -------------------------- 3 files changed, 1 insertion(+), 77 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 0dc2518a..097bd469 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -246,10 +246,6 @@ Sourced::CCC.configure do |c| # Pass a Sequel SQLite connection or a CCC::Store instance c.store = Sequel.sqlite('my_app.db') - # Register reactors (recommended when using Falcon — see below) - c.register(CourseDecider) - c.register(CourseCatalogProjector) - # Optional settings c.worker_count = 4 # background worker fibers (default: 2) c.batch_size = 50 # messages per claim (default: 50) @@ -260,17 +256,6 @@ Sourced::CCC.configure do |c| end ``` -### `CCC.setup!` - -`CCC.configure` stores its block and runs it immediately. You can re-run it later with `CCC.setup!` to create a fresh `Configuration` with new database connections. This is useful after process forks (e.g. Falcon), where SQLite connections become invalid. - -```ruby -# Re-run the configure block (creates fresh store, router, and registrations) -Sourced::CCC.setup! -``` - -`CCC::Falcon::Service` calls `setup!` automatically — you don't need to call it yourself when using the Falcon integration. - ## Failure handling and retries CCC already supports consumer-group retries on failure. @@ -326,18 +311,7 @@ After the configured retries are exhausted, the consumer group is marked as fail ## Registering reactors -Reactors can be registered inside the `configure` block or separately via `CCC.register`. - ```ruby -# Inside the configure block (recommended for Falcon — survives fork replay) -Sourced::CCC.configure do |c| - c.store = Sequel.sqlite('my_app.db') - c.register(CourseDecider) - c.register(EnrolmentDecider) - c.register(CourseCatalogProjector) -end - -# Or separately (fine for scripts and single-process apps) Sourced::CCC.register(CourseDecider) Sourced::CCC.register(EnrolmentDecider) Sourced::CCC.register(CourseCatalogProjector) @@ -373,9 +347,7 @@ Start with: bundle exec falcon host ``` -The service automatically calls `CCC.setup!` in each forked process, which replays the `CCC.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe — unlike Postgres, where Sequel reconnects lazily after fork. - -This is why registering reactors inside the `configure` block is recommended when using Falcon: registrations made via `CCC.register` outside the block won't be replayed after fork. +The service automatically calls `CCC.setup!` in each forked process, which replays the `CCC.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe. #### How it works diff --git a/lib/sourced/ccc/configuration.rb b/lib/sourced/ccc/configuration.rb index d76a8c78..fbe5ea34 100644 --- a/lib/sourced/ccc/configuration.rb +++ b/lib/sourced/ccc/configuration.rb @@ -35,7 +35,6 @@ def initialize @store = nil @router = nil @error_strategy = nil - @reactors = [] @setup = false end @@ -61,19 +60,12 @@ def error_strategy @error_strategy || Sourced.config.error_strategy end - # Buffer a reactor class for registration during {#setup!}. - # @param reactor [Class] a CCC reactor class - def register(reactor) - @reactors << reactor - end - def setup! return if @setup @store ||= Store.new(Sequel.sqlite) @store.install! @router ||= Router.new(store: @store) - @reactors.each { |r| @router.register(r) } @setup = true end end diff --git a/spec/sourced/ccc/configuration_spec.rb b/spec/sourced/ccc/configuration_spec.rb index f576cec8..56272a60 100644 --- a/spec/sourced/ccc/configuration_spec.rb +++ b/spec/sourced/ccc/configuration_spec.rb @@ -120,18 +120,6 @@ def self.name = 'SetupTestReactor' expect(store2).not_to be(store1) end - it 'replays registrations from the configure block' do - Sourced::CCC.configure do |c| - c.register(reactor_class) - end - - expect(Sourced::CCC.router.reactors).to include(reactor_class) - - Sourced::CCC.setup! - - expect(Sourced::CCC.router.reactors).to include(reactor_class) - end - it 'works without a configure block' do Sourced::CCC.setup! @@ -217,34 +205,6 @@ def self.name = 'SetupTestReactor' end end - describe '#register' do - let(:reactor_class) do - Class.new(Sourced::CCC::Projector::StateStored) do - def self.name = 'ConfigRegisterReactor' - - consumer_group 'config-register-reactor' - partition_by :thing_id - - state { |_| {} } - end - end - - it 'buffers reactors and registers them during setup!' do - config = described_class.new - config.register(reactor_class) - config.setup! - - expect(config.router.reactors).to include(reactor_class) - end - - it 'does not register before setup! is called' do - config = described_class.new - config.register(reactor_class) - - expect(config.router).to be_nil - end - end - describe '#setup!' do it 'is idempotent' do config = described_class.new From b0fd092ce0e81f11939d170a3abec33f8d8fe5fc Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 15:40:46 +0000 Subject: [PATCH 048/115] Support Sequel migrations, install store tables from migration --- lib/sourced/ccc.rb | 1 + lib/sourced/ccc/installer.rb | 80 ++++++ .../migrations/001_create_ccc_tables.rb.erb | 89 ++++++ lib/sourced/ccc/store.rb | 255 +++++++----------- spec/sourced/ccc/dispatcher_spec.rb | 2 +- spec/sourced/ccc/handle_spec.rb | 4 +- spec/sourced/ccc/router_spec.rb | 6 +- spec/sourced/ccc/stale_claim_reaper_spec.rb | 16 +- spec/sourced/ccc/store_spec.rb | 95 +++---- 9 files changed, 331 insertions(+), 217 deletions(-) create mode 100644 lib/sourced/ccc/installer.rb create mode 100644 lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index fba02d82..c146602d 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -223,6 +223,7 @@ def self.load(reactor_class, store: nil, **partition_attrs) require 'sourced/ccc/configuration' require 'sourced/ccc/message' +require 'sourced/ccc/installer' require 'sourced/ccc/store' require 'sourced/ccc/actions' require 'sourced/ccc/consumer' diff --git a/lib/sourced/ccc/installer.rb b/lib/sourced/ccc/installer.rb new file mode 100644 index 00000000..60707c38 --- /dev/null +++ b/lib/sourced/ccc/installer.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'sequel' +require 'sequel/extensions/migration' +require 'erb' + +module Sourced + module CCC + class Installer + TABLE_SUFFIXES = %i[messages key_pairs message_key_pairs scheduled_messages consumer_groups offsets offset_key_pairs workers].freeze + + attr_reader :messages_table, :key_pairs_table, :message_key_pairs_table, + :scheduled_messages_table, :consumer_groups_table, :offsets_table, + :offset_key_pairs_table, :workers_table + + def initialize(db, logger:, prefix: 'sourced', migration_template: '001_create_ccc_tables.rb.erb') + raise ArgumentError, "invalid prefix: #{prefix}" unless prefix.match?(/\A[a-zA-Z_]\w*\z/) + + @db = db + @logger = logger + @prefix = prefix + @migration_template = migration_template + + TABLE_SUFFIXES.each do |suffix| + instance_variable_set(:"@#{suffix}_table", :"#{prefix}_#{suffix}") + end + end + + # Eval the rendered migration and apply :up directly. + def install + migration.apply(db, :up) + logger.info("CCC tables installed (prefix: #{prefix})") + end + + # Check that all expected tables exist. + def installed? + all_table_names.all? { |t| db.table_exists?(t) } + end + + # Apply :down on the migration to drop tables. + def uninstall + raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test' + + migration.apply(db, :down) + end + + # Render the migration to a file for use with the host app's Sequel::Migrator. + # + # installer.copy_migration_to("db/migrations") + # installer.copy_migration_to { "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_ccc_tables.rb" } + # + def copy_migration_to(dir = nil, &block) + path = block ? block.call : File.join(dir, '001_create_ccc_tables.rb') + File.write(path, rendered_migration) + logger.info("Copied CCC migration to #{path}") + path + end + + private + + attr_reader :db, :logger, :prefix + + def migration + @migration ||= eval(rendered_migration) # rubocop:disable Security/Eval + end + + def rendered_migration + @rendered_migration ||= begin + template_path = File.join(__dir__, 'migrations', @migration_template) + ERB.new(File.read(template_path)).result(binding) + end + end + + # Returns actual symbols for use in installed? checks. + def all_table_names + TABLE_SUFFIXES.map { |suffix| instance_variable_get(:"@#{suffix}_table") } + end + end + end +end diff --git a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb new file mode 100644 index 00000000..758d452d --- /dev/null +++ b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +Sequel.migration do + up do + create_table?(:<%= messages_table %>) do + primary_key :position + String :message_id, null: false, unique: true + String :message_type, null: false + String :causation_id + String :correlation_id + String :payload, null: false + String :metadata + String :created_at, null: false + + index :message_type, name: 'idx_<%= prefix %>_ccc_message_type' + index :correlation_id, name: 'idx_<%= prefix %>_ccc_correlation_id' + end + + create_table?(:<%= key_pairs_table %>) do + primary_key :id + String :name, null: false + String :value, null: false + + index %i[name value], unique: true, name: 'idx_<%= prefix %>_ccc_key_pair_nv' + end + + create_table?(:<%= message_key_pairs_table %>) do + foreign_key :message_position, :<%= messages_table %>, key: :position + foreign_key :key_pair_id, :<%= key_pairs_table %> + primary_key %i[message_position key_pair_id] + + index %i[key_pair_id message_position], name: 'idx_<%= prefix %>_ccc_mkp_key' + end + + create_table?(:<%= scheduled_messages_table %>) do + primary_key :id + String :created_at, null: false + String :available_at, null: false + String :message, null: false + + index :available_at, name: 'idx_<%= prefix %>_ccc_scheduled_available_at' + end + + create_table?(:<%= consumer_groups_table %>) do + primary_key :id + String :group_id, null: false, unique: true + String :status, null: false, default: 'active' + Integer :highest_position, null: false, default: 0 + String :error_context + String :retry_at + String :created_at, null: false + String :updated_at, null: false + end + + create_table?(:<%= offsets_table %>) do + primary_key :id + foreign_key :consumer_group_id, :<%= consumer_groups_table %>, on_delete: :cascade + String :partition_key, null: false + Integer :last_position, null: false, default: 0 + Integer :claimed, null: false, default: 0 + String :claimed_at + String :claimed_by + + index %i[consumer_group_id partition_key], unique: true, name: 'idx_<%= prefix %>_ccc_offsets_cg_pk' + end + + create_table?(:<%= offset_key_pairs_table %>) do + foreign_key :offset_id, :<%= offsets_table %>, on_delete: :cascade + foreign_key :key_pair_id, :<%= key_pairs_table %> + primary_key %i[offset_id key_pair_id] + end + + create_table?(:<%= workers_table %>) do + String :id, primary_key: true, null: false + String :last_seen, null: false + end + end + + down do + drop_table?(:<%= offset_key_pairs_table %>) + drop_table?(:<%= offsets_table %>) + drop_table?(:<%= consumer_groups_table %>) + drop_table?(:<%= scheduled_messages_table %>) + drop_table?(:<%= message_key_pairs_table %>) + drop_table?(:<%= key_pairs_table %>) + drop_table?(:<%= messages_table %>) + drop_table?(:<%= workers_table %>) + end +end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index a908ebfb..13698bf3 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -2,6 +2,7 @@ require 'json' require 'sourced/inline_notifier' +require 'sourced/ccc/installer' module Sourced module CCC @@ -52,118 +53,56 @@ class Store # @return [Logger] attr_reader :logger + # @return [CCC::Installer] + attr_reader :installer + # @param db [Sequel::SQLite::Database] a Sequel SQLite connection # @param notifier [#notify_new_messages, #notify_reactor_resumed, nil] optional notifier for dispatch signals # @param logger [Logger, nil] optional logger (defaults to Sourced.config.logger) - def initialize(db, notifier: nil, logger: nil) + # @param prefix [String] table name prefix (default 'sourced') + def initialize(db, notifier: nil, logger: nil, prefix: 'sourced') @db = db @notifier = notifier || Sourced::InlineNotifier.new @logger = logger || Sourced.config.logger @db.run('PRAGMA foreign_keys = ON') @db.run('PRAGMA journal_mode = WAL') @db.run('PRAGMA busy_timeout = 5000') + + @installer = Installer.new(db, logger: @logger, prefix: prefix) + + # Source table name symbols from the installer + @messages_table = @installer.messages_table + @key_pairs_table = @installer.key_pairs_table + @message_key_pairs_table = @installer.message_key_pairs_table + @scheduled_messages_table = @installer.scheduled_messages_table + @consumer_groups_table = @installer.consumer_groups_table + @offsets_table = @installer.offsets_table + @offset_key_pairs_table = @installer.offset_key_pairs_table + @workers_table = @installer.workers_table end # Whether all required tables exist. # @return [Boolean] def installed? - db.table_exists?(:ccc_messages) && - db.table_exists?(:ccc_key_pairs) && - db.table_exists?(:ccc_message_key_pairs) && - db.table_exists?(:ccc_scheduled_messages) && - db.table_exists?(:ccc_consumer_groups) && - db.table_exists?(:ccc_offsets) && - db.table_exists?(:ccc_offset_key_pairs) && - db.table_exists?(:ccc_workers) + installer.installed? end # Create all required tables and indexes. Idempotent. # @return [void] def install! - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_messages ( - position INTEGER PRIMARY KEY AUTOINCREMENT, - message_id TEXT NOT NULL UNIQUE, - message_type TEXT NOT NULL, - causation_id TEXT, - correlation_id TEXT, - payload TEXT NOT NULL, - metadata TEXT, - created_at TEXT NOT NULL - ) - SQL - db.run('CREATE INDEX IF NOT EXISTS idx_ccc_message_type ON ccc_messages(message_type)') - db.run('CREATE INDEX IF NOT EXISTS idx_ccc_correlation_id ON ccc_messages(correlation_id)') - - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_key_pairs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - value TEXT NOT NULL, - UNIQUE(name, value) - ) - SQL - db.run('CREATE INDEX IF NOT EXISTS idx_ccc_key_pair_nv ON ccc_key_pairs(name, value)') - - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_message_key_pairs ( - message_position INTEGER NOT NULL REFERENCES ccc_messages(position), - key_pair_id INTEGER NOT NULL REFERENCES ccc_key_pairs(id), - PRIMARY KEY (message_position, key_pair_id) - ) - SQL - db.run('CREATE INDEX IF NOT EXISTS idx_ccc_mkp_key ON ccc_message_key_pairs(key_pair_id, message_position)') - - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_scheduled_messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL, - available_at TEXT NOT NULL, - message TEXT NOT NULL - ) - SQL - db.run('CREATE INDEX IF NOT EXISTS idx_ccc_scheduled_available_at ON ccc_scheduled_messages(available_at)') - - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_consumer_groups ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - group_id TEXT NOT NULL UNIQUE, - status TEXT NOT NULL DEFAULT '#{ACTIVE}', - highest_position INTEGER NOT NULL DEFAULT 0, - error_context TEXT, - retry_at TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ) - SQL - - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_offsets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - consumer_group_id INTEGER NOT NULL REFERENCES ccc_consumer_groups(id) ON DELETE CASCADE, - partition_key TEXT NOT NULL, - last_position INTEGER NOT NULL DEFAULT 0, - claimed INTEGER NOT NULL DEFAULT 0, - claimed_at TEXT, - claimed_by TEXT, - UNIQUE(consumer_group_id, partition_key) - ) - SQL + installer.install + end - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_offset_key_pairs ( - offset_id INTEGER NOT NULL REFERENCES ccc_offsets(id) ON DELETE CASCADE, - key_pair_id INTEGER NOT NULL REFERENCES ccc_key_pairs(id), - PRIMARY KEY (offset_id, key_pair_id) - ) - SQL + # Drop all tables. Test-only guard. + # @return [void] + def uninstall + installer.uninstall + end - db.run(<<~SQL) - CREATE TABLE IF NOT EXISTS ccc_workers ( - id TEXT PRIMARY KEY, - last_seen TEXT NOT NULL - ) - SQL + # Render the migration to a file for use with the host app's Sequel::Migrator. + # @see Installer#copy_migration_to + def copy_migration_to(dir = nil, &block) + installer.copy_migration_to(dir, &block) end # Append messages to the store. Extracts and indexes key-value pairs @@ -192,7 +131,7 @@ def append(messages, guard: nil) payload_json = msg.payload ? JSON.dump(msg.payload.to_h) : '{}' metadata_json = msg.metadata.empty? ? nil : JSON.dump(msg.metadata) - db[:ccc_messages].insert( + db[@messages_table].insert( message_id: msg.id, message_type: msg.type, causation_id: msg.causation_id, @@ -202,14 +141,14 @@ def append(messages, guard: nil) created_at: msg.created_at.iso8601 ) - last_position = db[:ccc_messages].where(message_id: msg.id).get(:position) + last_position = db[@messages_table].where(message_id: msg.id).get(:position) # Extract and index key pairs msg.extracted_keys.each do |name, value| - db.run("INSERT OR IGNORE INTO ccc_key_pairs (name, value) VALUES (#{db.literal(name)}, #{db.literal(value)})") - key_pair_id = db[:ccc_key_pairs].where(name: name, value: value).get(:id) + db.run("INSERT OR IGNORE INTO #{@key_pairs_table} (name, value) VALUES (#{db.literal(name)}, #{db.literal(value)})") + key_pair_id = db[@key_pairs_table].where(name: name, value: value).get(:id) - db[:ccc_message_key_pairs].insert( + db[@message_key_pairs_table].insert( message_position: last_position, key_pair_id: key_pair_id ) @@ -243,7 +182,7 @@ def schedule_messages(messages, at:) end db.transaction do - db[:ccc_scheduled_messages].multi_insert(rows) + db[@scheduled_messages_table].multi_insert(rows) end true @@ -259,7 +198,7 @@ def update_schedule! now = Time.now db.transaction do - rows = db[:ccc_scheduled_messages] + rows = db[@scheduled_messages_table] .where { available_at <= now.iso8601 } .order(:id) .limit(100) @@ -276,7 +215,7 @@ def update_schedule! append(messages) row_ids = rows.map { |row| row[:id] } - db[:ccc_scheduled_messages].where(id: row_ids).delete + db[@scheduled_messages_table].where(id: row_ids).delete rows.size end @@ -294,7 +233,7 @@ def update_schedule! # @param limit [Integer] max number of messages to return (default 50) # @return [Array] messages ordered by position def read_all(from_position: 0, limit: 50) - db[:ccc_messages] + db[@messages_table] .where { position > from_position } .order(:position) .limit(limit) @@ -367,7 +306,7 @@ def read(conditions, from_position: nil, limit: nil) def read_partition(partition_attrs, handled_types:, from_position: 0) # Resolve key_pair_ids for each partition attribute key_pair_ids = partition_attrs.filter_map do |name, value| - db[:ccc_key_pairs].where(name: name.to_s, value: value.to_s).get(:id) + db[@key_pairs_table].where(name: name.to_s, value: value.to_s).get(:id) end # If any key pair doesn't exist in the store, no messages can match @@ -413,7 +352,7 @@ def messages_since(conditions, position) def register_consumer_group(group_id) now = Time.now.iso8601 db.run(<<~SQL) - INSERT OR IGNORE INTO ccc_consumer_groups (group_id, status, highest_position, created_at, updated_at) + INSERT OR IGNORE INTO #{@consumer_groups_table} (group_id, status, highest_position, created_at, updated_at) VALUES (#{db.literal(group_id)}, '#{ACTIVE}', 0, #{db.literal(now)}, #{db.literal(now)}) SQL end @@ -424,7 +363,7 @@ def register_consumer_group(group_id) # @return [Boolean] def consumer_group_active?(group_id) group_id = resolve_group_id(group_id) - row = db[:ccc_consumer_groups].where(group_id: group_id).select(:status).first + row = db[@consumer_groups_table].where(group_id: group_id).select(:status).first return false unless row row[:status] == ACTIVE @@ -448,7 +387,7 @@ def stop_consumer_group(group_id, message = nil) # @return [void] def start_consumer_group(group_id) group_id = resolve_group_id(group_id) - db[:ccc_consumer_groups] + db[@consumer_groups_table] .where(group_id: group_id) .update(status: ACTIVE, retry_at: nil, error_context: nil, updated_at: Time.now.iso8601) notifier.notify_reactor_resumed(group_id) @@ -462,7 +401,7 @@ def start_consumer_group(group_id) # @yieldparam group [CCC::GroupUpdater] # @return [void] def updating_consumer_group(group_id) - dataset = db[:ccc_consumer_groups].where(group_id: group_id) + dataset = db[@consumer_groups_table].where(group_id: group_id) row = dataset.first raise ArgumentError, "Consumer group #{group_id} not found" unless row @@ -483,10 +422,10 @@ def updating_consumer_group(group_id) # @return [void] def reset_consumer_group(group_id) group_id = resolve_group_id(group_id) - cg = db[:ccc_consumer_groups].where(group_id: group_id).first + cg = db[@consumer_groups_table].where(group_id: group_id).first return unless cg - db[:ccc_offsets].where(consumer_group_id: cg[:id]).delete + db[@offsets_table].where(consumer_group_id: cg[:id]).delete end # Claim the next available partition for processing. @@ -513,7 +452,7 @@ def reset_consumer_group(group_id) def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: nil) partition_by = Array(partition_by).sort now = Time.now.iso8601 - cg = db[:ccc_consumer_groups] + cg = db[@consumer_groups_table] .where(group_id: group_id, status: ACTIVE) .where { Sequel.|({retry_at: nil}, Sequel.lit('retry_at <= ?', now)) } .first @@ -524,7 +463,7 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) return nil unless claimed - key_pair_ids = db[:ccc_offset_key_pairs] + key_pair_ids = db[@offset_key_pairs_table] .where(offset_id: claimed[:offset_id]) .select_map(:key_pair_id) @@ -538,7 +477,7 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: # Build partition_value hash from key_pairs partition_value = {} - db[:ccc_key_pairs].where(id: key_pair_ids).each do |kp| + db[@key_pairs_table].where(id: key_pair_ids).each do |kp| partition_value[kp[:name]] = kp[:value] end @@ -579,10 +518,10 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: # @param position [Integer] position of the last processed message # @return [void] def ack(group_id, offset_id:, position:) - cg = db[:ccc_consumer_groups].where(group_id: group_id).first + cg = db[@consumer_groups_table].where(group_id: group_id).first return unless cg - db[:ccc_offsets].where(id: offset_id, consumer_group_id: cg[:id]).update( + db[@offsets_table].where(id: offset_id, consumer_group_id: cg[:id]).update( last_position: position, claimed: 0, claimed_at: nil, @@ -591,7 +530,7 @@ def ack(group_id, offset_id:, position:) # Advance the high watermark (never decrease) if position > cg[:highest_position] - db[:ccc_consumer_groups].where(id: cg[:id]).update( + db[@consumer_groups_table].where(id: cg[:id]).update( highest_position: position, updated_at: Time.now.iso8601 ) @@ -605,10 +544,10 @@ def ack(group_id, offset_id:, position:) # @param offset_id [Integer] offset ID from the claim result # @return [void] def release(group_id, offset_id:) - cg = db[:ccc_consumer_groups].where(group_id: group_id).first + cg = db[@consumer_groups_table].where(group_id: group_id).first return unless cg - db[:ccc_offsets].where(id: offset_id, consumer_group_id: cg[:id]).update( + db[@offsets_table].where(id: offset_id, consumer_group_id: cg[:id]).update( claimed: 0, claimed_at: nil, claimed_by: nil @@ -627,7 +566,7 @@ def worker_heartbeat(worker_ids, at: Time.now) now = at.iso8601 ids.each do |id| db.run(<<~SQL) - INSERT INTO ccc_workers (id, last_seen) VALUES (#{db.literal(id)}, #{db.literal(now)}) + INSERT INTO #{@workers_table} (id, last_seen) VALUES (#{db.literal(id)}, #{db.literal(now)}) ON CONFLICT(id) DO UPDATE SET last_seen = #{db.literal(now)} SQL end @@ -641,13 +580,13 @@ def worker_heartbeat(worker_ids, at: Time.now) def release_stale_claims(ttl_seconds: 120) cutoff = (Time.now - ttl_seconds).iso8601 - stale_worker_ids = db[:ccc_workers] + stale_worker_ids = db[@workers_table] .where(Sequel.lit('last_seen <= ?', cutoff)) .select_map(:id) return 0 if stale_worker_ids.empty? - db[:ccc_offsets] + db[@offsets_table] .where(claimed: 1) .where(claimed_by: stale_worker_ids) .update(claimed: 0, claimed_at: nil, claimed_by: nil) @@ -662,21 +601,21 @@ def release_stale_claims(ttl_seconds: 120) # @param position [Integer] advance offset to at least this position # @return [void] def advance_offset(group_id, partition:, position:) - cg = db[:ccc_consumer_groups].where(group_id: group_id).first + cg = db[@consumer_groups_table].where(group_id: group_id).first return unless cg partition_by = partition.keys.sort bootstrap_offsets(cg[:id], partition_by) partition_key = build_partition_key(partition_by, partition) - offset = db[:ccc_offsets].where(consumer_group_id: cg[:id], partition_key: partition_key).first + offset = db[@offsets_table].where(consumer_group_id: cg[:id], partition_key: partition_key).first return unless offset return if offset[:last_position] >= position - db[:ccc_offsets].where(id: offset[:id]).update(last_position: position) + db[@offsets_table].where(id: offset[:id]).update(last_position: position) if position > cg[:highest_position] - db[:ccc_consumer_groups].where(id: cg[:id]).update( + db[@consumer_groups_table].where(id: cg[:id]).update( highest_position: position, updated_at: Time.now.iso8601 ) @@ -721,8 +660,8 @@ def stats COALESCE(MIN(CASE WHEN o.last_position > 0 THEN o.last_position END), 0) AS oldest_processed, COALESCE(MAX(o.last_position), 0) AS newest_processed, COUNT(o.id) AS partition_count - FROM ccc_consumer_groups cg - LEFT JOIN ccc_offsets o ON o.consumer_group_id = cg.id + FROM #{@consumer_groups_table} cg + LEFT JOIN #{@offsets_table} o ON o.consumer_group_id = cg.id GROUP BY cg.id, cg.group_id, cg.status, cg.retry_at, cg.error_context ORDER BY cg.group_id SQL @@ -741,12 +680,12 @@ def stats # @param message_id [String] UUID of any message in the correlation chain # @return [Array] correlated messages ordered by position, or [] if not found def read_correlation_batch(message_id) - correlation_id = db[:ccc_messages] + correlation_id = db[@messages_table] .where(message_id: message_id) .get(:correlation_id) return [] unless correlation_id - db[:ccc_messages] + db[@messages_table] .where(correlation_id: correlation_id) .order(:position) .map { |row| deserialize(row) } @@ -756,21 +695,21 @@ def read_correlation_batch(message_id) # # @return [Integer] max position, or 0 if the store is empty def latest_position - db[:ccc_messages].max(:position) || 0 + db[@messages_table].max(:position) || 0 end # Delete all data from all tables and reset autoincrement. For testing only. # # @return [void] def clear! - db[:ccc_offset_key_pairs].delete - db[:ccc_offsets].delete - db[:ccc_consumer_groups].delete - db[:ccc_message_key_pairs].delete - db[:ccc_key_pairs].delete - db[:ccc_messages].delete - db[:ccc_scheduled_messages].delete - db[:ccc_workers].delete + db[@offset_key_pairs_table].delete + db[@offsets_table].delete + db[@consumer_groups_table].delete + db[@message_key_pairs_table].delete + db[@key_pairs_table].delete + db[@messages_table].delete + db[@scheduled_messages_table].delete + db[@workers_table].delete db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) end @@ -806,8 +745,8 @@ def bootstrap_offsets(cg_id, partition_by) joins = [] selects = [] partition_by.each_with_index do |attr, i| - joins << "JOIN ccc_message_key_pairs mkp#{i} ON m.position = mkp#{i}.message_position" - joins << "JOIN ccc_key_pairs kp#{i} ON mkp#{i}.key_pair_id = kp#{i}.id AND kp#{i}.name = #{db.literal(attr)}" + joins << "JOIN #{@message_key_pairs_table} mkp#{i} ON m.position = mkp#{i}.message_position" + joins << "JOIN #{@key_pairs_table} kp#{i} ON mkp#{i}.key_pair_id = kp#{i}.id AND kp#{i}.name = #{db.literal(attr)}" selects << "kp#{i}.id AS kp_id_#{i}, kp#{i}.value AS val_#{i}" end @@ -815,7 +754,7 @@ def bootstrap_offsets(cg_id, partition_by) sql = <<~SQL SELECT #{selects.join(', ')} - FROM ccc_messages m + FROM #{@messages_table} m #{joins.join("\n")} GROUP BY #{group_by} SQL @@ -833,16 +772,16 @@ def bootstrap_offsets(cg_id, partition_by) # INSERT OR IGNORE the offset row db.run(<<~SQL) - INSERT OR IGNORE INTO ccc_offsets (consumer_group_id, partition_key, last_position, claimed) + INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) SQL - offset_id = db[:ccc_offsets].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) # INSERT OR IGNORE the offset_key_pairs join rows kp_ids.each do |kp_id| db.run(<<~SQL) - INSERT OR IGNORE INTO ccc_offset_key_pairs (offset_id, key_pair_id) + INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) SQL end @@ -863,10 +802,10 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) sql = <<~SQL SELECT o.id AS offset_id, o.partition_key, o.last_position, MIN(m.position) AS next_position - FROM ccc_offsets o - JOIN ccc_offset_key_pairs okp ON o.id = okp.offset_id - JOIN ccc_message_key_pairs mkp ON okp.key_pair_id = mkp.key_pair_id - JOIN ccc_messages m ON mkp.message_position = m.position + FROM #{@offsets_table} o + JOIN #{@offset_key_pairs_table} okp ON o.id = okp.offset_id + JOIN #{@message_key_pairs_table} mkp ON okp.key_pair_id = mkp.key_pair_id + JOIN #{@messages_table} m ON mkp.message_position = m.position WHERE o.consumer_group_id = #{db.literal(cg_id)} AND o.claimed = 0 AND m.position > o.last_position @@ -880,7 +819,7 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) return nil unless row now = Time.now.iso8601 - updated = db[:ccc_offsets] + updated = db[@offsets_table] .where(id: row[:offset_id], claimed: 0) .update(claimed: 1, claimed_at: now, claimed_by: worker_id) @@ -907,23 +846,23 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: sql = <<~SQL SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at - FROM ccc_messages m + FROM #{@messages_table} m WHERE m.position > #{db.literal(last_position)} AND m.message_type IN (#{types_list}) AND EXISTS ( - SELECT 1 FROM ccc_message_key_pairs mkp + SELECT 1 FROM #{@message_key_pairs_table} mkp WHERE mkp.message_position = m.position AND mkp.key_pair_id IN (#{kp_ids_list}) ) AND ( - SELECT COUNT(*) FROM ccc_message_key_pairs mkp + SELECT COUNT(*) FROM #{@message_key_pairs_table} mkp WHERE mkp.message_position = m.position AND mkp.key_pair_id IN (#{kp_ids_list}) ) = ( SELECT COUNT(DISTINCT kp_part.name) - FROM ccc_message_key_pairs mkp2 - JOIN ccc_key_pairs kp_msg ON mkp2.key_pair_id = kp_msg.id - JOIN ccc_key_pairs kp_part ON kp_part.id IN (#{kp_ids_list}) + FROM #{@message_key_pairs_table} mkp2 + JOIN #{@key_pairs_table} kp_msg ON mkp2.key_pair_id = kp_msg.id + JOIN #{@key_pairs_table} kp_part ON kp_part.id IN (#{kp_ids_list}) AND kp_part.name = kp_msg.name WHERE mkp2.message_position = m.position ) @@ -945,7 +884,7 @@ def query_messages(conditions, from_position: nil, limit: nil) # Step 1: resolve key_pair IDs key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } - key_rows = db.fetch("SELECT id, name, value FROM ccc_key_pairs WHERE #{or_clauses.join(' OR ')}").all + key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all key_pair_index = {} key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } @@ -962,8 +901,8 @@ def query_messages(conditions, from_position: nil, limit: nil) sql = <<~SQL SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at - FROM ccc_messages m - JOIN ccc_message_key_pairs mkp ON m.position = mkp.message_position + FROM #{@messages_table} m + JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position WHERE (#{where_parts.join(' OR ')}) SQL @@ -996,7 +935,7 @@ def max_position_for(conditions, from_position: nil) key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } - key_rows = db.fetch("SELECT id, name, value FROM ccc_key_pairs WHERE #{or_clauses.join(' OR ')}").all + key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all key_pair_index = {} key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } @@ -1011,8 +950,8 @@ def max_position_for(conditions, from_position: nil) sql = <<~SQL SELECT MAX(m.position) AS max_pos - FROM ccc_messages m - JOIN ccc_message_key_pairs mkp ON m.position = mkp.message_position + FROM #{@messages_table} m + JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position WHERE (#{where_parts.join(' OR ')}) SQL sql += " AND m.position > #{db.literal(from_position)}" if from_position diff --git a/spec/sourced/ccc/dispatcher_spec.rb b/spec/sourced/ccc/dispatcher_spec.rb index 5d3cb26e..0cd010ed 100644 --- a/spec/sourced/ccc/dispatcher_spec.rb +++ b/spec/sourced/ccc/dispatcher_spec.rb @@ -361,7 +361,7 @@ def task.async; end ) expect(router.handle_next_for(DispatchTestDecider)).to be true - expect(db[:ccc_scheduled_messages].count).to eq(1) + expect(db[:sourced_scheduled_messages].count).to eq(1) Timecop.freeze(Time.now + 3) do expect(store.update_schedule!).to eq(1) diff --git a/spec/sourced/ccc/handle_spec.rb b/spec/sourced/ccc/handle_spec.rb index 128107fb..b20529d3 100644 --- a/spec/sourced/ccc/handle_spec.rb +++ b/spec/sourced/ccc/handle_spec.rb @@ -101,7 +101,7 @@ class HandleTestDecider < Sourced::CCC::Decider _cmd, _reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) # Both command and event should be in the store - all = store.db[:ccc_messages].order(:position).all + all = store.db[:sourced_messages].order(:position).all expect(all.size).to eq(2) expect(all[0][:message_type]).to eq('handle_test.create_device') expect(all[1][:message_type]).to eq('handle_test.device_created') @@ -156,7 +156,7 @@ class HandleTestDecider < Sourced::CCC::Decider expect(events).to eq([]) # Nothing appended - expect(store.db[:ccc_messages].count).to eq(0) + expect(store.db[:sourced_messages].count).to eq(0) end end diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 203dfde1..b62117e6 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -233,7 +233,7 @@ def self.handle_batch(claim) router.handle_next_for(RouterTestDecider) expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false - row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first expect(row[:status]).to eq('failed') end @@ -249,7 +249,7 @@ def self.handle_batch(claim) router.handle_next_for(RouterTestDecider) - row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first expect(row[:error_context]).not_to be_nil expect(row[:status]).to eq('failed') end @@ -271,7 +271,7 @@ def self.handle_batch(claim) router.handle_next_for(RouterTestDecider) - row = db[:ccc_consumer_groups].where(group_id: RouterTestDecider.group_id).first + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first expect(row[:retry_at]).not_to be_nil expect(row[:status]).to eq('active') diff --git a/spec/sourced/ccc/stale_claim_reaper_spec.rb b/spec/sourced/ccc/stale_claim_reaper_spec.rb index a32b1047..ab01e028 100644 --- a/spec/sourced/ccc/stale_claim_reaper_spec.rb +++ b/spec/sourced/ccc/stale_claim_reaper_spec.rb @@ -33,7 +33,7 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored count = store.worker_heartbeat(['w1', 'w2']) expect(count).to eq(2) - rows = db[:ccc_workers].all + rows = db[:sourced_workers].all expect(rows.size).to eq(2) expect(rows.map { |r| r[:id] }).to contain_exactly('w1', 'w2') rows.each { |r| expect(r[:last_seen]).not_to be_nil } @@ -46,14 +46,14 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored later = Time.now store.worker_heartbeat(['w1'], at: later) - row = db[:ccc_workers].where(id: 'w1').first + row = db[:sourced_workers].where(id: 'w1').first expect(row[:last_seen]).to eq(later.iso8601) end it 'deduplicates worker IDs' do count = store.worker_heartbeat(['w1', 'w1', 'w1']) expect(count).to eq(1) - expect(db[:ccc_workers].count).to eq(1) + expect(db[:sourced_workers].count).to eq(1) end it 'returns 0 for empty array' do @@ -88,7 +88,7 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored expect(released).to eq(1) # Verify the offset is unclaimed - offset = db[:ccc_offsets].where(id: @claim.offset_id).first + offset = db[:sourced_offsets].where(id: @claim.offset_id).first expect(offset[:claimed]).to eq(0) expect(offset[:claimed_at]).to be_nil expect(offset[:claimed_by]).to be_nil @@ -102,7 +102,7 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored expect(released).to eq(0) # Verify the offset is still claimed - offset = db[:ccc_offsets].where(id: @claim.offset_id).first + offset = db[:sourced_offsets].where(id: @claim.offset_id).first expect(offset[:claimed]).to eq(1) expect(offset[:claimed_by]).to eq('w1') end @@ -146,11 +146,11 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored # Only w1's claims should be released if claim1 expect(released).to eq(1) - offset1 = db[:ccc_offsets].where(id: claim1.offset_id).first + offset1 = db[:sourced_offsets].where(id: claim1.offset_id).first expect(offset1[:claimed]).to eq(0) end - offset2 = db[:ccc_offsets].where(id: claim2.offset_id).first + offset2 = db[:sourced_offsets].where(id: claim2.offset_id).first expect(offset2[:claimed]).to eq(1) expect(offset2[:claimed_by]).to eq('w2') end @@ -176,7 +176,7 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored reaper.send(:heartbeat) - rows = db[:ccc_workers].all + rows = db[:sourced_workers].all expect(rows.size).to eq(2) expect(rows.map { |r| r[:id] }).to contain_exactly('w-0', 'w-1') end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index bf3bfa04..916fd267 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -62,10 +62,15 @@ module CCCStoreTestMessages end describe '#install!' do - it 'creates the three tables' do - expect(db.table_exists?(:ccc_messages)).to be true - expect(db.table_exists?(:ccc_key_pairs)).to be true - expect(db.table_exists?(:ccc_message_key_pairs)).to be true + it 'creates the tables' do + expect(db.table_exists?(:sourced_messages)).to be true + expect(db.table_exists?(:sourced_key_pairs)).to be true + expect(db.table_exists?(:sourced_message_key_pairs)).to be true + expect(db.table_exists?(:sourced_scheduled_messages)).to be true + expect(db.table_exists?(:sourced_consumer_groups)).to be true + expect(db.table_exists?(:sourced_offsets)).to be true + expect(db.table_exists?(:sourced_offset_key_pairs)).to be true + expect(db.table_exists?(:sourced_workers)).to be true end it 'is idempotent' do @@ -97,13 +102,13 @@ module CCCStoreTestMessages ) store.append(msg) - key_pairs = db[:ccc_key_pairs].all + key_pairs = db[:sourced_key_pairs].all expect(key_pairs.map { |r| [r[:name], r[:value]] }).to contain_exactly( ['device_id', 'dev-1'], ['name', 'Sensor A'] ) - join_rows = db[:ccc_message_key_pairs].all + join_rows = db[:sourced_message_key_pairs].all expect(join_rows.size).to eq(2) end @@ -113,12 +118,12 @@ module CCCStoreTestMessages store.append([msg1, msg2]) # 'device_id'/'dev-1' should exist once in key_pairs - count = db[:ccc_key_pairs].where(name: 'device_id', value: 'dev-1').count + count = db[:sourced_key_pairs].where(name: 'device_id', value: 'dev-1').count expect(count).to eq(1) # But both messages reference it via the join table - kp_id = db[:ccc_key_pairs].where(name: 'device_id', value: 'dev-1').get(:id) - join_count = db[:ccc_message_key_pairs].where(key_pair_id: kp_id).count + kp_id = db[:sourced_key_pairs].where(name: 'device_id', value: 'dev-1').get(:id) + join_count = db[:sourced_message_key_pairs].where(key_pair_id: kp_id).count expect(join_count).to eq(2) end @@ -129,7 +134,7 @@ module CCCStoreTestMessages ) store.append(msg) - row = db[:ccc_messages].first + row = db[:sourced_messages].first meta = JSON.parse(row[:metadata], symbolize_names: true) expect(meta[:user_id]).to eq(42) end @@ -171,7 +176,7 @@ module CCCStoreTestMessages expect(store.schedule_messages([delayed], at: delayed.created_at)).to be true expect(store.latest_position).to eq(0) - expect(db[:ccc_scheduled_messages].count).to eq(1) + expect(db[:sourced_scheduled_messages].count).to eq(1) expect(store.update_schedule!).to eq(0) end @@ -188,7 +193,7 @@ module CCCStoreTestMessages expect(store.update_schedule!).to eq(1) end - expect(db[:ccc_scheduled_messages].count).to eq(0) + expect(db[:sourced_scheduled_messages].count).to eq(0) expect(store.latest_position).to eq(1) cond = Sourced::CCC::QueryCondition.new( @@ -257,7 +262,7 @@ module CCCStoreTestMessages store.append(conflicting) position_before = store.latest_position - count_before = db[:ccc_messages].count + count_before = db[:sourced_messages].count new_msg = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) expect { @@ -265,7 +270,7 @@ module CCCStoreTestMessages }.to raise_error(Sourced::ConcurrentAppendError) expect(store.latest_position).to eq(position_before) - expect(db[:ccc_messages].count).to eq(count_before) + expect(db[:sourced_messages].count).to eq(count_before) end it 'works with a manually constructed guard' do @@ -549,9 +554,9 @@ module CCCStoreTestMessages store.clear! expect(store.latest_position).to eq(0) - expect(db[:ccc_messages].count).to eq(0) - expect(db[:ccc_key_pairs].count).to eq(0) - expect(db[:ccc_message_key_pairs].count).to eq(0) + expect(db[:sourced_messages].count).to eq(0) + expect(db[:sourced_key_pairs].count).to eq(0) + expect(db[:sourced_message_key_pairs].count).to eq(0) end it 'resets autoincrement so next position starts from 1' do @@ -578,16 +583,16 @@ module CCCStoreTestMessages store.clear! - expect(db[:ccc_consumer_groups].count).to eq(0) - expect(db[:ccc_offsets].count).to eq(0) - expect(db[:ccc_offset_key_pairs].count).to eq(0) + expect(db[:sourced_consumer_groups].count).to eq(0) + expect(db[:sourced_offsets].count).to eq(0) + expect(db[:sourced_offset_key_pairs].count).to eq(0) end end describe '#register_consumer_group' do it 'creates row with active status' do store.register_consumer_group('my-group') - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first expect(row).not_to be_nil expect(row[:status]).to eq('active') expect(row[:created_at]).not_to be_nil @@ -597,7 +602,7 @@ module CCCStoreTestMessages it 'is idempotent' do store.register_consumer_group('my-group') expect { store.register_consumer_group('my-group') }.not_to raise_error - expect(db[:ccc_consumer_groups].where(group_id: 'my-group').count).to eq(1) + expect(db[:sourced_consumer_groups].where(group_id: 'my-group').count).to eq(1) end end @@ -653,13 +658,13 @@ module CCCStoreTestMessages group.retry(Time.now + 60, retry_count: 1) end - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first expect(row[:retry_at]).not_to be_nil expect(row[:error_context]).not_to be_nil store.start_consumer_group('my-group') - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first expect(row[:retry_at]).to be_nil expect(row[:error_context]).to be_nil expect(row[:status]).to eq('active') @@ -677,7 +682,7 @@ module CCCStoreTestMessages group.retry(Time.now + 30, retry_count: 1) end - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first ctx = JSON.parse(row[:error_context], symbolize_names: true) expect(ctx[:retry_count]).to eq(1) expect(row[:retry_at]).not_to be_nil @@ -693,7 +698,7 @@ module CCCStoreTestMessages group.retry(Time.now + 60, retry_count: 2) end - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first ctx = JSON.parse(row[:error_context], symbolize_names: true) expect(ctx[:retry_count]).to eq(2) end @@ -714,7 +719,7 @@ module CCCStoreTestMessages group.stop(message: 'operator requested shutdown') end - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first expect(row[:status]).to eq('stopped') expect(row[:retry_at]).to be_nil end @@ -729,7 +734,7 @@ module CCCStoreTestMessages group.fail(exception: err) end - row = db[:ccc_consumer_groups].where(group_id: 'my-group').first + row = db[:sourced_consumer_groups].where(group_id: 'my-group').first expect(row[:status]).to eq('failed') expect(row[:retry_at]).to be_nil ctx = JSON.parse(row[:error_context], symbolize_names: true) @@ -749,9 +754,9 @@ module CCCStoreTestMessages handled_types: ['store_test.device.registered'], worker_id: 'w-1') - expect(db[:ccc_offsets].count).to be > 0 + expect(db[:sourced_offsets].count).to be > 0 store.reset_consumer_group('my-group') - expect(db[:ccc_offsets].count).to eq(0) + expect(db[:sourced_offsets].count).to eq(0) end it 'accepts an object responding to #group_id' do @@ -765,9 +770,9 @@ module CCCStoreTestMessages worker_id: 'w-1') reactor = double('reactor', group_id: 'my-group') - expect(db[:ccc_offsets].count).to be > 0 + expect(db[:sourced_offsets].count).to be > 0 store.reset_consumer_group(reactor) - expect(db[:ccc_offsets].count).to eq(0) + expect(db[:sourced_offsets].count).to eq(0) end end @@ -785,7 +790,7 @@ module CCCStoreTestMessages ) store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') - expect(db[:ccc_offsets].count).to be >= 1 + expect(db[:sourced_offsets].count).to be >= 1 end it 'returns nil when no pending messages' do @@ -1244,7 +1249,7 @@ module CCCStoreTestMessages store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) - offset = db[:ccc_offsets].where(id: result.offset_id).first + offset = db[:sourced_offsets].where(id: result.offset_id).first expect(offset[:last_position]).to eq(result.messages.last.position) expect(offset[:claimed]).to eq(0) expect(offset[:claimed_at]).to be_nil @@ -1286,8 +1291,8 @@ module CCCStoreTestMessages position: 1 ) - offset = db[:ccc_offsets].join(:ccc_consumer_groups, id: :consumer_group_id) - .where(Sequel[:ccc_consumer_groups][:group_id] => group_id) + offset = db[:sourced_offsets].join(:sourced_consumer_groups, id: :consumer_group_id) + .where(Sequel[:sourced_consumer_groups][:group_id] => group_id) .first expect(offset[:last_position]).to eq(1) end @@ -1302,7 +1307,7 @@ module CCCStoreTestMessages position: 1 ) - cg = db[:ccc_consumer_groups].where(group_id: group_id).first + cg = db[:sourced_consumer_groups].where(group_id: group_id).first expect(cg[:highest_position]).to eq(1) end @@ -1323,8 +1328,8 @@ module CCCStoreTestMessages position: 1 ) - offset = db[:ccc_offsets].join(:ccc_consumer_groups, id: :consumer_group_id) - .where(Sequel[:ccc_consumer_groups][:group_id] => group_id) + offset = db[:sourced_offsets].join(:sourced_consumer_groups, id: :consumer_group_id) + .where(Sequel[:sourced_consumer_groups][:group_id] => group_id) .first expect(offset[:last_position]).to eq(2) end @@ -1345,7 +1350,7 @@ module CCCStoreTestMessages position: 1 ) - cg = db[:ccc_consumer_groups].where(group_id: group_id).first + cg = db[:sourced_consumer_groups].where(group_id: group_id).first expect(cg[:highest_position]).to eq(2) end @@ -1401,7 +1406,7 @@ module CCCStoreTestMessages ) }.not_to raise_error - expect(db[:ccc_offsets].count).to eq(0) + expect(db[:sourced_offsets].count).to eq(0) end it 'is a no-op when partition has no messages in the store' do @@ -1412,7 +1417,7 @@ module CCCStoreTestMessages ) }.not_to raise_error - expect(db[:ccc_offsets].count).to eq(0) + expect(db[:sourced_offsets].count).to eq(0) end it 'works with composite partitions' do @@ -1425,8 +1430,8 @@ module CCCStoreTestMessages position: 1 ) - offset = db[:ccc_offsets].join(:ccc_consumer_groups, id: :consumer_group_id) - .where(Sequel[:ccc_consumer_groups][:group_id] => group_id) + offset = db[:sourced_offsets].join(:sourced_consumer_groups, id: :consumer_group_id) + .where(Sequel[:sourced_consumer_groups][:group_id] => group_id) .first expect(offset[:last_position]).to eq(1) expect(offset[:partition_key]).to eq('course_name:Algebra|user_id:joe') @@ -1579,7 +1584,7 @@ module CCCStoreTestMessages store.release(group_id, offset_id: result.offset_id) - offset = db[:ccc_offsets].where(id: result.offset_id).first + offset = db[:sourced_offsets].where(id: result.offset_id).first expect(offset[:last_position]).to eq(0) # not advanced expect(offset[:claimed]).to eq(0) end From 8daadaeb026f754082a9b14875f25ffe06118d76 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 15:43:18 +0000 Subject: [PATCH 049/115] Document CCC Sequel migration support --- lib/sourced/ccc/README.md | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 097bd469..d070db51 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -106,6 +106,65 @@ messages = store.read_all(from_position: messages.last.position, limit: 20) Returns an array of `PositionedMessage` instances ordered by position, or `[]` if the store is empty or there are no more messages after `from_position`. +### Database setup + +`Store#install!` creates all required tables directly (useful for scripts, tests, and quick prototyping). For production apps using Sequel migrations, the store can export a migration file instead. + +#### Quick setup (e.g. scripts, tests) + +```ruby +db = Sequel.sqlite('my_app.db') +store = Sourced::CCC::Store.new(db) +store.install! +``` + +#### Exporting a Sequel migration + +Use `Store#copy_migration_to` to generate a migration file compatible with `Sequel::Migrator`: + +```ruby +db = Sequel.sqlite('my_app.db') +store = Sourced::CCC::Store.new(db) + +# Option 1: pass a directory (uses a default filename) +store.copy_migration_to('db/migrations') + +# Option 2: pass a block for full control over the path +store.copy_migration_to do + "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_ccc_tables.rb" +end +``` + +Then run your migrations as usual: + +```bash +sequel -m db/migrations sqlite://my_app.db +``` + +#### Custom table prefix + +By default, tables are prefixed with `sourced_` (e.g. `sourced_messages`, `sourced_consumer_groups`). Pass a `prefix:` to `Store.new` to customise this — for example when running multiple CCC stores in the same database: + +```ruby +store = Sourced::CCC::Store.new(db, prefix: 'billing') +store.install! +# Creates: billing_messages, billing_key_pairs, billing_consumer_groups, ... +``` + +The prefix is carried through to exported migrations automatically. + +#### Using the Installer directly + +The installer is also available as a standalone object, which is useful for Rake tasks or setup scripts: + +```ruby +installer = Sourced::CCC::Installer.new(db, logger: Logger.new($stdout), prefix: 'sourced') +installer.install # create tables +installer.installed? # check if tables exist +installer.uninstall # drop tables (test env only) +installer.copy_migration_to('db/migrations') +``` + ## Deciders Deciders handle commands, enforce invariants, and produce events. They rebuild state from event history before each decision. From 3938e9fd160628229bf594b872b6ce5d9904a1b3 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 16:12:16 +0000 Subject: [PATCH 050/115] handle_batch -> handle_claim --- lib/sourced/ccc.rb | 2 +- lib/sourced/ccc/decider.rb | 2 +- lib/sourced/ccc/projector.rb | 4 ++-- lib/sourced/ccc/router.rb | 4 ++-- spec/sourced/ccc/decider_spec.rb | 10 +++++----- spec/sourced/ccc/projector_spec.rb | 22 +++++++++++----------- spec/sourced/ccc/router_spec.rb | 18 +++++++++--------- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index c146602d..e86a5821 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -131,7 +131,7 @@ def self.handle!(reactor_class, command, store: nil) end # Load history if the reactor needs it (Deciders always do) - needs_history = Injector.resolve_args(reactor_class, :handle_batch).include?(:history) + needs_history = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) if needs_history instance, read_result = load(reactor_class, store: store, **partition_attrs) end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index 4130e7b3..5608e7dd 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -38,7 +38,7 @@ def command(message_class, &block) # @param claim [ClaimResult] claimed partition batch # @param history [ReadResult] event history for the partition # @return [Array, PositionedMessage)>] action/source pairs - def handle_batch(claim, history:) + def handle_claim(claim, history:) values = partition_keys.map { |k| claim.partition_value[k.to_s] } instance = new(values) instance.evolve(history.messages) diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index a0b8b92f..f38f0fc8 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -56,7 +56,7 @@ class StateStored < self class << self # @param claim [ClaimResult] claimed partition batch # @return [Array, PositionedMessage)>] action/source pairs - def handle_batch(claim) + def handle_claim(claim) instance = build_instance(claim) instance.evolve(claim.messages) build_action_pairs(instance, claim) @@ -70,7 +70,7 @@ class << self # @param claim [ClaimResult] claimed partition batch # @param history [ReadResult] full partition history # @return [Array, PositionedMessage)>] action/source pairs - def handle_batch(claim, history:) + def handle_claim(claim, history:) instance = build_instance(claim) instance.evolve(history.messages) build_action_pairs(instance, claim) diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index 5481a3cf..76a9680b 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -16,7 +16,7 @@ def initialize(store:) def register(reactor_class) @reactors << reactor_class store.register_consumer_group(reactor_class.group_id) - @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_batch).include?(:history) + @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) end def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) @@ -39,7 +39,7 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) kwargs[:history] = store.read(conditions) end - action_pairs = reactor_class.handle_batch(claim, **kwargs) + action_pairs = reactor_class.handle_claim(claim, **kwargs) if action_pairs == Actions::RETRY store.release(reactor_class.group_id, offset_id: claim.offset_id) diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/sourced/ccc/decider_spec.rb index a6fd616f..825d964e 100644 --- a/spec/sourced/ccc/decider_spec.rb +++ b/spec/sourced/ccc/decider_spec.rb @@ -140,7 +140,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider end end - describe '.handle_batch' do + describe '.handle_claim' do let(:db) { Sequel.sqlite } let(:store) { Sourced::CCC::Store.new(db) } @@ -166,7 +166,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider messages: [cmd_positioned], replaying: false, guard: guard ) - pairs = TestDeviceDecider.handle_batch(claim, history: history) + pairs = TestDeviceDecider.handle_claim(claim, history: history) # Should have action pairs from the command expect(pairs).to be_a(Array) @@ -205,7 +205,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider messages: [reg_positioned], replaying: false, guard: guard ) - pairs = TestDeviceDecider.handle_batch(claim, history: history) + pairs = TestDeviceDecider.handle_claim(claim, history: history) expect(pairs.size).to eq(1) actions, source_msg = pairs.first @@ -228,7 +228,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider guard: guard ) - pairs = TestDelayedReactionDecider.handle_batch(claim, history: history) + pairs = TestDelayedReactionDecider.handle_claim(claim, history: history) actions = pairs.first.first schedule_action = Array(actions).find { |action| action.is_a?(Sourced::CCC::Actions::Schedule) } @@ -251,7 +251,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider # No history → state[:exists] is false → raises 'Not found' expect { - TestDeviceDecider.handle_batch(claim, history: history) + TestDeviceDecider.handle_claim(claim, history: history) }.to raise_error(RuntimeError, 'Not found') end end diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb index 44aeeea6..c56a10e9 100644 --- a/spec/sourced/ccc/projector_spec.rb +++ b/spec/sourced/ccc/projector_spec.rb @@ -102,7 +102,7 @@ class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored end end - describe '.handle_batch' do + describe '.handle_claim' do let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 2) } def make_claim(messages, replaying: false) @@ -124,7 +124,7 @@ def make_claim(messages, replaying: false) ] claim = make_claim(msgs) - pairs = TestItemProjector.handle_batch(claim) + pairs = TestItemProjector.handle_claim(claim) # Last pair should contain sync actions sync_pair = pairs.last @@ -143,7 +143,7 @@ def make_claim(messages, replaying: false) ] claim = make_claim(msgs, replaying: false) - pairs = TestItemProjector.handle_batch(claim) + pairs = TestItemProjector.handle_claim(claim) # Should have reaction pair + sync pair append_actions = pairs.flat_map { |actions, _| Array(actions) } @@ -161,7 +161,7 @@ def make_claim(messages, replaying: false) ] claim = make_claim(msgs, replaying: false) - pairs = TestDelayedItemProjector.handle_batch(claim) + pairs = TestDelayedItemProjector.handle_claim(claim) schedule_actions = pairs.flat_map { |actions, _| Array(actions) } .select { |action| action.is_a?(Sourced::CCC::Actions::Schedule) } @@ -178,7 +178,7 @@ def make_claim(messages, replaying: false) ] claim = make_claim(msgs, replaying: true) - pairs = TestItemProjector.handle_batch(claim) + pairs = TestItemProjector.handle_claim(claim) append_actions = pairs.flat_map { |actions, _| Array(actions) } .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } @@ -194,7 +194,7 @@ def make_claim(messages, replaying: false) ] claim = make_claim(msgs, replaying: true) - pairs = TestItemProjector.handle_batch(claim) + pairs = TestItemProjector.handle_claim(claim) # Execute the sync action to verify replaying is passed through sync_pair = pairs.last @@ -238,7 +238,7 @@ def make_history(messages) claim = make_claim(claim_msgs) history = make_history(history_msgs) - pairs = TestItemESProjector.handle_batch(claim, history: history) + pairs = TestItemESProjector.handle_claim(claim, history: history) # Sync pair should be the last one, acked against claim's last message sync_pair = pairs.last @@ -260,7 +260,7 @@ def make_history(messages) claim = make_claim(claim_msgs, replaying: false) history = make_history(history_msgs) - pairs = TestItemESProjector.handle_batch(claim, history: history) + pairs = TestItemESProjector.handle_claim(claim, history: history) append_actions = pairs.flat_map { |actions, _| Array(actions) } .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } @@ -279,7 +279,7 @@ def make_history(messages) claim = make_claim(history_msgs, replaying: true) history = make_history(history_msgs) - pairs = TestItemESProjector.handle_batch(claim, history: history) + pairs = TestItemESProjector.handle_claim(claim, history: history) append_actions = pairs.flat_map { |actions, _| Array(actions) } .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } @@ -288,12 +288,12 @@ def make_history(messages) end it 'is detected by Injector as needing history' do - needs = Sourced::Injector.resolve_args(TestItemESProjector, :handle_batch) + needs = Sourced::Injector.resolve_args(TestItemESProjector, :handle_claim) expect(needs).to include(:history) end it 'StateStored is not detected as needing history' do - needs = Sourced::Injector.resolve_args(TestItemProjector, :handle_batch) + needs = Sourced::Injector.resolve_args(TestItemProjector, :handle_claim) expect(needs).not_to include(:history) end end diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index b62117e6..41280eba 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -83,7 +83,7 @@ class RouterTestProjector < Sourced::CCC::Projector::StateStored end end -# Simple reactor: just extends Consumer, defines handled_messages, implements handle_batch. +# Simple reactor: just extends Consumer, defines handled_messages, implements handle_claim. # Logs an audit trail message for every DeviceRegistered or DeviceBound it sees. class RouterTestAuditReactor extend Sourced::CCC::Consumer @@ -95,7 +95,7 @@ def self.handled_messages [CCCRouterTestMessages::DeviceRegistered, CCCRouterTestMessages::DeviceBound] end - def self.handle_batch(claim) + def self.handle_claim(claim) each_with_partial_ack(claim.messages) do |msg| audit = CCCRouterTestMessages::DeviceAudited.new( payload: { device_id: msg.payload.device_id, event_type: msg.type } @@ -115,7 +115,7 @@ def self.handle_batch(claim) end describe '#register' do - it 'creates consumer group and introspects handle_batch signature' do + it 'creates consumer group and introspects handle_claim signature' do router.register(RouterTestDecider) expect(store.consumer_group_active?('router-test-decider')).to be true @@ -143,7 +143,7 @@ def self.handle_batch(claim) expect(result).to be false end - it 'claims, calls handle_batch, executes actions + acks in transaction' do + it 'claims, calls handle_claim, executes actions + acks in transaction' do # Set up: register device first (as history), then send bind command store.append( CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) @@ -212,8 +212,8 @@ def self.handle_batch(claim) CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) - # Make handle_batch raise - allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + # Make handle_claim raise + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') allow(RouterTestDecider).to receive(:on_exception) result = router.handle_next_for(RouterTestDecider) @@ -229,7 +229,7 @@ def self.handle_batch(claim) CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) - allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') router.handle_next_for(RouterTestDecider) expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false @@ -245,7 +245,7 @@ def self.handle_batch(claim) CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) - allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') router.handle_next_for(RouterTestDecider) @@ -267,7 +267,7 @@ def self.handle_batch(claim) end allow(Sourced::CCC).to receive_message_chain(:config, :error_strategy).and_return(retry_strategy) - allow(RouterTestDecider).to receive(:handle_batch).and_raise(RuntimeError, 'boom') + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') router.handle_next_for(RouterTestDecider) From fad69ed941724fa68fb0b4b875412543ad9d1560 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 16:35:31 +0000 Subject: [PATCH 051/115] Make reactors support .handle_batch --- lib/sourced/ccc/actions.rb | 4 +- lib/sourced/ccc/decider.rb | 22 +++--- lib/sourced/ccc/projector.rb | 37 +++++---- spec/sourced/ccc/projector_spec.rb | 119 +++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 27 deletions(-) diff --git a/lib/sourced/ccc/actions.rb b/lib/sourced/ccc/actions.rb index 39de31f2..b3e8c984 100644 --- a/lib/sourced/ccc/actions.rb +++ b/lib/sourced/ccc/actions.rb @@ -22,9 +22,9 @@ def self.build_for(messages, guard: nil, source: nil, correlated: false) now = Time.now to_schedule, to_append = messages.partition { |message| message.created_at > now } - actions << Append.new(to_append, guard: guard, source: source, correlated: correlated) if to_append.any? + actions << Append.new(to_append, guard:, source:, correlated:) if to_append.any? to_schedule.group_by(&:created_at).each do |at, scheduled_messages| - actions << Schedule.new(scheduled_messages, at:, source: source, correlated: correlated) + actions << Schedule.new(scheduled_messages, at:, source:, correlated:) end actions diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index 5608e7dd..ebd7047a 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -33,17 +33,11 @@ def command(message_class, &block) define_method(Sourced.message_method_name('ccc_decide', message_class.to_s), &block) end - # Build executable actions for a claimed batch. - # - # @param claim [ClaimResult] claimed partition batch - # @param history [ReadResult] event history for the partition - # @return [Array, PositionedMessage)>] action/source pairs - def handle_claim(claim, history:) - values = partition_keys.map { |k| claim.partition_value[k.to_s] } - instance = new(values) + def handle_batch(partition_values, new_messages, history:) + instance = new(partition_values) instance.evolve(history.messages) - each_with_partial_ack(claim.messages) do |msg| + each_with_partial_ack(new_messages) do |msg| if handled_commands.include?(msg.class) raw_events = instance.decide(msg) correlated_events = raw_events.map { |e| msg.correlate(e) } @@ -69,6 +63,16 @@ def handle_claim(claim, history:) end end + # Build executable actions for a claimed batch. + # + # @param claim [ClaimResult] claimed partition batch + # @param history [ReadResult] event history for the partition + # @return [Array, PositionedMessage)>] action/source pairs + def handle_claim(claim, history:) + values = partition_keys.map { |k| claim.partition_value[k.to_s] } + handle_batch(values, claim.messages, history:) + end + # Copy registered command handlers into subclasses. # # @param subclass [Class] subclass being created diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index f38f0fc8..6c05f8fe 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -19,20 +19,15 @@ def handled_messages private - def build_instance(claim) - values = partition_keys.map { |k| claim.partition_value[k.to_s] } - new(values) - end - - def build_action_pairs(instance, claim) + def build_action_pairs(instance, messages, replaying:) sync_actions = instance.sync_actions( - state: instance.state, messages: claim.messages, replaying: claim.replaying + state: instance.state, messages: messages, replaying: replaying ) - reaction_pairs = if claim.replaying + reaction_pairs = if replaying [] else - each_with_partial_ack(claim.messages) do |msg| + each_with_partial_ack(messages) do |msg| next unless instance.reacts_to?(msg) reaction_msgs = Array(instance.react(msg)) actions = Actions.build_for(reaction_msgs) @@ -40,7 +35,7 @@ def build_action_pairs(instance, claim) end end - reaction_pairs + [[sync_actions, claim.messages.last]] + reaction_pairs + [[sync_actions, messages.last]] end end @@ -54,12 +49,17 @@ def initialize(partition_values = []) # Projector variant that evolves only the claimed messages on top of stored state. class StateStored < self class << self + def handle_batch(partition_values, new_messages, replaying: false) + instance = new(partition_values) + instance.evolve(new_messages) + build_action_pairs(instance, new_messages, replaying: replaying) + end + # @param claim [ClaimResult] claimed partition batch # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim) - instance = build_instance(claim) - instance.evolve(claim.messages) - build_action_pairs(instance, claim) + values = partition_keys.map { |k| claim.partition_value[k.to_s] } + handle_batch(values, claim.messages, replaying: claim.replaying) end end end @@ -67,13 +67,18 @@ def handle_claim(claim) # Projector variant that rebuilds state from full history each time. class EventSourced < self class << self + def handle_batch(partition_values, new_messages, history:, replaying: false) + instance = new(partition_values) + instance.evolve(history.messages) + build_action_pairs(instance, new_messages, replaying: replaying) + end + # @param claim [ClaimResult] claimed partition batch # @param history [ReadResult] full partition history # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim, history:) - instance = build_instance(claim) - instance.evolve(history.messages) - build_action_pairs(instance, claim) + values = partition_keys.map { |k| claim.partition_value[k.to_s] } + handle_batch(values, claim.messages, history: history, replaying: claim.replaying) end end end diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb index c56a10e9..5e23c006 100644 --- a/spec/sourced/ccc/projector_spec.rb +++ b/spec/sourced/ccc/projector_spec.rb @@ -102,6 +102,125 @@ class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored end end + describe '.handle_batch (StateStored)' do + it 'evolves from new_messages and includes sync actions' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ) + ] + + pairs = TestItemProjector.handle_batch(['L1'], msgs) + + sync_pair = pairs.last + sync_actions, source_msg = sync_pair + expect(source_msg).to eq(msgs.last) + + sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::Sync) } + expect(sync_action).not_to be_nil + end + + it 'runs reactions when not replaying' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + + pairs = TestItemProjector.handle_batch(['L1'], msgs) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions.size).to eq(1) + expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) + end + + it 'skips reactions when replaying' do + msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + + pairs = TestItemProjector.handle_batch(['L1'], msgs, replaying: true) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions).to be_empty + end + end + + describe '.handle_batch (EventSourced)' do + let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 5) } + + def make_history(messages) + Sourced::CCC::ReadResult.new(messages: messages, guard: guard) + end + + it 'evolves from full history, not just new messages' do + history_msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 3 + ) + ] + new_msgs = [history_msgs.last] + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(['L1'], new_msgs, history: history) + + sync_pair = pairs.last + _sync_actions, source_msg = sync_pair + expect(source_msg).to eq(new_msgs.last) + end + + it 'runs reactions only on new messages, not full history' do + history_msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Old' }), 1 + ), + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'New' }), 2 + ) + ] + new_msgs = [history_msgs.last] + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(['L1'], new_msgs, history: history) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions.size).to eq(1) + end + + it 'skips reactions when replaying' do + history_msgs = [ + Sourced::CCC::PositionedMessage.new( + CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(['L1'], history_msgs, history: history, replaying: true) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + + expect(append_actions).to be_empty + end + end + describe '.handle_claim' do let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 2) } From 6d0c253739a39a56c8c1d0c0db1664056b925a6a Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 17:11:54 +0000 Subject: [PATCH 052/115] no need for argument --- lib/sourced/ccc/projector.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index 6c05f8fe..bfd2585c 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -78,7 +78,7 @@ def handle_batch(partition_values, new_messages, history:, replaying: false) # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim, history:) values = partition_keys.map { |k| claim.partition_value[k.to_s] } - handle_batch(values, claim.messages, history: history, replaying: claim.replaying) + handle_batch(values, claim.messages, history:, replaying: claim.replaying) end end end From b6f034e74166b6c1515399ebc0d3e184320d021d Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 17:19:06 +0000 Subject: [PATCH 053/115] Replicate with_reactor RSpec helper for CCC --- lib/sourced/ccc/README.md | 138 ++++++++++++ lib/sourced/ccc/testing/rspec.rb | 285 +++++++++++++++++++++++++ spec/sourced/ccc/testing/rspec_spec.rb | 279 ++++++++++++++++++++++++ 3 files changed, 702 insertions(+) create mode 100644 lib/sourced/ccc/testing/rspec.rb create mode 100644 spec/sourced/ccc/testing/rspec_spec.rb diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index d070db51..3a0a4c68 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -518,6 +518,144 @@ stats.groups.each do |g| end ``` +## Testing + +CCC ships with RSpec helpers for Given-When-Then testing of deciders and projectors. The helpers call `handle_batch` directly — no store, router, or consumer group setup needed. + +```ruby +require 'sourced/ccc/testing/rspec' + +RSpec.configure do |config| + config.include Sourced::CCC::Testing::RSpec +end +``` + +### Testing deciders + +`with_reactor` takes a decider class and partition attributes, then chains `.given` (history), `.when` (command), and `.then` (expected outcomes). + +```ruby +RSpec.describe CourseDecider do + include Sourced::CCC::Testing::RSpec + + it 'creates a course' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') + .then(CourseCreated, course_id: 'c1', course_name: 'Algebra') + end + + it 'rejects duplicate course names' do + with_reactor(CourseDecider, course_name: 'Algebra') + .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') + .when(CreateCourse, course_id: 'c2', course_name: 'Algebra') + .then(RuntimeError, "Course 'Algebra' already exists") + end + + it 'produces no events for a no-op command' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(SomeNoopCommand, course_name: 'Algebra') + .then([]) + end +end +``` + +#### Multiple expected messages + +When a decider produces events and reactions, pass all expected messages as instances: + +```ruby +it 'produces event and reaction' do + with_reactor(EnrolmentDecider, course_id: 'c1') + .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') + .when(EnrolStudent, course_id: 'c1', student_id: 's1') + .then( + StudentEnrolled.new(payload: { course_id: 'c1', student_id: 's1' }), + NotifyStudent.new(payload: { student_id: 's1' }) + ) +end +``` + +#### Block form + +Pass a block to `.then` to receive the raw action pairs for custom assertions: + +```ruby +it 'inspects action pairs' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') + .then { |pairs| + actions, source_msg = pairs.first + append = Array(actions).find { |a| a.is_a?(Sourced::CCC::Actions::Append) } + expect(append.messages.first).to be_a(CourseCreated) + } +end +``` + +#### `.then!` — run sync actions + +Use `.then!` instead of `.then` to execute sync actions before assertions: + +```ruby +it 'runs sync block' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') + .then! { |pairs| ... } +end +``` + +### Testing projectors + +Projectors use `.given` (events to evolve) and `.then` with a block that receives the projected state. `.when` is not supported — projectors don't handle commands. + +#### StateStored + +```ruby +RSpec.describe ItemProjector do + include Sourced::CCC::Testing::RSpec + + it 'builds state from events' do + with_reactor(ItemProjector, list_id: 'L1') + .given(ItemAdded, list_id: 'L1', name: 'Apple') + .given(ItemAdded, list_id: 'L1', name: 'Banana') + .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } + end + + it 'handles removal' do + with_reactor(ItemProjector, list_id: 'L1') + .given(ItemAdded, list_id: 'L1', name: 'Apple') + .and(ItemArchived, list_id: 'L1', name: 'Apple') + .then { |state| expect(state[:items]).to eq([]) } + end + + it 'runs sync actions with then!' do + with_reactor(ItemProjector, list_id: 'L1') + .given(ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |state| expect(state[:synced]).to be true } + end +end +``` + +#### EventSourced + +Same API — the helper creates an instance, evolves from all given messages, and yields state: + +```ruby +RSpec.describe CatalogProjector do + include Sourced::CCC::Testing::RSpec + + it 'rebuilds state from full history' do + with_reactor(CatalogProjector, course_id: 'c1') + .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') + .given(StudentEnrolled, course_id: 'c1', student_id: 's1') + .then { |state| expect(state[:students]).to eq(['s1']) } + end +end +``` + +### Message matching + +`.then` compares messages by **class** and **payload** only. Fields like `id`, `created_at`, `causation_id`, `correlation_id`, and `metadata` are ignored, so tests don't need to match auto-generated values. + ## Full example See `examples/ccc_app/` for a complete Sinatra application with: diff --git a/lib/sourced/ccc/testing/rspec.rb b/lib/sourced/ccc/testing/rspec.rb new file mode 100644 index 00000000..c64174d4 --- /dev/null +++ b/lib/sourced/ccc/testing/rspec.rb @@ -0,0 +1,285 @@ +# frozen_string_literal: true + +require 'sourced/ccc' + +module Sourced + module CCC + module Testing + module RSpec + NONE = [].freeze + + # Entry point for CCC reactor GWT tests. + # + # @param reactor_class [Class] a CCC::Decider or CCC::Projector subclass + # @param partition_attrs [Hash] partition key-value pairs (e.g. device_id: 'd1') + # @return [GWT] + # + # @example Decider + # with_reactor(MyDecider, device_id: 'd1') + # .given(DeviceRegistered, device_id: 'd1', name: 'Sensor') + # .when(BindDevice, device_id: 'd1', asset_id: 'a1') + # .then(DeviceBound, device_id: 'd1', asset_id: 'a1') + # + # @example Projector + # with_reactor(MyProjector, list_id: 'L1') + # .given(ItemAdded, list_id: 'L1', name: 'Apple') + # .then { |state| expect(state[:items]).to eq(['Apple']) } + def with_reactor(reactor_class, **partition_attrs) + GWT.new(reactor_class, **partition_attrs) + end + + class MessageMatcher + def initialize(expected_messages) + @expected_messages = Array(expected_messages) + @errors = [] + @mismatching = Hash.new { |h, k| h[k] = [] } + end + + def matches?(actual_messages) + if @expected_messages.size != actual_messages.size + @errors << "Expected #{@expected_messages.size} messages, but got #{actual_messages.size}" + @errors << actual_messages.inspect + return false + end + + @expected_messages.each.with_index do |expected, idx| + actual = actual_messages[idx] + @mismatching[idx] << "expected a #{expected.class}, got #{actual.class}" unless actual.class == expected.class + @mismatching[idx] << "expected payload #{expected.payload.to_h.inspect}, got #{actual.payload.to_h.inspect}" unless expected.payload == actual.payload + end + + return false if @mismatching.any? + + true + end + + def failure_message + err = +@errors.join("\n") + @mismatching.each do |idx, errors| + err << "Message #{idx}: \n" + errors.each do |e| + err << "- #{e}\n" + end + err << "\n" + end + err + end + end + + class GWT + def initialize(reactor_class, **partition_attrs) + @reactor_class = reactor_class + @partition_attrs = partition_attrs + @partition_values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } + @given_messages = [] + @when_message = nil + @asserted = false + end + + # Accumulate history/context messages. + # For Deciders: these become the history (ReadResult). + # For Projectors: these are evolved onto the instance. + # + # @param klass [Class] message class + # @param payload [Hash] payload attributes + # @return [self] + def given(klass_or_instance = nil, **payload) + raise 'test case already asserted' if @asserted + + msg = build_message(klass_or_instance, **payload) + @given_messages << msg + self + end + + alias_method :and, :given + + # Set the command to decide on (Deciders only). + # + # @param klass [Class] command class + # @param payload [Hash] payload attributes + # @return [self] + def when(klass_or_instance = nil, **payload) + raise 'test case already asserted' if @asserted + raise ArgumentError, '.when is not supported for Projectors' if projector? + + @when_message = build_message(klass_or_instance, **payload) + self + end + + # Assert expected outcomes. + # + # For Deciders: + # - Pass message class + payload pairs to assert produced messages + # - Pass [] or NONE to assert no messages + # - Pass an Exception class (+ optional message) to assert invariant violation + # - Pass a block to receive action pairs for custom assertions + # + # For Projectors: + # - Requires a block that receives the evolved state + # + # @return [self] + def then(*expected, **payload, &block) + run_then(false, *expected, **payload, &block) + end + + # Like #then, but runs sync actions before yielding state (Projectors) + # or before extracting messages (Deciders). + def then!(*expected, **payload, &block) + run_then(true, *expected, **payload, &block) + end + + private + + def decider? + @reactor_class < CCC::Decider + end + + def projector? + @reactor_class < CCC::Projector + end + + def build_message(klass_or_instance, **payload) + if klass_or_instance.is_a?(CCC::Message) + klass_or_instance + else + klass_or_instance.new(payload: payload) + end + end + + def run_then(sync, *expected, **payload, &block) + @asserted = true + + # Shorthand: .then(Class, key: val) → build message from class + payload + if expected.size == 1 && expected[0].is_a?(Class) && !(expected[0] < ::Exception) && payload.any? + expected = [expected[0].new(payload: payload)] + end + + if decider? + run_decider_then(sync, *expected, &block) + elsif projector? + run_projector_then(sync, *expected, &block) + else + raise ArgumentError, "unsupported reactor type: #{@reactor_class}" + end + + self + end + + def run_decider_then(sync, *expected, &block) + # Exception expectation + if expected.size >= 1 && exception_expectation?(expected[0]) + expect_exception(expected[0], expected[1]) + return + end + + pairs = run_decider + + if sync + pairs.each do |actions, _| + Array(actions).select { |a| a.is_a?(CCC::Actions::Sync) }.each(&:call) + end + end + + if block_given? + block.call(pairs) + return + end + + # Extract messages from action pairs + actual_messages = extract_messages(pairs) + + # Build expected messages + expected_msgs = build_expected(*expected) + + if expected_msgs.empty? + unless actual_messages.empty? + ::RSpec::Expectations.fail_with( + "Expected no messages, but got #{actual_messages.size}: #{actual_messages.inspect}" + ) + end + return + end + + matcher = MessageMatcher.new(expected_msgs) + unless matcher.matches?(actual_messages) + ::RSpec::Expectations.fail_with(matcher.failure_message) + end + end + + def run_projector_then(sync, *_expected, &block) + raise ArgumentError, '.then for Projectors requires a block' unless block_given? + + instance = @reactor_class.new(@partition_values) + + if @reactor_class < CCC::Projector::EventSourced + # EventSourced: evolve from full history + instance.evolve(@given_messages) + else + # StateStored: evolve from given messages + instance.evolve(@given_messages) + end + + if sync + instance.sync_actions( + state: instance.state, messages: @given_messages, replaying: false + ).each(&:call) + end + + block.call(instance.state) + end + + def run_decider + guard = ConsistencyGuard.new(conditions: [], last_position: 0) + history = ReadResult.new(messages: @given_messages, guard: guard) + @reactor_class.handle_batch(@partition_values, [@when_message], history: history) + end + + def extract_messages(pairs) + pairs.flat_map { |actions, _| + Array(actions) + .select { |a| a.respond_to?(:messages) } + .flat_map(&:messages) + } + end + + def build_expected(*args) + return [] if args == [[]] || args == [NONE] + return [] if args.empty? + + args.map do |arg| + case arg + when CCC::Message + arg + else + raise ArgumentError, "unsupported expected message: #{arg.inspect}" + end + end + end + + def exception_expectation?(arg) + arg.is_a?(Class) && arg < ::Exception + end + + def expect_exception(exception_class, message = nil) + guard = ConsistencyGuard.new(conditions: [], last_position: 0) + history = ReadResult.new(messages: @given_messages, guard: guard) + + begin + @reactor_class.handle_batch(@partition_values, [@when_message], history: history) + rescue exception_class => e + if message && e.message != message + ::RSpec::Expectations.fail_with( + "expected #{exception_class} with message #{message.inspect}, " \ + "but got #{e.message.inspect}" + ) + end + return + end + + ::RSpec::Expectations.fail_with("expected #{exception_class} to be raised, but nothing was raised") + end + end + end + end + end +end diff --git a/spec/sourced/ccc/testing/rspec_spec.rb b/spec/sourced/ccc/testing/rspec_spec.rb new file mode 100644 index 00000000..041b0b82 --- /dev/null +++ b/spec/sourced/ccc/testing/rspec_spec.rb @@ -0,0 +1,279 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sourced/ccc/testing/rspec' + +# Reuse message definitions from decider_spec and projector_spec +module CCCGWTTestMessages + DeviceRegistered = Sourced::CCC::Message.define('gwt_test.device.registered') do + attribute :device_id, String + attribute :name, String + end + + DeviceBound = Sourced::CCC::Message.define('gwt_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDevice = Sourced::CCC::Message.define('gwt_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end + + NotifyBound = Sourced::CCC::Message.define('gwt_test.notify_bound') do + attribute :device_id, String + end + + NoopCommand = Sourced::CCC::Message.define('gwt_test.noop_command') do + attribute :device_id, String + end + + ItemAdded = Sourced::CCC::Message.define('gwt_test.item.added') do + attribute :list_id, String + attribute :name, String + end + + ItemArchived = Sourced::CCC::Message.define('gwt_test.item.archived') do + attribute :list_id, String + attribute :name, String + end + + NotifyArchive = Sourced::CCC::Message.define('gwt_test.notify_archive') do + attribute :list_id, String + end +end + +class GWTTestDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'gwt-test-decider' + + state { |_| { exists: false, bound: false } } + + evolve CCCGWTTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + evolve CCCGWTTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command CCCGWTTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event CCCGWTTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end + + command CCCGWTTestMessages::NoopCommand do |_state, _cmd| + # intentionally produces no events + end + + reaction CCCGWTTestMessages::DeviceBound do |_state, evt| + CCCGWTTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) + end + + sync do |state:, messages:, events:| + state[:synced] = true + end +end + +# Decider without reactions (produces only events) +class GWTTestSimpleDecider < Sourced::CCC::Decider + partition_by :device_id + consumer_group 'gwt-test-simple-decider' + + state { |_| { exists: false } } + + evolve CCCGWTTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + command CCCGWTTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + event CCCGWTTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end +end + +class GWTTestStateStoredProjector < Sourced::CCC::Projector::StateStored + partition_by :list_id + consumer_group 'gwt-test-ss-projector' + + state do |(list_id)| + { list_id: list_id, items: [], synced: false } + end + + evolve CCCGWTTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end + + evolve CCCGWTTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + end +end + +class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced + partition_by :list_id + consumer_group 'gwt-test-es-projector' + + state do |(list_id)| + { list_id: list_id, items: [], synced: false } + end + + evolve CCCGWTTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end + + evolve CCCGWTTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + end +end + +RSpec.describe Sourced::CCC::Testing::RSpec do + include Sourced::CCC::Testing::RSpec + + describe 'Decider' do + it 'given history + when command → then expected messages (event + reaction)' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then( + CCCGWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), + CCCGWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) + ) + end + + it 'then with shorthand (Class, **payload) for single expected message' do + with_reactor(GWTTestSimpleDecider, device_id: 'd1') + .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then(CCCGWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') + end + + it 'no given + when command → then exception (invariant violation)' do + with_reactor(GWTTestDecider, device_id: 'd1') + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then(RuntimeError, 'Not found') + end + + it 'then with block form yields action pairs' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then { |pairs| + expect(pairs).to be_a(Array) + actions, _source = pairs.first + append_actions = Array(actions).select { |a| a.respond_to?(:messages) } + expect(append_actions).not_to be_empty + } + end + + it 'then with [] expects no messages' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(CCCGWTTestMessages::NoopCommand, device_id: 'd1') + .then([]) + end + + it 'then! runs sync actions' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then! { |pairs| + expect(pairs).to be_a(Array) + } + end + + it 'given with message instances' do + reg = CCCGWTTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + + with_reactor(GWTTestDecider, device_id: 'd1') + .given(reg) + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then( + CCCGWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), + CCCGWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) + ) + end + + it 'supports .and as alias for .given' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .and(CCCGWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') + .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a2') + .then(RuntimeError, 'Already bound') + end + end + + describe 'Projector (StateStored)' do + it 'given events → then block asserts evolved state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then { |state| expect(state[:items]).to eq(['Apple']) } + end + + it 'given multiple events → then block sees cumulative state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') + .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } + end + + it 'then! runs sync actions before yielding state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |state| expect(state[:synced]).to be true } + end + + it 'given events with archive → state reflects removal' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .and(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') + .and(CCCGWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') + .then { |state| expect(state[:items]).to eq(['Banana']) } + end + + it '.when raises ArgumentError' do + expect { + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .when(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + }.to raise_error(ArgumentError, '.when is not supported for Projectors') + end + end + + describe 'Projector (EventSourced)' do + it 'given events → then block asserts evolved state' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') + .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } + end + + it 'given events with archive → state reflects removal' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .and(CCCGWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') + .then { |state| expect(state[:items]).to eq([]) } + end + + it 'then! runs sync actions before yielding state' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |state| expect(state[:synced]).to be true } + end + + it '.when raises ArgumentError' do + expect { + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .when(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + }.to raise_error(ArgumentError, '.when is not supported for Projectors') + end + end +end From f92f298d906a28f53431adc60e2e15735b51fcb3 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 17:42:03 +0000 Subject: [PATCH 054/115] Lazy-load CCC::Store --- lib/sourced/ccc.rb | 2 -- lib/sourced/ccc/configuration.rb | 12 ++++++++---- spec/sourced/ccc/store_spec.rb | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index e86a5821..1a675ebe 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -223,8 +223,6 @@ def self.load(reactor_class, store: nil, **partition_attrs) require 'sourced/ccc/configuration' require 'sourced/ccc/message' -require 'sourced/ccc/installer' -require 'sourced/ccc/store' require 'sourced/ccc/actions' require 'sourced/ccc/consumer' require 'sourced/ccc/evolve' diff --git a/lib/sourced/ccc/configuration.rb b/lib/sourced/ccc/configuration.rb index fbe5ea34..c6e30842 100644 --- a/lib/sourced/ccc/configuration.rb +++ b/lib/sourced/ccc/configuration.rb @@ -43,9 +43,10 @@ def initialize # Accepts a CCC::Store, a Sequel::SQLite::Database (auto-wrapped), # or any object implementing StoreInterface. def store=(s) - @store = case s - when Store then s - when ->(v) { v.class.name == 'Sequel::SQLite::Database' } then Store.new(s) + @store = case s.class.name + when 'Sequel::SQLite::Database' + require 'sourced/ccc/store' + Store.new(s) else StoreInterface.parse(s) end end @@ -63,7 +64,10 @@ def error_strategy def setup! return if @setup - @store ||= Store.new(Sequel.sqlite) + unless @store + require 'sourced/ccc/store' + @store = Store.new(Sequel.sqlite) + end @store.install! @router ||= Router.new(store: @store) @setup = true diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 916fd267..d3b4428c 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' require 'sourced/ccc' +require 'sourced/ccc/store' require 'sequel' # Define test messages for store specs (namespaced to avoid collisions) From e20bc1cdb61de11044edf55bcc14c5f0a7dfc1ba Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 12 Mar 2026 23:14:04 +0000 Subject: [PATCH 055/115] Update offsets in a transaction to avoid locking on SQLite's single writer --- lib/sourced/ccc/store.rb | 45 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 13698bf3..3dde1a1c 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -759,31 +759,36 @@ def bootstrap_offsets(cg_id, partition_by) GROUP BY #{group_by} SQL - db.fetch(sql).each do |row| - # Build the values hash and collect key_pair_ids - values = {} - kp_ids = [] - partition_by.each_with_index do |attr, i| - values[attr] = row[:"val_#{i}"] - kp_ids << row[:"kp_id_#{i}"] - end - - partition_key = build_partition_key(partition_by, values) + rows = db.fetch(sql).all + return if rows.empty? - # INSERT OR IGNORE the offset row - db.run(<<~SQL) - INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) - VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) - SQL + db.transaction do + rows.each do |row| + # Build the values hash and collect key_pair_ids + values = {} + kp_ids = [] + partition_by.each_with_index do |attr, i| + values[attr] = row[:"val_#{i}"] + kp_ids << row[:"kp_id_#{i}"] + end - offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + partition_key = build_partition_key(partition_by, values) - # INSERT OR IGNORE the offset_key_pairs join rows - kp_ids.each do |kp_id| + # INSERT OR IGNORE the offset row db.run(<<~SQL) - INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) - VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) + INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) + VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) SQL + + offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + + # INSERT OR IGNORE the offset_key_pairs join rows + kp_ids.each do |kp_id| + db.run(<<~SQL) + INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) + VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) + SQL + end end end end From 3076cefafc4f9b668d0c5a35624367ae57798f04 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 14:18:03 +0000 Subject: [PATCH 056/115] Add CCC::CommandContext for building commands from raw attributes Mirrors Sourced::CommandContext but without stream_id (CCC is stream-less). Supports building from type string + payload hash or explicit class, with metadata injection, string key symbolization, and scoped registries. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 1 + lib/sourced/ccc/README.md | 38 ++++++++++++++ lib/sourced/ccc/command_context.rb | 35 +++++++++++++ spec/sourced/ccc/command_context_spec.rb | 67 ++++++++++++++++++++++++ 4 files changed, 141 insertions(+) create mode 100644 lib/sourced/ccc/command_context.rb create mode 100644 spec/sourced/ccc/command_context_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 1a675ebe..583729d1 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -234,5 +234,6 @@ def self.load(reactor_class, store: nil, **partition_attrs) require 'sourced/ccc/worker' require 'sourced/ccc/stale_claim_reaper' require 'sourced/ccc/dispatcher' +require 'sourced/ccc/command_context' require 'sourced/ccc/topology' require 'sourced/ccc/supervisor' diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 3a0a4c68..00d90a1c 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -212,6 +212,44 @@ end Raises `Sourced::ConcurrentAppendError` on conflicts, or `RuntimeError` on domain invariant violations (e.g. "Course already exists"). +### CommandContext + +`CCC::CommandContext` is a factory for building CCC commands from raw Hash attributes (e.g. HTTP params), injecting defaults like `metadata`. It mirrors `Sourced::CommandContext` but without `stream_id`, since CCC messages are stream-less. + +```ruby +# In a web controller, build a context with shared metadata +ctx = Sourced::CCC::CommandContext.new( + metadata: { user_id: session[:user_id] } +) + +# Build from a type string + payload hash (e.g. from JSON params) +cmd = ctx.build(type: 'courses.create', payload: { course_id: 'c1', course_name: 'Algebra' }) +cmd.metadata[:user_id] # => session[:user_id] + +# Or pass an explicit command class +cmd = ctx.build(CreateCourse, payload: { course_id: 'c1', course_name: 'Algebra' }) +``` + +String keys are automatically symbolized, so `ctx.build('type' => '...', 'payload' => { ... })` works too. + +#### Scoping to a command subset + +By default, `CommandContext` looks up types in `CCC::Command.registry`. Pass a `scope:` to restrict lookups to a specific command subclass — attempts to build commands outside the scope raise `Sourced::UnknownMessageError`. + +```ruby +class PublicCommand < Sourced::CCC::Command; end + +CreateCourse = PublicCommand.define('courses.create') do + attribute :course_id, String + attribute :course_name, String +end + +# Only PublicCommand subclasses are allowed +ctx = Sourced::CCC::CommandContext.new(scope: PublicCommand) +ctx.build(type: 'courses.create', payload: { ... }) # OK +ctx.build(type: 'admin.delete_all', payload: {}) # raises UnknownMessageError +``` + ### Loading a decider's state ```ruby diff --git a/lib/sourced/ccc/command_context.rb b/lib/sourced/ccc/command_context.rb new file mode 100644 index 00000000..53715b1f --- /dev/null +++ b/lib/sourced/ccc/command_context.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'sourced/types' + +module Sourced + module CCC + class CommandContext + # @option metadata [Hash] metadata to add to commands built by this context + # @option scope [CCC::Message] Message class to use as command registry + def initialize(metadata: Plumb::BLANK_HASH, scope: CCC::Command) + @defaults = { metadata: }.freeze + @scope = scope + end + + # @param args [Array] either [Hash] or [Class, Hash] + # @return [CCC::Message] + def build(*args) + case args + in [Class => klass, Hash => attrs] + attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) + klass.parse(attrs) + in [Hash => attrs] + attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) + scope.from(attrs) + else + raise ArgumentError, "Invalid arguments: #{args.inspect}" + end + end + + private + + attr_reader :defaults, :scope + end + end +end diff --git a/spec/sourced/ccc/command_context_spec.rb b/spec/sourced/ccc/command_context_spec.rb new file mode 100644 index 00000000..1eb09191 --- /dev/null +++ b/spec/sourced/ccc/command_context_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' + +module CccContextTest + Add = Sourced::CCC::Command.define('ccc_ctest.add') do + attribute :value, Integer + end + + Added = Sourced::CCC::Event.define('ccc_ctest.added') +end + +RSpec.describe Sourced::CCC::CommandContext do + describe '#build' do + it 'builds command from type string with metadata' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) + expect(cmd).to be_a(CccContextTest::Add) + expect(cmd.payload.value).to eq(1) + expect(cmd.metadata[:user_id]).to eq(10) + end + + it 'can take a command class' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd).to be_a(CccContextTest::Add) + expect(cmd.payload.value).to eq(1) + expect(cmd.metadata[:user_id]).to eq(10) + end + + it 'symbolizes string keys' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build('type' => 'ccc_ctest.add', 'payload' => { 'value' => 1 }) + expect(cmd).to be_a(CccContextTest::Add) + expect(cmd.payload.value).to eq(1) + expect(cmd.metadata[:user_id]).to eq(10) + end + + it 'raises UnknownMessageError for unknown types' do + ctx = described_class.new(metadata: { user_id: 10 }) + expect do + ctx.build('type' => 'nope', 'payload' => { 'value' => 1 }) + end.to raise_error(Sourced::UnknownMessageError) + end + + it 'raises UnknownMessageError for event types when scoped to Command' do + ctx = described_class.new(metadata: { user_id: 10 }) + expect do + ctx.build('type' => 'ccc_ctest.added', 'payload' => {}) + end.to raise_error(Sourced::UnknownMessageError) + end + + it 'allows scoping to a custom command subclass' do + custom_scope = Class.new(Sourced::CCC::Command) + custom_cmd = custom_scope.define('ccc_ctest.custom') do + attribute :name, String + end + + ctx = described_class.new(metadata: { user_id: 10 }, scope: custom_scope) + cmd = ctx.build(type: 'ccc_ctest.custom', payload: { name: 'hello' }) + expect(cmd).to be_a(custom_cmd) + expect(cmd.payload.name).to eq('hello') + expect(cmd.metadata[:user_id]).to eq(10) + end + end +end From 95a33143b79a7e0a8501bd8069d48347ddda2942 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 14:38:02 +0000 Subject: [PATCH 057/115] Add order: :desc option to Store#read_all for reverse-chronological browsing Pagination uses position < from_position when descending, so cursor-based paging works in both directions. Default from_position changed to nil (no filter) so the first page naturally starts from the latest or earliest. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 13 +++++++++---- spec/sourced/ccc/store_spec.rb | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 3dde1a1c..b8ecd360 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -232,10 +232,15 @@ def update_schedule! # @param from_position [Integer] return messages after this position (default 0) # @param limit [Integer] max number of messages to return (default 50) # @return [Array] messages ordered by position - def read_all(from_position: 0, limit: 50) - db[@messages_table] - .where { position > from_position } - .order(:position) + def read_all(from_position: nil, limit: 50, order: :asc) + desc = order == :desc + ds = db[@messages_table] + + if from_position + ds = desc ? ds.where { position < from_position } : ds.where { position > from_position } + end + + ds.order(desc ? Sequel.desc(:position) : :position) .limit(limit) .map { |row| deserialize(row) } end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index d3b4428c..b053041d 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -334,6 +334,35 @@ module CCCStoreTestMessages expect(messages.first).to be_a(Sourced::CCC::PositionedMessage) expect(messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) end + + context 'with order: :desc' do + before do + 5.times do |i| + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) + end + end + + it 'returns messages in descending position order' do + messages = store.read_all(order: :desc) + expect(messages.map(&:position)).to eq([5, 4, 3, 2, 1]) + end + + it 'paginates in descending order using from_position' do + page1 = store.read_all(order: :desc, limit: 2) + expect(page1.map(&:position)).to eq([5, 4]) + + page2 = store.read_all(from_position: page1.last.position, order: :desc, limit: 2) + expect(page2.map(&:position)).to eq([3, 2]) + + page3 = store.read_all(from_position: page2.last.position, order: :desc, limit: 2) + expect(page3.map(&:position)).to eq([1]) + end + + it 'returns [] when no messages before from_position' do + messages = store.read_all(from_position: 1, order: :desc) + expect(messages).to eq([]) + end + end end describe '#read' do From cd92ea166004b89b29178882c4f07d75381d8d58 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 14:51:42 +0000 Subject: [PATCH 058/115] Return ReadAllResult from Store#read_all with Enumerable, to_enum, and last_position ReadAllResult wraps messages with last_position (max global position) so clients know when to stop paginating. Includes Enumerable for current-page iteration and #to_enum for lazy auto-paginating enumeration across all pages. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/README.md | 46 ++++++++++-- lib/sourced/ccc/store.rb | 30 +++++++- spec/sourced/ccc/store_spec.rb | 125 ++++++++++++++++++++++++++------- 3 files changed, 168 insertions(+), 33 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 00d90a1c..211e375d 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -94,17 +94,51 @@ result = store.read_partition( ### Browsing the global log -`read_all` paginates the entire message log in position order, without requiring query conditions or partition attributes. +`read_all` paginates the entire message log, without requiring query conditions or partition attributes. It returns a `ReadAllResult` with `messages` and `last_position` (the current max position in the store), so clients know whether more pages exist. ```ruby -# First page (default limit: 50) -messages = store.read_all(limit: 20) +# First page (default limit: 50, ascending order) +result = store.read_all(limit: 20) +result.messages # => [PositionedMessage, ...] +result.last_position # => 100 (max position in the store) -# Next page — pass the last position from the previous page -messages = store.read_all(from_position: messages.last.position, limit: 20) +# Next page — pass the last message's position as cursor +result = store.read_all(from_position: result.messages.last.position, limit: 20) + +# Check if there are more pages +has_more = result.messages.any? && result.messages.last.position < result.last_position + +# Destructuring is also supported +messages, last_position = store.read_all(limit: 20) +``` + +Use `order: :desc` for reverse-chronological browsing (newest first). Pagination works the same way — `from_position` fetches messages *before* the given position. + +```ruby +result = store.read_all(order: :desc, limit: 20) + +# Next page of older messages +result = store.read_all(from_position: result.messages.last.position, order: :desc, limit: 20) ``` -Returns an array of `PositionedMessage` instances ordered by position, or `[]` if the store is empty or there are no more messages after `from_position`. +#### Iterating all messages with `to_enum` + +`ReadAllResult#to_enum` returns a lazy `Enumerator` that transparently fetches subsequent pages as you iterate, using the same `order` and `limit` from the original query. + +```ruby +# Iterate all messages in pages of 50 +store.read_all(limit: 50).to_enum.each do |msg| + puts "#{msg.position}: #{msg.type}" +end + +# Works with Enumerable methods +store.read_all(order: :desc, limit: 100).to_enum.map(&:type) + +# Supports lazy enumeration — stops fetching pages once satisfied +store.read_all(limit: 20).to_enum.lazy.select { |m| + m.type == 'courses.created' +}.first(5) +``` ### Database setup diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index b8ecd360..8f134f12 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -33,6 +33,29 @@ def instance_of?(klass) = __getobj__.instance_of?(klass) def to_ary = [messages, guard] end + ReadAllResult = Data.define(:messages, :last_position, :fetcher) do + include Enumerable + + def to_ary = [messages, last_position] + + # Iterates messages in the current page. + def each(&block) = messages.each(&block) + + # Returns an Enumerator that lazily paginates through all messages, + # fetching subsequent pages as needed. + def to_enum + Enumerator.new do |y| + result = self + loop do + break if result.messages.empty? + + result.messages.each { |m| y << m } + result = result.fetcher.call(result.messages.last.position) + end + end + end + end + Stats = Data.define(:max_position, :groups) # SQLite-backed store for CCC's flat, globally-ordered message log. @@ -231,7 +254,7 @@ def update_schedule! # # @param from_position [Integer] return messages after this position (default 0) # @param limit [Integer] max number of messages to return (default 50) - # @return [Array] messages ordered by position + # @return [ReadAllResult] messages and last global position def read_all(from_position: nil, limit: 50, order: :asc) desc = order == :desc ds = db[@messages_table] @@ -240,9 +263,12 @@ def read_all(from_position: nil, limit: 50, order: :asc) ds = desc ? ds.where { position < from_position } : ds.where { position > from_position } end - ds.order(desc ? Sequel.desc(:position) : :position) + messages = ds.order(desc ? Sequel.desc(:position) : :position) .limit(limit) .map { |row| deserialize(row) } + + fetcher = ->(pos) { read_all(from_position: pos, limit: limit, order: order) } + ReadAllResult.new(messages: messages, last_position: latest_position, fetcher: fetcher) end # Query messages by conditions. Each condition matches on diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index b053041d..c0f63525 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -296,16 +296,48 @@ module CCCStoreTestMessages end describe '#read_all' do - it 'returns messages in position order' do + it 'returns a ReadAllResult with messages and last_position' do store.append([ CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }), CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' }) ]) - messages = store.read_all - expect(messages.size).to eq(3) - expect(messages.map(&:position)).to eq([1, 2, 3]) + result = store.read_all + expect(result).to be_a(Sourced::CCC::ReadAllResult) + expect(result.messages.size).to eq(3) + expect(result.messages.map(&:position)).to eq([1, 2, 3]) + expect(result.last_position).to eq(3) + end + + it 'supports #each to iterate current page messages' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }) + ]) + + result = store.read_all + positions = result.map(&:position) + expect(positions).to eq([1, 2]) + end + + it '#each without a block returns an enumerator for chaining' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }) + ]) + + result = store.read_all + pairs = result.each.with_index.map { |msg, i| [i, msg.position] } + expect(pairs).to eq([[0, 1], [1, 2]]) + end + + it 'supports destructuring into messages and last_position' do + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + messages, last_position = store.read_all + expect(messages.size).to eq(1) + expect(last_position).to eq(1) end it 'paginates with from_position and limit' do @@ -313,26 +345,29 @@ module CCCStoreTestMessages store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) end - page1 = store.read_all(limit: 2) - expect(page1.map(&:position)).to eq([1, 2]) + result1 = store.read_all(limit: 2) + expect(result1.messages.map(&:position)).to eq([1, 2]) + expect(result1.last_position).to eq(5) - page2 = store.read_all(from_position: page1.last.position, limit: 2) - expect(page2.map(&:position)).to eq([3, 4]) + result2 = store.read_all(from_position: result1.messages.last.position, limit: 2) + expect(result2.messages.map(&:position)).to eq([3, 4]) - page3 = store.read_all(from_position: page2.last.position, limit: 2) - expect(page3.map(&:position)).to eq([5]) + result3 = store.read_all(from_position: result2.messages.last.position, limit: 2) + expect(result3.messages.map(&:position)).to eq([5]) end - it 'returns [] for an empty store' do - expect(store.read_all).to eq([]) + it 'returns empty messages with last_position 0 for an empty store' do + result = store.read_all + expect(result.messages).to eq([]) + expect(result.last_position).to eq(0) end it 'returns PositionedMessage instances' do store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) - messages = store.read_all - expect(messages.first).to be_a(Sourced::CCC::PositionedMessage) - expect(messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) + result = store.read_all + expect(result.messages.first).to be_a(Sourced::CCC::PositionedMessage) + expect(result.messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) end context 'with order: :desc' do @@ -343,24 +378,64 @@ module CCCStoreTestMessages end it 'returns messages in descending position order' do - messages = store.read_all(order: :desc) - expect(messages.map(&:position)).to eq([5, 4, 3, 2, 1]) + result = store.read_all(order: :desc) + expect(result.messages.map(&:position)).to eq([5, 4, 3, 2, 1]) + expect(result.last_position).to eq(5) end it 'paginates in descending order using from_position' do - page1 = store.read_all(order: :desc, limit: 2) - expect(page1.map(&:position)).to eq([5, 4]) + result1 = store.read_all(order: :desc, limit: 2) + expect(result1.messages.map(&:position)).to eq([5, 4]) - page2 = store.read_all(from_position: page1.last.position, order: :desc, limit: 2) - expect(page2.map(&:position)).to eq([3, 2]) + result2 = store.read_all(from_position: result1.messages.last.position, order: :desc, limit: 2) + expect(result2.messages.map(&:position)).to eq([3, 2]) - page3 = store.read_all(from_position: page2.last.position, order: :desc, limit: 2) - expect(page3.map(&:position)).to eq([1]) + result3 = store.read_all(from_position: result2.messages.last.position, order: :desc, limit: 2) + expect(result3.messages.map(&:position)).to eq([1]) end it 'returns [] when no messages before from_position' do - messages = store.read_all(from_position: 1, order: :desc) - expect(messages).to eq([]) + result = store.read_all(from_position: 1, order: :desc) + expect(result.messages).to eq([]) + expect(result.last_position).to eq(5) + end + end + + describe '#to_enum' do + before do + 5.times do |i| + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) + end + end + + it 'iterates all messages across pages in ascending order' do + result = store.read_all(limit: 2) + positions = result.to_enum.map(&:position) + expect(positions).to eq([1, 2, 3, 4, 5]) + end + + it 'iterates all messages across pages in descending order' do + result = store.read_all(order: :desc, limit: 2) + positions = result.to_enum.map(&:position) + expect(positions).to eq([5, 4, 3, 2, 1]) + end + + it 'works when all messages fit in a single page' do + result = store.read_all(limit: 100) + positions = result.to_enum.map(&:position) + expect(positions).to eq([1, 2, 3, 4, 5]) + end + + it 'returns an empty enumerator for an empty store' do + store.clear! + result = store.read_all(limit: 2) + expect(result.to_enum.to_a).to eq([]) + end + + it 'supports lazy enumeration' do + result = store.read_all(limit: 2) + first_three = result.to_enum.lazy.take(3).map(&:position).to_a + expect(first_three).to eq([1, 2, 3]) end end end From 67c3d9df9dfed273edd95d7cc527aa7844ea4696 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 16:23:22 +0000 Subject: [PATCH 059/115] Change QueryCondition to support compound attrs with AND semantics QueryCondition now holds a Hash of attrs instead of a single key_name/key_value pair. Attributes within a condition are AND'd (a message must match all), while separate conditions are still OR'd. This fixes composite partition reads where OR semantics caused cross-partition events to leak into history (e.g. seat B9's events appearing in seat C7's context because they shared a showing_id). - QueryCondition: Data.define(:message_type, :attrs) replaces (:message_type, :key_name, :key_value) - Message.to_conditions: returns one condition per type with all matching attrs - Store#query_messages: COUNT/HAVING-based AND within each condition, UNION across - Store#max_position_for: same AND-within/OR-across pattern - Add store specs for compound condition AND, cross-partition exclusion, and guards Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/message.rb | 25 ++-- lib/sourced/ccc/store.rb | 84 ++++++++----- spec/sourced/ccc/decider_spec.rb | 2 +- spec/sourced/ccc/message_spec.rb | 27 ++-- spec/sourced/ccc/store_spec.rb | 205 +++++++++++++++++++++---------- 5 files changed, 225 insertions(+), 118 deletions(-) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 63d3ccbf..4bdd05f0 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -5,9 +5,9 @@ module Sourced module CCC # A query condition for reading messages from the store. - # Matches on (message_type AND key_name AND key_value). + # Matches on (message_type AND all attrs key-value pairs). # Multiple conditions are OR'd when passed to {Store#read}. - QueryCondition = Data.define(:message_type, :key_name, :key_value) + QueryCondition = Data.define(:message_type, :attrs) # Returned by {Store#read} and {Store#claim_next} for optimistic concurrency. # Pass to {Store#append} via +guard:+ to detect conflicting writes. @@ -182,28 +182,25 @@ def correlate(message) # @return [Array] attribute names (e.g. +[:course_name, :user_id]+) def self.payload_attribute_names = EMPTY_ARRAY - # Build {QueryCondition}s for the intersection of this message's declared + # Build a {QueryCondition} for the intersection of this message's declared # attributes and the given key-value pairs. Attributes not declared on this - # message class are silently ignored. + # message class are silently ignored. Returns an array with a single condition + # containing all matching attrs, or an empty array if none match. # # @param attrs [Hash{Symbol => String}] partition attribute values # @return [Array] # # @example # CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') - # # => [QueryCondition('course.created', 'course_name', 'Algebra')] + # # => [QueryCondition('course.created', { course_name: 'Algebra' })] # # user_id ignored — CourseCreated doesn't declare it def self.to_conditions(**attrs) supported = payload_attribute_names - attrs.filter_map do |key, value| - next unless supported.include?(key) - - QueryCondition.new( - message_type: type, - key_name: key.to_s, - key_value: value.to_s - ) - end + matched = attrs.select { |key, _| supported.include?(key) } + .transform_values(&:to_s) + return [] if matched.empty? + + [QueryCondition.new(message_type: type, attrs: matched)] end # Auto-extract key-value pairs from all top-level payload attributes. diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 8f134f12..cbab61e7 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -910,36 +910,51 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: end # Core query logic shared by {#read} and {#check_conflicts}. - # Resolves key_pair IDs from conditions, then queries messages via OR'd clauses. + # Resolves key_pair IDs from conditions, then queries messages. + # Attributes within each condition are AND'd; conditions are OR'd. # # @param conditions [Array] # @param from_position [Integer, nil] # @param limit [Integer, nil] # @return [Array] def query_messages(conditions, from_position: nil, limit: nil) - # Step 1: resolve key_pair IDs - key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq - or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } + # Step 1: resolve all key_pair IDs across all conditions + all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq + return [] if all_lookups.empty? + + or_clauses = all_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all key_pair_index = {} key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } - # Build condition clauses using resolved key_pair IDs - where_parts = conditions.filter_map do |c| - kp_id = key_pair_index[[c.key_name, c.key_value]] - next unless kp_id # key pair not in DB means no matches for this condition - - "(m.message_type = #{db.literal(c.message_type)} AND mkp.key_pair_id = #{db.literal(kp_id)})" + # Step 2: build per-condition subqueries (AND within, OR across) + subqueries = conditions.filter_map do |c| + kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } + # If any attr's key pair is missing from DB, this condition can't match + next if kp_ids.size < c.attrs.size + + kp_ids_list = kp_ids.map { |id| db.literal(id) }.join(', ') + + <<~SQL + SELECT DISTINCT m.position + FROM #{@messages_table} m + JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position + WHERE m.message_type = #{db.literal(c.message_type)} + AND mkp.key_pair_id IN (#{kp_ids_list}) + GROUP BY m.position + HAVING COUNT(DISTINCT mkp.key_pair_id) = #{kp_ids.size} + SQL end - return [] if where_parts.empty? + return [] if subqueries.empty? + + union = subqueries.join(" UNION ") sql = <<~SQL - SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at + SELECT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at FROM #{@messages_table} m - JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position - WHERE (#{where_parts.join(' OR ')}) + WHERE m.position IN (#{union}) SQL sql += " AND m.position > #{db.literal(from_position)}" if from_position @@ -960,7 +975,8 @@ def check_conflicts(conditions, after_position) query_messages(conditions, from_position: after_position) end - # Max position among messages matching the given conditions (OR semantics). + # Max position among messages matching the given conditions. + # Attributes within each condition are AND'd; conditions are OR'd. # Returns from_position (or latest_position) if no matches. # # @param conditions [Array] @@ -969,28 +985,38 @@ def check_conflicts(conditions, after_position) def max_position_for(conditions, from_position: nil) return from_position || latest_position if conditions.empty? - key_lookups = conditions.map { |c| [c.key_name, c.key_value] }.uniq - or_clauses = key_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } + all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq + return from_position || latest_position if all_lookups.empty? + + or_clauses = all_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all key_pair_index = {} key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } - where_parts = conditions.filter_map do |c| - kp_id = key_pair_index[[c.key_name, c.key_value]] - next unless kp_id - "(m.message_type = #{db.literal(c.message_type)} AND mkp.key_pair_id = #{db.literal(kp_id)})" + subqueries = conditions.filter_map do |c| + kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } + next if kp_ids.size < c.attrs.size + + kp_ids_list = kp_ids.map { |id| db.literal(id) }.join(', ') + + <<~SQL + SELECT m.position + FROM #{@messages_table} m + JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position + WHERE m.message_type = #{db.literal(c.message_type)} + AND mkp.key_pair_id IN (#{kp_ids_list}) + GROUP BY m.position + HAVING COUNT(DISTINCT mkp.key_pair_id) = #{kp_ids.size} + SQL end - return from_position || latest_position if where_parts.empty? + return from_position || latest_position if subqueries.empty? - sql = <<~SQL - SELECT MAX(m.position) AS max_pos - FROM #{@messages_table} m - JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position - WHERE (#{where_parts.join(' OR ')}) - SQL - sql += " AND m.position > #{db.literal(from_position)}" if from_position + union = subqueries.join(" UNION ") + + sql = "SELECT MAX(position) AS max_pos FROM (#{union})" + sql += " WHERE position > #{db.literal(from_position)}" if from_position row = db.fetch(sql).first row[:max_pos] || from_position || latest_position diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/sourced/ccc/decider_spec.rb index 825d964e..be8ea528 100644 --- a/spec/sourced/ccc/decider_spec.rb +++ b/spec/sourced/ccc/decider_spec.rb @@ -273,7 +273,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider types = conditions.map(&:message_type).uniq.sort expect(types).to include('decider_test.device.registered') expect(types).to include('decider_test.device.bound') - expect(conditions.all? { |c| c.key_name == 'device_id' && c.key_value == 'd1' }).to be true + expect(conditions.all? { |c| c.attrs[:device_id] == 'd1' }).to be true end end diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index 237d08e1..848ce39f 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -150,18 +150,17 @@ module CCCTestMessages end describe '.to_conditions' do - it 'returns conditions only for attributes the message class has' do + it 'returns one condition with only attributes the message class has' do conditions = CCCTestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', asset_id: 'asset-1') expect(conditions.size).to eq(1) expect(conditions.first.message_type).to eq('device.registered') - expect(conditions.first.key_name).to eq('device_id') - expect(conditions.first.key_value).to eq('dev-1') + expect(conditions.first.attrs).to eq({ device_id: 'dev-1' }) end - it 'returns conditions for all matching attributes' do + it 'includes all matching attributes in one condition' do conditions = CCCTestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', name: 'Sensor A') - expect(conditions.size).to eq(2) - expect(conditions.map(&:key_name).sort).to eq(['device_id', 'name']) + expect(conditions.size).to eq(1) + expect(conditions.first.attrs).to eq({ device_id: 'dev-1', name: 'Sensor A' }) end it 'returns empty array when no attributes match' do @@ -222,15 +221,21 @@ module CCCTestMessages end describe Sourced::CCC::QueryCondition do - it 'is a Data struct with message_type, key_name, key_value' do + it 'is a Data struct with message_type and attrs hash' do cond = Sourced::CCC::QueryCondition.new( message_type: 'device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) expect(cond.message_type).to eq('device.registered') - expect(cond.key_name).to eq('device_id') - expect(cond.key_value).to eq('dev-1') + expect(cond.attrs).to eq({ device_id: 'dev-1' }) + end + + it 'supports multiple attrs for compound conditions' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'seat.selected', + attrs: { showing_id: 'show-1', seat_id: 'C7' } + ) + expect(cond.attrs).to eq({ showing_id: 'show-1', seat_id: 'C7' }) end end end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index c0f63525..1a5ecd9d 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -149,8 +149,8 @@ module CCCStoreTestMessages ) store.append([source, caused]) - cond1 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.device.registered', key_name: 'device_id', key_value: 'dev-1') - cond2 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.asset.registered', key_name: 'asset_id', key_value: 'asset-1') + cond1 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' }) + cond2 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.asset.registered', attrs: { asset_id: 'asset-1' }) messages, = store.read([cond1, cond2]) src = messages.find { |m| m.type == 'store_test.device.registered' } @@ -199,8 +199,7 @@ module CCCStoreTestMessages cond = Sourced::CCC::QueryCondition.new( message_type: due.type, - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) result = store.read([cond]) msg = result.messages.first @@ -220,8 +219,7 @@ module CCCStoreTestMessages let(:cond) do Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) end @@ -453,8 +451,7 @@ module CCCStoreTestMessages it 'queries by message_type and key condition' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) results, guard = store.read([cond]) expect(results.size).to eq(1) @@ -465,8 +462,7 @@ module CCCStoreTestMessages it 'returns messages with position' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) results, _guard = store.read([cond]) expect(results.first.position).to eq(1) @@ -476,13 +472,11 @@ module CCCStoreTestMessages conditions = [ Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ), Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.bound', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) ] results, _guard = store.read(conditions) @@ -496,8 +490,7 @@ module CCCStoreTestMessages it 'filters with from_position' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) # dev-1 registered is at position 1, so from_position: 1 should return nothing results, _guard = store.read([cond], from_position: 1) @@ -512,13 +505,11 @@ module CCCStoreTestMessages conditions = [ Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ), Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.bound', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) ] results, _guard = store.read(conditions, limit: 1) @@ -529,8 +520,7 @@ module CCCStoreTestMessages it 'returns empty for non-matching conditions' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'nonexistent' + attrs: { device_id: 'nonexistent' } ) results, _guard = store.read([cond]) expect(results).to be_empty @@ -544,8 +534,7 @@ module CCCStoreTestMessages it 'deserializes into correct message subclasses' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) results, _guard = store.read([cond]) msg = results.first @@ -557,8 +546,7 @@ module CCCStoreTestMessages it 'returns a ConsistencyGuard with correct conditions' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) _results, guard = store.read([cond]) expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) @@ -568,8 +556,7 @@ module CCCStoreTestMessages it 'guard last_position reflects the last result position' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-2' + attrs: { device_id: 'dev-2' } ) results, guard = store.read([cond]) # dev-2 is at position 4 @@ -580,8 +567,7 @@ module CCCStoreTestMessages it 'guard last_position falls back to latest_position when no results' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'nonexistent' + attrs: { device_id: 'nonexistent' } ) _results, guard = store.read([cond]) expect(guard.last_position).to eq(store.latest_position) @@ -590,14 +576,113 @@ module CCCStoreTestMessages it 'guard last_position falls back to from_position when no results and from_position given' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'nonexistent' + attrs: { device_id: 'nonexistent' } ) _results, guard = store.read([cond], from_position: 2) expect(guard.last_position).to eq(2) end end + describe '#read with compound conditions (AND within, OR across)' do + before do + store.append([ + # Two DeviceBound events with different (device_id, asset_id) combinations + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-2' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-2', asset_id: 'asset-1' }), + # A single-attr message sharing device_id + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + ]) + end + + it 'compound condition ANDs attrs — only matches messages with all specified key pairs' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + attrs: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + results, _guard = store.read([cond]) + expect(results.size).to eq(1) + expect(results.first.payload.device_id).to eq('dev-1') + expect(results.first.payload.asset_id).to eq('asset-1') + end + + it 'excludes messages that match only one attr of a compound condition' do + # dev-1/asset-2 shares device_id but not asset_id + # dev-2/asset-1 shares asset_id but not device_id + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + attrs: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + results, _guard = store.read([cond]) + expect(results.size).to eq(1) + pairs = results.map { |m| [m.payload.device_id, m.payload.asset_id] } + expect(pairs).to eq([['dev-1', 'asset-1']]) + end + + it 'ORs across separate conditions — compound and single-attr mixed' do + conditions = [ + # Compound: only dev-1/asset-1 + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + attrs: { device_id: 'dev-1', asset_id: 'asset-1' } + ), + # Single-attr: dev-1 registered + Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'dev-1' } + ) + ] + results, _guard = store.read(conditions) + expect(results.size).to eq(2) + expect(results.map(&:type)).to contain_exactly( + 'store_test.device.bound', + 'store_test.device.registered' + ) + end + + it 'guard with compound condition detects conflicts correctly' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + attrs: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + _results, guard = store.read([cond]) + + # Concurrent write: same device_id + asset_id combination + conflicting = CCCStoreTestMessages::DeviceBound.new( + payload: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + store.append(conflicting) + + new_msg = CCCStoreTestMessages::DeviceBound.new( + payload: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + expect { + store.append(new_msg, guard: guard) + }.to raise_error(Sourced::ConcurrentAppendError) + end + + it 'guard with compound condition does NOT flag writes to a different partition as conflicts' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.bound', + attrs: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + _results, guard = store.read([cond]) + + # Write to a DIFFERENT partition (dev-1/asset-2) — should NOT conflict + other_partition = CCCStoreTestMessages::DeviceBound.new( + payload: { device_id: 'dev-1', asset_id: 'asset-2' } + ) + store.append(other_partition) + + new_msg = CCCStoreTestMessages::DeviceBound.new( + payload: { device_id: 'dev-1', asset_id: 'asset-1' } + ) + expect { + store.append(new_msg, guard: guard) + }.not_to raise_error + end + end + describe '#messages_since' do it 'returns messages after the given position' do msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) @@ -609,8 +694,7 @@ module CCCStoreTestMessages cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) conflicts, guard = store.messages_since([cond], pos) @@ -626,8 +710,7 @@ module CCCStoreTestMessages cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', - key_name: 'device_id', - key_value: 'dev-1' + attrs: { device_id: 'dev-1' } ) conflicts, _guard = store.messages_since([cond], pos) @@ -1050,7 +1133,7 @@ module CCCStoreTestMessages expect(r2.partition_value).to eq({ 'device_id' => 'dev-3' }) end - it 'returns a guard with conditions only for key_names each type actually has' do + it 'returns a guard with conditions only for attrs each type actually has' do store.append( CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) @@ -1061,12 +1144,11 @@ module CCCStoreTestMessages expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) expect(guard.last_position).to eq(result.messages.last.position) - # 1 key_pair (device_id=dev-1) × 1 handled_type = 1 condition + # 1 handled_type with device_id attr = 1 condition expect(guard.conditions.size).to eq(1) cond = guard.conditions.first expect(cond.message_type).to eq('store_test.device.registered') - expect(cond.key_name).to eq('device_id') - expect(cond.key_value).to eq('dev-1') + expect(cond.attrs).to eq({ device_id: 'dev-1' }) end it 'guard can be used for optimistic concurrency on append' do @@ -1282,7 +1364,7 @@ module CCCStoreTestMessages expect(courses.uniq.size).to eq(1) end - it 'returns guard with conditions only for key_names each message type actually has' do + it 'returns guard with one condition per message type containing only matching attrs' do store.append([ CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) @@ -1293,28 +1375,25 @@ module CCCStoreTestMessages expect(guard.last_position).to eq(result.messages.last.position) - # Expected conditions (derived from message class definitions, not store data): - # CourseCreated has course_name only → 1 condition - # UserRegistered has user_id (+ name, not a partition attr) → 1 condition - # CourseClosed has course_name only → 1 condition - # UserJoinedCourse has course_name + user_id → 2 conditions - # Total: 5 conditions - expect(guard.conditions.size).to eq(5) - - # CourseCreated should NOT have a user_id condition - course_created_conditions = guard.conditions.select { |c| c.message_type == 'store_test.course.created' } - expect(course_created_conditions.size).to eq(1) - expect(course_created_conditions.first.key_name).to eq('course_name') - - # UserRegistered should NOT have a course_name condition - user_registered_conditions = guard.conditions.select { |c| c.message_type == 'store_test.user.registered' } - expect(user_registered_conditions.size).to eq(1) - expect(user_registered_conditions.first.key_name).to eq('user_id') - - # UserJoinedCourse has both - joined_conditions = guard.conditions.select { |c| c.message_type == 'store_test.user.joined_course' } - expect(joined_conditions.size).to eq(2) - expect(joined_conditions.map(&:key_name).sort).to eq(['course_name', 'user_id']) + # Expected conditions (one per message type, compound attrs): + # CourseCreated has course_name only → 1 condition with { course_name: 'Algebra' } + # UserRegistered has user_id (+ name, not a partition attr) → 1 condition with { user_id: 'joe' } + # CourseClosed has course_name only → 1 condition with { course_name: 'Algebra' } + # UserJoinedCourse has course_name + user_id → 1 condition with { course_name: 'Algebra', user_id: 'joe' } + # Total: 4 conditions + expect(guard.conditions.size).to eq(4) + + # CourseCreated has only course_name + course_created_cond = guard.conditions.find { |c| c.message_type == 'store_test.course.created' } + expect(course_created_cond.attrs).to eq({ course_name: 'Algebra' }) + + # UserRegistered has only user_id + user_registered_cond = guard.conditions.find { |c| c.message_type == 'store_test.user.registered' } + expect(user_registered_cond.attrs).to eq({ user_id: 'joe' }) + + # UserJoinedCourse has both attrs in a single condition + joined_cond = guard.conditions.find { |c| c.message_type == 'store_test.user.joined_course' } + expect(joined_cond.attrs).to eq({ course_name: 'Algebra', user_id: 'joe' }) end it 'guard detects concurrent writes in composite partition' do From 7bb9685fcdb6a045bd0f39072f2d86679a4695e5 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 16:34:49 +0000 Subject: [PATCH 060/115] Extract condition_position_subqueries to deduplicate query_messages and max_position_for - Extract shared key-pair resolution and subquery building into a single helper - Remove redundant SELECT DISTINCT (GROUP BY already deduplicates) - Push from_position filter into subqueries so the DB skips historical rows early Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 67 ++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index cbab61e7..c5959936 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -918,35 +918,7 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: # @param limit [Integer, nil] # @return [Array] def query_messages(conditions, from_position: nil, limit: nil) - # Step 1: resolve all key_pair IDs across all conditions - all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq - return [] if all_lookups.empty? - - or_clauses = all_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } - key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all - - key_pair_index = {} - key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } - - # Step 2: build per-condition subqueries (AND within, OR across) - subqueries = conditions.filter_map do |c| - kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } - # If any attr's key pair is missing from DB, this condition can't match - next if kp_ids.size < c.attrs.size - - kp_ids_list = kp_ids.map { |id| db.literal(id) }.join(', ') - - <<~SQL - SELECT DISTINCT m.position - FROM #{@messages_table} m - JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position - WHERE m.message_type = #{db.literal(c.message_type)} - AND mkp.key_pair_id IN (#{kp_ids_list}) - GROUP BY m.position - HAVING COUNT(DISTINCT mkp.key_pair_id) = #{kp_ids.size} - SQL - end - + subqueries = condition_position_subqueries(conditions, from_position: from_position) return [] if subqueries.empty? union = subqueries.join(" UNION ") @@ -955,10 +927,8 @@ def query_messages(conditions, from_position: nil, limit: nil) SELECT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at FROM #{@messages_table} m WHERE m.position IN (#{union}) + ORDER BY m.position SQL - - sql += " AND m.position > #{db.literal(from_position)}" if from_position - sql += ' ORDER BY m.position' sql += " LIMIT #{db.literal(limit)}" if limit db.fetch(sql).map { |row| deserialize(row) } @@ -985,8 +955,24 @@ def check_conflicts(conditions, after_position) def max_position_for(conditions, from_position: nil) return from_position || latest_position if conditions.empty? + subqueries = condition_position_subqueries(conditions, from_position: from_position) + return from_position || latest_position if subqueries.empty? + + union = subqueries.join(" UNION ") + row = db.fetch("SELECT MAX(position) AS max_pos FROM (#{union})").first + row[:max_pos] || from_position || latest_position + end + + # Build per-condition position subqueries with AND-within/OR-across semantics. + # Resolves key_pair IDs, then builds one SQL subquery per condition. + # Each subquery selects positions where the message matches ALL attrs in the condition. + # + # @param conditions [Array] + # @param from_position [Integer, nil] only include positions after this + # @return [Array] SQL subquery strings (empty if no conditions can match) + def condition_position_subqueries(conditions, from_position: nil) all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq - return from_position || latest_position if all_lookups.empty? + return [] if all_lookups.empty? or_clauses = all_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all @@ -994,7 +980,9 @@ def max_position_for(conditions, from_position: nil) key_pair_index = {} key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } - subqueries = conditions.filter_map do |c| + position_filter = from_position ? "AND m.position > #{db.literal(from_position)}" : "" + + conditions.filter_map do |c| kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } next if kp_ids.size < c.attrs.size @@ -1006,20 +994,11 @@ def max_position_for(conditions, from_position: nil) JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position WHERE m.message_type = #{db.literal(c.message_type)} AND mkp.key_pair_id IN (#{kp_ids_list}) + #{position_filter} GROUP BY m.position HAVING COUNT(DISTINCT mkp.key_pair_id) = #{kp_ids.size} SQL end - - return from_position || latest_position if subqueries.empty? - - union = subqueries.join(" UNION ") - - sql = "SELECT MAX(position) AS max_pos FROM (#{union})" - sql += " WHERE position > #{db.literal(from_position)}" if from_position - - row = db.fetch(sql).first - row[:max_pos] || from_position || latest_position end # Deserialize a database row into a {PositionedMessage}. From bb0c9deca4d7754174a2d0c4020c1b0e3e5c8306 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 17:59:57 +0000 Subject: [PATCH 061/115] Change partition_values from Array to Hash for self-documenting key access Callers can now pass arbitrary kwargs (including values beyond partition_keys) and each key is explicit rather than relying on positional convention. - Decider, Projector, Evolve: initialize defaults to {} instead of [] - handle_claim methods build Hash via .to_h instead of Array via .map - CCC.load scopes partition_attrs from the full values hash - Testing helper passes partition_attrs hash directly - Evolve spec uses constructor instead of instance_variable_set Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc.rb | 10 +++++----- lib/sourced/ccc/README.md | 6 +++--- lib/sourced/ccc/decider.rb | 6 +++--- lib/sourced/ccc/evolve.rb | 6 +++--- lib/sourced/ccc/projector.rb | 8 ++++---- lib/sourced/ccc/testing/rspec.rb | 2 +- spec/sourced/ccc/evolve_spec.rb | 11 +++++++---- spec/sourced/ccc/load_spec.rb | 6 +++--- 8 files changed, 29 insertions(+), 26 deletions(-) diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 583729d1..69a67100 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -163,20 +163,20 @@ def self.handle!(reactor_class, command, store: nil) # @param reactor_class [Class] a CCC reactor class (Decider, Projector, or any class # extending CCC::Consumer that includes CCC::Evolve) # @param store [CCC::Store, nil] the store to read from (defaults to CCC.store) - # @param partition_attrs [Hash{Symbol => String}] partition attribute values + # @param values [Hash{Symbol => String}] partition attribute values # @return [Array(reactor_instance, ReadResult)] # # @example # decider, read_result = Sourced::CCC.load(MyDecider, course_id: 'Algebra', student_id: 'joe') # decider.state # evolved state # read_result.guard # ConsistencyGuard for subsequent appends - def self.load(reactor_class, store: nil, **partition_attrs) + def self.load(reactor_class, store: nil, **values) store ||= self.store + partition_attrs = reactor_class.partition_keys.to_h { |k| [k, values[k]] } handled_types = reactor_class.handled_messages_for_evolve.map(&:type).uniq - read_result = store.read_partition(partition_attrs, handled_types: handled_types) - - values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } + read_result = store.read_partition(partition_attrs, handled_types:) instance = reactor_class.new(values) + instance.evolve(read_result.messages) [instance, read_result] diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 211e375d..62a0b417 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -62,7 +62,7 @@ position = store.append(cmd) # returns the assigned position ```ruby # Build conditions for a specific course conditions = CourseCreated.to_conditions(course_id: 'c1') -# => [QueryCondition('courses.created', 'course_id', 'c1')] +# => [QueryCondition('courses.created', attrs: { course_id: 'c1' })] # Read matching messages result = store.read(conditions) @@ -208,7 +208,7 @@ class CourseDecider < Sourced::CCC::Decider # Defines the consistency boundary partition_by :course_name - # Initial state factory (receives partition values array) + # Initial state factory (receives partition values hash) state do |_partition_values| { name_taken: false } end @@ -335,7 +335,7 @@ class MyProjector < Sourced::CCC::Projector::StateStored state do |partition_values| # Load existing state from your storage - existing = MyDB.find(partition_values.first) + existing = MyDB.find(partition_values[:course_id]) existing || { course_id: nil, students: [] } end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index ebd7047a..09a93dda 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -69,7 +69,7 @@ def handle_batch(partition_values, new_messages, history:) # @param history [ReadResult] event history for the partition # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim, history:) - values = partition_keys.map { |k| claim.partition_value[k.to_s] } + values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } handle_batch(values, claim.messages, history:) end @@ -87,8 +87,8 @@ def inherited(subclass) attr_reader :partition_values - # @param partition_values [Array] values for the decider's partition keys - def initialize(partition_values = []) + # @param partition_values [Hash{Symbol => String}] partition key-value pairs + def initialize(partition_values = {}) @partition_values = partition_values @uncommitted_events = [] end diff --git a/lib/sourced/ccc/evolve.rb b/lib/sourced/ccc/evolve.rb index 8d9c8fc6..f7bda036 100644 --- a/lib/sourced/ccc/evolve.rb +++ b/lib/sourced/ccc/evolve.rb @@ -4,7 +4,7 @@ module Sourced module CCC # Evolve mixin for CCC reactors. # Adapted from Sourced::Evolve for CCC::Message (no stream_id/seq). - # State block receives partition values array instead of stream id. + # State block receives partition values hash instead of stream id. module Evolve PREFIX = 'ccc_evolution' @@ -22,7 +22,7 @@ def state end def partition_values - @partition_values ||= [] + @partition_values ||= {} end # Apply messages to state via registered handlers. @@ -47,7 +47,7 @@ def handled_messages_for_evolve @handled_messages_for_evolve ||= [] end - # Define initial state factory. Block receives partition values array. + # Define initial state factory. Block receives partition values hash. def state(&blk) define_method(:init_state, &blk) end diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index bfd2585c..c99324b7 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -41,8 +41,8 @@ def build_action_pairs(instance, messages, replaying:) attr_reader :partition_values - # @param partition_values [Array] values for the projector's partition keys - def initialize(partition_values = []) + # @param partition_values [Hash{Symbol => String}] partition key-value pairs + def initialize(partition_values = {}) @partition_values = partition_values end @@ -58,7 +58,7 @@ def handle_batch(partition_values, new_messages, replaying: false) # @param claim [ClaimResult] claimed partition batch # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim) - values = partition_keys.map { |k| claim.partition_value[k.to_s] } + values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } handle_batch(values, claim.messages, replaying: claim.replaying) end end @@ -77,7 +77,7 @@ def handle_batch(partition_values, new_messages, history:, replaying: false) # @param history [ReadResult] full partition history # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim, history:) - values = partition_keys.map { |k| claim.partition_value[k.to_s] } + values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } handle_batch(values, claim.messages, history:, replaying: claim.replaying) end end diff --git a/lib/sourced/ccc/testing/rspec.rb b/lib/sourced/ccc/testing/rspec.rb index c64174d4..819a4be8 100644 --- a/lib/sourced/ccc/testing/rspec.rb +++ b/lib/sourced/ccc/testing/rspec.rb @@ -70,7 +70,7 @@ class GWT def initialize(reactor_class, **partition_attrs) @reactor_class = reactor_class @partition_attrs = partition_attrs - @partition_values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } + @partition_values = partition_attrs @given_messages = [] @when_message = nil @asserted = false diff --git a/spec/sourced/ccc/evolve_spec.rb b/spec/sourced/ccc/evolve_spec.rb index 61efe6c5..b9f6fe61 100644 --- a/spec/sourced/ccc/evolve_spec.rb +++ b/spec/sourced/ccc/evolve_spec.rb @@ -23,6 +23,10 @@ module CCCEvolveTestMessages Class.new do include Sourced::CCC::Evolve + def initialize(partition_values = {}) + @partition_values = partition_values + end + state do |partition_values| { items: [], partition_values: partition_values } end @@ -38,10 +42,9 @@ module CCCEvolveTestMessages end describe '.state' do - it 'initializes state with partition values array' do - instance = evolver_class.new - instance.instance_variable_set(:@partition_values, ['val1', 'val2']) - expect(instance.state[:partition_values]).to eq(['val1', 'val2']) + it 'initializes state with partition values hash' do + instance = evolver_class.new(key1: 'val1', key2: 'val2') + expect(instance.state[:partition_values]).to eq({ key1: 'val1', key2: 'val2' }) end end diff --git a/spec/sourced/ccc/load_spec.rb b/spec/sourced/ccc/load_spec.rb index 00d5b030..d0806635 100644 --- a/spec/sourced/ccc/load_spec.rb +++ b/spec/sourced/ccc/load_spec.rb @@ -41,8 +41,8 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored partition_by :course_id consumer_group 'load-test-projector' - state do |(course_id)| - { course_id: course_id, title: nil, student_count: 0 } + state do |partition_values| + { course_id: partition_values[:course_id], title: nil, student_count: 0 } end evolve CCCLoadTestMessages::CourseCreated do |state, evt| @@ -105,7 +105,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored course_id: 'algebra', student_id: 'joe' ) - expect(instance.partition_values).to eq(%w[algebra joe]) + expect(instance.partition_values).to eq({ course_id: 'algebra', student_id: 'joe' }) end it 'read_result contains the messages used for evolution' do From 064f01e9563ee5f66a1d296fba0274b11793ba68 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 21:54:16 +0000 Subject: [PATCH 062/115] Fix composite partition false positives in find_and_claim_partition Add NOT EXISTS clause to find_and_claim_partition so it uses AND semantics directly in SQL. A message only counts as pending for a partition when every attribute name they share has the same value. This prevents stale partitions with shared key_pairs from blocking partitions with real work. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/store.rb | 17 ++++++- spec/sourced/ccc/store_spec.rb | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index c5959936..cc9b1f2c 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -825,8 +825,11 @@ def bootstrap_offsets(cg_id, partition_by) end # Find the next unclaimed partition with pending messages and claim it. - # Uses OR semantics for detection (any matching key_pair is sufficient); - # exact conditional AND filtering happens at fetch time. + # Uses OR joins for candidate detection (any matching key_pair), then + # a NOT EXISTS clause to exclude messages that conflict on a shared + # attribute name (same name, different value). This gives AND semantics: + # a message only counts as pending for a partition when every attribute + # it shares with the partition has the same value. # # @param cg_id [Integer] consumer group internal ID # @param handled_types [Array] message type strings @@ -846,6 +849,16 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) AND o.claimed = 0 AND m.position > o.last_position AND m.message_type IN (#{types_list}) + AND NOT EXISTS ( + SELECT 1 + FROM #{@offset_key_pairs_table} okp2 + JOIN #{@key_pairs_table} kp_part ON okp2.key_pair_id = kp_part.id + JOIN #{@message_key_pairs_table} mkp2 ON mkp2.message_position = m.position + JOIN #{@key_pairs_table} kp_msg ON mkp2.key_pair_id = kp_msg.id + WHERE okp2.offset_id = o.id + AND kp_msg.name = kp_part.name + AND kp_msg.value != kp_part.value + ) GROUP BY o.id ORDER BY next_position ASC LIMIT 1 diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 1a5ecd9d..7e841181 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1416,6 +1416,98 @@ module CCCStoreTestMessages end end + describe '#claim_next (composite partition — cross-partition false positives)' do + let(:group_id) { 'cross-partition-test' } + let(:handled_types) { ['store_test.user.joined_course'] } + + before do + store.register_consumer_group(group_id) + end + + it 'acked partition with shared key_pair does not block other partitions with pending work' do + # Partition by (course_name, user_id). + # Two partitions share course_name='Algebra' but differ on user_id. + # + # After both are processed, a new message arrives for jake. + # joe's acked partition still has a lower last_position, and the OR + # join in find_and_claim_partition matches jake's new message via the + # shared course_name key_pair. joe gets claimed first (lower OR-based + # next_position), fetch_partition_messages (AND semantics) finds nothing, + # and claim_next returns nil — never reaching jake's partition. + + # Step 1: append and process both partitions + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ) + r1 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) + + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) + ) + r2 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r2.offset_id, position: r2.messages.last.position) + + # joe: last_position=1, jake: last_position=2. Both fully caught up. + + # Step 2: new message for jake only + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) + ) + + # Step 3: claim_next should find jake (the partition with real work), + # not return nil because joe's OR-match on course_name got in the way. + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + expect(result).not_to be_nil + expect(result.partition_value).to eq({ 'course_name' => 'Algebra', 'user_id' => 'jake' }) + expect(result.messages.size).to eq(1) + expect(result.messages.first.payload.user_id).to eq('jake') + end + + it 'processes all partitions when many share a key_pair and new messages arrive' do + # Simulates the seat selection scenario: many (showing_id, seat_id) + # partitions share the same showing_id. After processing all initial + # messages, new messages for specific partitions should be claimable + # even though other acked partitions share the showing_id key_pair. + seats = %w[A1 A2 A3 A4 A5] + + # Append and process one message per seat + seats.each do |seat| + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: seat }) + ) + end + seats.size.times do + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + expect(result).not_to be_nil + store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) + end + + # All caught up. Now append new messages for A3 and A5 only. + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'A3' }) + ) + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'A5' }) + ) + + # Should claim A3 and A5, not be blocked by A1/A2/A4's stale OR matches + claimed_users = [] + 2.times do |i| + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + expect(result).not_to be_nil, "Expected partition #{i + 1} of 2 to be claimable but got nil (claimed so far: #{claimed_users})" + claimed_users << result.partition_value['user_id'] + store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) + end + + expect(claimed_users).to contain_exactly('A3', 'A5') + + # No more work + result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') + expect(result).to be_nil + end + end + describe '#ack' do let(:group_id) { 'ack-test' } From 19fb36886233a2c6dd44dc60076aa41dba29214c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 22:32:54 +0000 Subject: [PATCH 063/115] Add index on offsets(consumer_group_id, claimed) for faster partition claiming Lets SQLite jump straight to unclaimed offsets for a consumer group instead of scanning all partitions and filtering claimed=0 in memory. Co-Authored-By: Claude Opus 4.6 --- lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb index 758d452d..01979396 100644 --- a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb +++ b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb @@ -62,6 +62,7 @@ Sequel.migration do String :claimed_by index %i[consumer_group_id partition_key], unique: true, name: 'idx_<%= prefix %>_ccc_offsets_cg_pk' + index %i[consumer_group_id claimed], name: 'idx_<%= prefix %>_ccc_offsets_cg_claimed' end create_table?(:<%= offset_key_pairs_table %>) do From 78d5ad7f44363f9a93c906b59de65c9ef492f916 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 13 Mar 2026 22:38:26 +0000 Subject: [PATCH 064/115] Logline --- lib/sourced/ccc/installer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sourced/ccc/installer.rb b/lib/sourced/ccc/installer.rb index 60707c38..b4aca702 100644 --- a/lib/sourced/ccc/installer.rb +++ b/lib/sourced/ccc/installer.rb @@ -29,7 +29,7 @@ def initialize(db, logger:, prefix: 'sourced', migration_template: '001_create_c # Eval the rendered migration and apply :up directly. def install migration.apply(db, :up) - logger.info("CCC tables installed (prefix: #{prefix})") + logger.info("Sourced tables installed (prefix: #{prefix})") end # Check that all expected tables exist. From b03b4d43541367e4b1e058af927e99ea869e144d Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 10:39:07 +0000 Subject: [PATCH 065/115] CCC::Sync.after_sync actions run after the DB transaction Useful for notifying (pubsub) changes after data is committed to DB --- lib/sourced/ccc/actions.rb | 19 ++++++++++++++++ lib/sourced/ccc/decider.rb | 2 +- lib/sourced/ccc/projector.rb | 2 +- lib/sourced/ccc/router.rb | 10 ++++++++- lib/sourced/ccc/sync.rb | 23 ++++++++++++++++++++ lib/sourced/ccc/testing/rspec.rb | 6 ++++-- spec/sourced/ccc/decider_spec.rb | 26 ++++++++++++++++++++++ spec/sourced/ccc/projector_spec.rb | 13 ++++++++++- spec/sourced/ccc/testing/rspec_spec.rb | 30 +++++++++++++++++++++++--- 9 files changed, 122 insertions(+), 9 deletions(-) diff --git a/lib/sourced/ccc/actions.rb b/lib/sourced/ccc/actions.rb index b3e8c984..455b622c 100644 --- a/lib/sourced/ccc/actions.rb +++ b/lib/sourced/ccc/actions.rb @@ -121,6 +121,25 @@ def execute(_store, _source_message) nil end end + + # Execute a side effect after the transaction commits. + class AfterSync + # @param work [#call] callable to execute + def initialize(work) + @work = work + end + + # @return [Object] the callable's return value + def call = @work.call + + # @param _store [Object] unused + # @param _source_message [Object] unused + # @return [nil] + def execute(_store, _source_message) + call + nil + end + end end end end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index 09a93dda..87b9468c 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -52,7 +52,7 @@ def handle_batch(partition_values, new_messages, history:) actions.concat(Actions.build_for(reaction_msgs, source: evt)) end - actions += instance.sync_actions( + actions += instance.collect_actions( state: instance.state, messages: [msg], events: raw_events ) diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index c99324b7..ec7609e7 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -20,7 +20,7 @@ def handled_messages private def build_action_pairs(instance, messages, replaying:) - sync_actions = instance.sync_actions( + sync_actions = instance.collect_actions( state: instance.state, messages: messages, replaying: replaying ) diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index 76a9680b..c764ff14 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -79,11 +79,17 @@ def drain(limit = Float::INFINITY) private def execute_actions(action_pairs, claim, group_id) + after_sync_actions = [] + store.db.transaction do last_position = nil Array(action_pairs).each do |(actions, source_message)| Array(actions).each do |action| - action.execute(store, source_message) unless action == Actions::OK + if action.is_a?(Actions::AfterSync) + after_sync_actions << action + elsif action != Actions::OK + action.execute(store, source_message) + end end last_position = source_message.position if source_message.respond_to?(:position) end @@ -94,6 +100,8 @@ def execute_actions(action_pairs, claim, group_id) store.release(group_id, offset_id: claim.offset_id) end end + + after_sync_actions.each(&:call) end end end diff --git a/lib/sourced/ccc/sync.rb b/lib/sourced/ccc/sync.rb index 8f17b3d4..f7667f60 100644 --- a/lib/sourced/ccc/sync.rb +++ b/lib/sourced/ccc/sync.rb @@ -17,12 +17,27 @@ def sync_actions(**args) end end + # Build Actions::AfterSync wrappers for all registered after_sync blocks. + def after_sync_actions(**args) + self.class.after_sync_blocks.map do |block| + Actions::AfterSync.new(proc { instance_exec(**args, &block) }) + end + end + + # Build all sync and after_sync actions together. + def collect_actions(**args) + sync_actions(**args) + after_sync_actions(**args) + end + module ClassMethods def inherited(subclass) super sync_blocks.each do |blk| subclass.sync_blocks << blk end + after_sync_blocks.each do |blk| + subclass.after_sync_blocks << blk + end end def sync_blocks @@ -32,6 +47,14 @@ def sync_blocks def sync(&block) sync_blocks << block end + + def after_sync_blocks + @after_sync_blocks ||= [] + end + + def after_sync(&block) + after_sync_blocks << block + end end end end diff --git a/lib/sourced/ccc/testing/rspec.rb b/lib/sourced/ccc/testing/rspec.rb index 819a4be8..5b354076 100644 --- a/lib/sourced/ccc/testing/rspec.rb +++ b/lib/sourced/ccc/testing/rspec.rb @@ -176,7 +176,9 @@ def run_decider_then(sync, *expected, &block) if sync pairs.each do |actions, _| - Array(actions).select { |a| a.is_a?(CCC::Actions::Sync) }.each(&:call) + Array(actions).select { |a| + a.is_a?(CCC::Actions::Sync) || a.is_a?(CCC::Actions::AfterSync) + }.each(&:call) # collect_actions produces both types; filter from pairs end end @@ -220,7 +222,7 @@ def run_projector_then(sync, *_expected, &block) end if sync - instance.sync_actions( + instance.collect_actions( state: instance.state, messages: @given_messages, replaying: false ).each(&:call) end diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/sourced/ccc/decider_spec.rb index be8ea528..07570cb9 100644 --- a/spec/sourced/ccc/decider_spec.rb +++ b/spec/sourced/ccc/decider_spec.rb @@ -68,6 +68,10 @@ class TestDeviceDecider < Sourced::CCC::Decider reaction CCCDeciderTestMessages::DeviceBound do |_state, evt| CCCDeciderTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) end + + after_sync do |state:, messages:, events:| + state[:after_synced] = true + end end class TestDelayedReactionDecider < Sourced::CCC::Decider @@ -192,6 +196,28 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider end end + it 'includes after_sync actions in action pairs' do + reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + history_msgs = [Sourced::CCC::PositionedMessage.new(reg, 1)] + guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 1) + history = Sourced::CCC::ReadResult.new(messages: history_msgs, guard: guard) + + cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + cmd_positioned = Sourced::CCC::PositionedMessage.new(cmd, 2) + + claim = Sourced::CCC::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [cmd_positioned], replaying: false, guard: guard + ) + + pairs = TestDeviceDecider.handle_claim(claim, history: history) + actions = Array(pairs.first.first) + + after_sync_action = actions.find { |a| a.is_a?(Sourced::CCC::Actions::AfterSync) } + expect(after_sync_action).not_to be_nil + end + it 'returns [OK, msg] for non-command messages' do reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) reg_positioned = Sourced::CCC::PositionedMessage.new(reg, 1) diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb index 5e23c006..d56e4b75 100644 --- a/spec/sourced/ccc/projector_spec.rb +++ b/spec/sourced/ccc/projector_spec.rb @@ -47,6 +47,10 @@ class TestItemProjector < Sourced::CCC::Projector::StateStored state[:synced] = true state[:last_replaying] = replaying end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end end class TestItemESProjector < Sourced::CCC::Projector::EventSourced @@ -73,6 +77,10 @@ class TestItemESProjector < Sourced::CCC::Projector::EventSourced state[:synced] = true state[:last_replaying] = replaying end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end end class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored @@ -103,7 +111,7 @@ class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored end describe '.handle_batch (StateStored)' do - it 'evolves from new_messages and includes sync actions' do + it 'evolves from new_messages and includes sync and after_sync actions' do msgs = [ Sourced::CCC::PositionedMessage.new( CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 @@ -121,6 +129,9 @@ class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::Sync) } expect(sync_action).not_to be_nil + + after_sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::AfterSync) } + expect(after_sync_action).not_to be_nil end it 'runs reactions when not replaying' do diff --git a/spec/sourced/ccc/testing/rspec_spec.rb b/spec/sourced/ccc/testing/rspec_spec.rb index 041b0b82..dd43d93e 100644 --- a/spec/sourced/ccc/testing/rspec_spec.rb +++ b/spec/sourced/ccc/testing/rspec_spec.rb @@ -75,6 +75,10 @@ class GWTTestDecider < Sourced::CCC::Decider sync do |state:, messages:, events:| state[:synced] = true end + + after_sync do |state:, messages:, events:| + state[:after_synced] = true + end end # Decider without reactions (produces only events) @@ -99,7 +103,7 @@ class GWTTestStateStoredProjector < Sourced::CCC::Projector::StateStored consumer_group 'gwt-test-ss-projector' state do |(list_id)| - { list_id: list_id, items: [], synced: false } + { list_id: list_id, items: [], synced: false, after_synced: false } end evolve CCCGWTTestMessages::ItemAdded do |state, msg| @@ -113,6 +117,10 @@ class GWTTestStateStoredProjector < Sourced::CCC::Projector::StateStored sync do |state:, messages:, replaying:| state[:synced] = true end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end end class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced @@ -120,7 +128,7 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced consumer_group 'gwt-test-es-projector' state do |(list_id)| - { list_id: list_id, items: [], synced: false } + { list_id: list_id, items: [], synced: false, after_synced: false } end evolve CCCGWTTestMessages::ItemAdded do |state, msg| @@ -134,6 +142,10 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced sync do |state:, messages:, replaying:| state[:synced] = true end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end end RSpec.describe Sourced::CCC::Testing::RSpec do @@ -182,7 +194,7 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced .then([]) end - it 'then! runs sync actions' do + it 'then! runs sync and after_sync actions' do with_reactor(GWTTestDecider, device_id: 'd1') .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') @@ -232,6 +244,12 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced .then! { |state| expect(state[:synced]).to be true } end + it 'then! runs after_sync actions before yielding state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |state| expect(state[:after_synced]).to be true } + end + it 'given events with archive → state reflects removal' do with_reactor(GWTTestStateStoredProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') @@ -269,6 +287,12 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced .then! { |state| expect(state[:synced]).to be true } end + it 'then! runs after_sync actions before yielding state' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |state| expect(state[:after_synced]).to be true } + end + it '.when raises ArgumentError' do expect { with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') From 68f55cef701c5a838b161a799bd89daeeb3ab2e6 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 10:41:50 +0000 Subject: [PATCH 066/115] Document --- lib/sourced/ccc/README.md | 45 +++++++++++++++++++++++++++++++++++++-- lib/sourced/ccc/sync.rb | 36 ++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 62a0b417..83d1bc9f 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -322,6 +322,13 @@ class CourseCatalogProjector < Sourced::CCC::Projector::EventSourced # Write projection to disk, database, cache, etc. File.write("projections/#{state[:course_id]}.json", state.to_json) end + + # After-sync block runs after the transaction commits. + # Use for side effects that should only happen on successful commit + # (e.g. sending emails, HTTP calls, pushing to external queues). + after_sync do |state:, messages:, **| + NotificationService.notify("Course #{state[:course_name]} updated") + end end ``` @@ -368,6 +375,40 @@ end Reactions are skipped during replay (when `replaying: true`), so side effects don't re-fire. +## Sync and After-Sync Blocks + +Both deciders and projectors support `sync` and `after_sync` blocks for running side effects during message processing. + +- **`sync`** blocks run **inside** the store transaction, alongside event persistence and offset acknowledgement. Use them for writes that must be atomic with the event append (e.g. updating a database projection). +- **`after_sync`** blocks run **after** the transaction commits. Use them for side effects that should only happen if the commit succeeds (e.g. sending emails, HTTP calls, pushing to external queues). + +Both receive the same keyword arguments as the reactor's action-building step: + +| Reactor type | Keyword arguments | +|--------------|---------------------------------------| +| Decider | `state:`, `messages:`, `events:` | +| Projector | `state:`, `messages:`, `replaying:` | + +```ruby +class OrderDecider < Sourced::CCC::Decider + partition_by :order_id + + # ... evolve / command handlers ... + + sync do |state:, messages:, events:| + # Runs inside the transaction + OrderCache.update(state[:order_id], state) + end + + after_sync do |state:, messages:, events:| + # Runs after successful commit + Mailer.send_confirmation(state[:order_id]) if events.any? { |e| e.is_a?(OrderPlaced) } + end +end +``` + +Multiple `sync` and `after_sync` blocks can be registered; they execute in registration order. Blocks are inherited by subclasses. + ## Configuration ```ruby @@ -663,9 +704,9 @@ it 'inspects action pairs' do end ``` -#### `.then!` — run sync actions +#### `.then!` — run sync and after_sync actions -Use `.then!` instead of `.then` to execute sync actions before assertions: +Use `.then!` instead of `.then` to execute both `sync` and `after_sync` actions before assertions: ```ruby it 'runs sync block' do diff --git a/lib/sourced/ccc/sync.rb b/lib/sourced/ccc/sync.rb index f7667f60..bbc95ecf 100644 --- a/lib/sourced/ccc/sync.rb +++ b/lib/sourced/ccc/sync.rb @@ -3,21 +3,28 @@ module Sourced module CCC # Sync mixin for CCC reactors. - # Registers blocks that run within the store transaction. + # Registers blocks that run within the store transaction (+sync+) + # or after the transaction commits (+after_sync+). module Sync def self.included(base) super base.extend ClassMethods end - # Build Actions::Sync wrappers for all registered sync blocks. + # Build {Actions::Sync} wrappers for all registered +sync+ blocks. + # + # @param args [Hash] keyword arguments forwarded to each block + # @return [Array] def sync_actions(**args) self.class.sync_blocks.map do |block| Actions::Sync.new(proc { instance_exec(**args, &block) }) end end - # Build Actions::AfterSync wrappers for all registered after_sync blocks. + # Build {Actions::AfterSync} wrappers for all registered +after_sync+ blocks. + # + # @param args [Hash] keyword arguments forwarded to each block + # @return [Array] def after_sync_actions(**args) self.class.after_sync_blocks.map do |block| Actions::AfterSync.new(proc { instance_exec(**args, &block) }) @@ -25,11 +32,15 @@ def after_sync_actions(**args) end # Build all sync and after_sync actions together. + # + # @param args [Hash] keyword arguments forwarded to each block + # @return [Array] def collect_actions(**args) sync_actions(**args) + after_sync_actions(**args) end module ClassMethods + # @api private def inherited(subclass) super sync_blocks.each do |blk| @@ -40,18 +51,37 @@ def inherited(subclass) end end + # @return [Array] registered sync blocks def sync_blocks @sync_blocks ||= [] end + # Register a block to run inside the store transaction. + # + # The block receives the same keyword arguments as the reactor's + # action-building step (e.g. +state:+, +messages:+, +events:+ + # for deciders, or +state:+, +messages:+, +replaying:+ for projectors). + # + # @yield [**args] side-effect block executed within the transaction + # @return [void] def sync(&block) sync_blocks << block end + # @return [Array] registered after_sync blocks def after_sync_blocks @after_sync_blocks ||= [] end + # Register a block to run after the store transaction commits. + # + # Use this for side effects that should only happen on successful + # commit (e.g. sending emails, HTTP calls, pushing to external queues). + # + # The block receives the same keyword arguments as +sync+. + # + # @yield [**args] side-effect block executed after transaction commit + # @return [void] def after_sync(&block) after_sync_blocks << block end From c4137bb844a3376f2ee491c8231a965de650892f Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 16:42:03 +0000 Subject: [PATCH 067/115] Add missing specs for CCC::Message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover #with_metadata, #at, Registry#keys, Payload#[]/#fetch, default payload initialization, ConsistencyGuard, and Command/Event subclass registry delegation. (26 → 47 examples) Co-Authored-By: Claude Opus 4.6 (1M context) --- spec/sourced/ccc/message_spec.rb | 128 +++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index 848ce39f..2e2cef12 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -220,6 +220,134 @@ module CCCTestMessages end end + describe '#with_metadata' do + it 'merges new metadata into existing metadata' do + msg = CCCTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + updated = msg.with_metadata(request_id: 'req-1') + expect(updated.metadata).to eq({ user_id: 42, request_id: 'req-1' }) + end + + it 'returns self when given empty hash' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.with_metadata({})).to equal(msg) + end + + it 'does not mutate the original message' do + msg = CCCTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + updated = msg.with_metadata(request_id: 'req-1') + expect(msg.metadata).to eq({ user_id: 42 }) + expect(updated).not_to equal(msg) + end + end + + describe '#at' do + it 'returns new message with updated created_at' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + future = msg.created_at + 3600 + updated = msg.at(future) + expect(updated.created_at).to eq(future) + expect(updated).not_to equal(msg) + end + + it 'raises PastMessageDateError when given a past time' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + past = msg.created_at - 3600 + expect { msg.at(past) }.to raise_error(Sourced::PastMessageDateError) + end + + it 'does not mutate the original message' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + original_time = msg.created_at + msg.at(msg.created_at + 3600) + expect(msg.created_at).to eq(original_time) + end + end + + describe 'Registry' do + describe '#keys' do + it 'returns array of registered type strings' do + keys = Sourced::CCC::Message.registry.keys + expect(keys).to include('device.registered', 'asset.registered', 'system.updated') + end + end + end + + describe 'Payload' do + let(:msg) { CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) } + + describe '#[]' do + it 'returns attribute value by symbol key' do + expect(msg.payload[:device_id]).to eq('dev-1') + expect(msg.payload[:name]).to eq('Sensor A') + end + end + + describe '#fetch' do + it 'returns attribute value for existing key' do + expect(msg.payload.fetch(:device_id)).to eq('dev-1') + end + + it 'raises KeyError for missing key' do + expect { msg.payload.fetch(:missing) }.to raise_error(KeyError) + end + + it 'supports default value' do + expect(msg.payload.fetch(:missing, 'default')).to eq('default') + end + + it 'supports block fallback' do + expect(msg.payload.fetch(:missing) { 'from_block' }).to eq('from_block') + end + end + end + + describe '#initialize default payload' do + it 'creates message without explicit payload arg' do + bare = Sourced::CCC::Message.define('test.init_default') + msg = bare.new + expect(msg.payload).to be_nil + expect(msg.id).not_to be_nil + expect(msg.type).to eq('test.init_default') + end + end + + describe Sourced::CCC::ConsistencyGuard do + it 'is a Data struct with conditions and last_position' do + conditions = [Sourced::CCC::QueryCondition.new(message_type: 'device.registered', attrs: { device_id: 'dev-1' })] + guard = Sourced::CCC::ConsistencyGuard.new(conditions: conditions, last_position: 42) + expect(guard.conditions).to eq(conditions) + expect(guard.last_position).to eq(42) + end + end + + describe 'Command and Event subclass registries' do + let!(:test_cmd) { Sourced::CCC::Command.define('test.do_something') { attribute :name, String } } + let!(:test_evt) { Sourced::CCC::Event.define('test.something_happened') { attribute :name, String } } + + it 'registers Command types in Command registry' do + expect(Sourced::CCC::Command.registry['test.do_something']).to eq(test_cmd) + end + + it 'registers Event types in Event registry' do + expect(Sourced::CCC::Event.registry['test.something_happened']).to eq(test_evt) + end + + it 'does not register Command types in Event registry' do + expect(Sourced::CCC::Event.registry['test.do_something']).to be_nil + end + + it 'Message.registry can look up types from subclass registries' do + expect(Sourced::CCC::Message.registry['test.do_something']).to eq(test_cmd) + expect(Sourced::CCC::Message.registry['test.something_happened']).to eq(test_evt) + end + end + describe Sourced::CCC::QueryCondition do it 'is a Data struct with message_type and attrs hash' do cond = Sourced::CCC::QueryCondition.new( From c3a782523acd944e9569539e407cd98947099b5e Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 16:48:33 +0000 Subject: [PATCH 068/115] Add specs for CCC::Message#with_payload Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/message.rb | 6 ++++++ spec/sourced/ccc/message_spec.rb | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 4bdd05f0..6592de16 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -149,6 +149,12 @@ def with_metadata(meta = {}) with(metadata: metadata.merge(meta)) end + def with_payload(attrs = {}) + hash = to_h + (hash[:payload] ||= {}).merge!(attrs) + self.class.new(hash) + end + def at(datetime) if datetime < created_at raise Sourced::PastMessageDateError, "Message #{type} can't be delayed to a date in the past" diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index 2e2cef12..e146d892 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -246,6 +246,41 @@ module CCCTestMessages end end + describe '#with_payload' do + it 'merges new attributes into existing payload' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + updated = msg.with_payload(name: 'Sensor B') + expect(updated.payload.device_id).to eq('dev-1') + expect(updated.payload.name).to eq('Sensor B') + end + + it 'preserves id and other attributes' do + msg = CCCTestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + updated = msg.with_payload(name: 'Sensor B') + expect(updated.id).to eq(msg.id) + expect(updated.metadata).to eq({ user_id: 42 }) + expect(updated.type).to eq('device.registered') + end + + it 'returns a new instance without mutating the original' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + updated = msg.with_payload(name: 'Sensor B') + expect(updated).not_to equal(msg) + expect(msg.payload.name).to eq('Sensor A') + end + + it 'works with empty hash (returns equivalent copy)' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + updated = msg.with_payload({}) + expect(updated.payload.device_id).to eq('dev-1') + expect(updated.payload.name).to eq('Sensor A') + expect(updated).not_to equal(msg) + end + end + describe '#at' do it 'returns new message with updated created_at' do msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) From e911969482ab414268b6ef96552f64dde6251f18 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 17:14:04 +0000 Subject: [PATCH 069/115] Add per-message and `any` callback hooks to CCC::CommandContext Class-level `on(MessageClass)` and `any` DSL methods allow subclasses to enrich/transform commands at build time (e.g. injecting session data, adding request metadata). Blocks receive the request-scoped `app` object and the command, run in order (on before any), and are inherited by subclasses. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/README.md | 41 ++++++++ lib/sourced/ccc/command_context.rb | 112 ++++++++++++++++++---- spec/sourced/ccc/command_context_spec.rb | 114 +++++++++++++++++++++++ 3 files changed, 251 insertions(+), 16 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 83d1bc9f..74351519 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -266,6 +266,47 @@ cmd = ctx.build(CreateCourse, payload: { course_id: 'c1', course_name: 'Algebra' String keys are automatically symbolized, so `ctx.build('type' => '...', 'payload' => { ... })` works too. +#### Callback hooks (`on` and `any`) + +Subclass `CommandContext` and register class-level hooks to enrich or transform commands at build time — e.g. injecting session data or adding metadata from the request scope. + +- **`on(MessageClass)`** — runs for a specific command type (one block per class, last-write-wins) +- **`any`** — runs for all commands (multiple blocks allowed, executed in order) + +Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. + +```ruby +class AppCommandContext < Sourced::CCC::CommandContext + # Enrich a specific command with data from the app scope + on CreateCourse do |app, cmd| + cmd.with_payload(created_by: app.current_user.id) + end + + # Add metadata to every command + any do |app, cmd| + cmd.with_metadata( + request_id: app.request_id, + session_id: app.session_id + ) + end +end +``` + +Pass the request-scoped `app` object at construction time: + +```ruby +# In a web controller +ctx = AppCommandContext.new( + metadata: { user_id: session[:user_id] }, + app: self # e.g. Sinatra app instance, Rack env wrapper, etc. +) + +cmd = ctx.build(type: 'courses.create', payload: { course_id: 'c1', course_name: 'Algebra' }) +cmd.metadata[:request_id] # => set by the `any` hook +``` + +`app` defaults to `nil`, so existing callers without hooks are unaffected. Hooks are inherited by subclasses. + #### Scoping to a command subset By default, `CommandContext` looks up types in `CCC::Command.registry`. Pass a `scope:` to restrict lookups to a specific command subclass — attempts to build commands outside the scope raise `Sourced::UnknownMessageError`. diff --git a/lib/sourced/ccc/command_context.rb b/lib/sourced/ccc/command_context.rb index 53715b1f..96716056 100644 --- a/lib/sourced/ccc/command_context.rb +++ b/lib/sourced/ccc/command_context.rb @@ -4,32 +4,112 @@ module Sourced module CCC + # Builds CCC command instances with shared default metadata. + # + # @example Build a command from a type string + # ctx = CommandContext.new(metadata: { user_id: 10 }) + # cmd = ctx.build(type: 'orders.place', payload: { item: 'hat' }) + # cmd.metadata[:user_id] # => 10 + # + # @example Scope to a custom command subclass + # scope = Class.new(CCC::Command) + # MyCmd = scope.define('my.cmd') { attribute :name, String } + # ctx = CommandContext.new(scope: scope) + # cmd = ctx.build(type: 'my.cmd', payload: { name: 'hello' }) class CommandContext - # @option metadata [Hash] metadata to add to commands built by this context - # @option scope [CCC::Message] Message class to use as command registry - def initialize(metadata: Plumb::BLANK_HASH, scope: CCC::Command) + class << self + # Register a block to run when building a specific command type. + # The block receives the app scope and the command, and must return the (possibly modified) command. + # + # @param message_class [Class] the command class to match + # @yield [app, cmd] transformation block + # @return [void] + def on(message_class, &block) + message_blocks[message_class] = block + end + + # Register a block to run for all commands. + # The block receives the app scope and the command, and must return the (possibly modified) command. + # + # @yield [app, cmd] transformation block + # @return [void] + def any(&block) + any_blocks << block + end + + # @api private + def message_blocks + @message_blocks ||= {} + end + + # @api private + def any_blocks + @any_blocks ||= [] + end + + # @api private + def inherited(subclass) + super + message_blocks.each { |k, v| subclass.message_blocks[k] = v } + any_blocks.each { |blk| subclass.any_blocks << blk } + end + end + + # @param metadata [Hash] default metadata merged into every command built by this context + # @param scope [Class] message class whose registry is used to resolve type strings (default: {CCC::Command}) + # @param app [Object, nil] request-scoped object passed to callback blocks + # + # @example + # ctx = CommandContext.new(metadata: { user_id: 42 }, app: rack_app) + def initialize(metadata: Plumb::BLANK_HASH, scope: CCC::Command, app: nil) @defaults = { metadata: }.freeze @scope = scope + @app = app end - # @param args [Array] either [Hash] or [Class, Hash] - # @return [CCC::Message] + # Build a command instance, merging in default metadata. + # + # @overload build(attrs) + # Resolve the command class from the +type+ key in +attrs+ via the scope's registry. + # @param attrs [Hash] must include +:type+ and +:payload+ keys + # @return [CCC::Message] + # @example + # ctx.build(type: 'orders.place', payload: { item: 'hat' }) + # + # @overload build(klass, attrs) + # Use the given command class directly. + # @param klass [Class] a CCC::Command subclass + # @param attrs [Hash] must include +:payload+ key + # @return [CCC::Message] + # @example + # ctx.build(PlaceOrder, payload: { item: 'hat' }) + # + # @raise [ArgumentError] if arguments don't match either form + # @raise [Sourced::UnknownMessageError] if the type string is not registered in the scope def build(*args) - case args - in [Class => klass, Hash => attrs] - attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) - klass.parse(attrs) - in [Hash => attrs] - attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) - scope.from(attrs) - else - raise ArgumentError, "Invalid arguments: #{args.inspect}" - end + cmd = case args + in [Class => klass, Hash => attrs] + attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) + klass.parse(attrs) + in [Hash => attrs] + attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) + scope.from(attrs) + else + raise ArgumentError, "Invalid arguments: #{args.inspect}" + end + run_pipeline(cmd) end private - attr_reader :defaults, :scope + attr_reader :defaults, :scope, :app + + def run_pipeline(cmd) + block = self.class.message_blocks[cmd.class] + cmd = block.call(app, cmd) if block + self.class.any_blocks.each { |blk| cmd = blk.call(app, cmd) } + cmd + end end end end diff --git a/spec/sourced/ccc/command_context_spec.rb b/spec/sourced/ccc/command_context_spec.rb index 1eb09191..029a595c 100644 --- a/spec/sourced/ccc/command_context_spec.rb +++ b/spec/sourced/ccc/command_context_spec.rb @@ -8,6 +8,10 @@ module CccContextTest attribute :value, Integer end + Remove = Sourced::CCC::Command.define('ccc_ctest.remove') do + attribute :value, Integer + end + Added = Sourced::CCC::Event.define('ccc_ctest.added') end @@ -64,4 +68,114 @@ module CccContextTest expect(cmd.metadata[:user_id]).to eq(10) end end + + describe 'callback hooks' do + it 'runs on block for matching command type' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.payload.value).to eq(11) + end + + it 'does not run on block for non-matching command type' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Remove, payload: { value: 1 }) + expect(cmd.payload.value).to eq(1) + end + + it 'passes app scope to on block' do + app = double('app', session_id: 'abc') + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |a, cmd| cmd.with_metadata(session_id: a.session_id) } + + ctx = klass.new(app: app) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:session_id]).to eq('abc') + end + + it 'runs any block for all commands' do + klass = Class.new(described_class) + klass.any { |_app, cmd| cmd.with_metadata(source: 'web') } + + ctx = klass.new + add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) + expect(add_cmd.metadata[:source]).to eq('web') + expect(remove_cmd.metadata[:source]).to eq('web') + end + + it 'runs on before any (pipeline order)' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_metadata(step: 'on') } + klass.any { |_app, cmd| cmd.with_metadata(step: "#{cmd.metadata[:step]}_any") } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:step]).to eq('on_any') + end + + it 'runs multiple any blocks in order' do + klass = Class.new(described_class) + klass.any { |_app, cmd| cmd.with_metadata(steps: ['first']) } + klass.any { |_app, cmd| cmd.with_metadata(steps: cmd.metadata[:steps] + ['second']) } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:steps]).to eq(%w[first second]) + end + + it 'passes through unchanged when no hooks registered' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.payload.value).to eq(1) + expect(cmd.metadata[:user_id]).to eq(10) + end + + it 'app defaults to nil' do + klass = Class.new(described_class) + received_app = :not_set + klass.any { |a, cmd| received_app = a; cmd } + + ctx = klass.new + ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(received_app).to be_nil + end + + it 'subclass inherits parent blocks' do + parent = Class.new(described_class) + parent.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 100) } + parent.any { |_app, cmd| cmd.with_metadata(inherited: true) } + + child = Class.new(parent) + + ctx = child.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.payload.value).to eq(101) + expect(cmd.metadata[:inherited]).to eq(true) + end + + it 'subclass blocks do not affect parent' do + parent = Class.new(described_class) + child = Class.new(parent) + child.any { |_app, cmd| cmd.with_metadata(child_only: true) } + + ctx = parent.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata).not_to have_key(:child_only) + end + + it 'works with build from type string + on block' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 5) } + + ctx = klass.new + cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) + expect(cmd.payload.value).to eq(6) + end + end end From 5fdcf72c6b44e3218735b6be47401bdae578d362 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 17:22:28 +0000 Subject: [PATCH 070/115] Support multiple message types in CommandContext.on `on` now accepts variadic arguments, registering the same block for each class: `on(Cmd1, Cmd2) { |app, cmd| ... }` Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/README.md | 7 ++++++- lib/sourced/ccc/command_context.rb | 8 ++++---- spec/sourced/ccc/command_context_spec.rb | 11 +++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 74351519..96b15c66 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -270,7 +270,7 @@ String keys are automatically symbolized, so `ctx.build('type' => '...', 'payloa Subclass `CommandContext` and register class-level hooks to enrich or transform commands at build time — e.g. injecting session data or adding metadata from the request scope. -- **`on(MessageClass)`** — runs for a specific command type (one block per class, last-write-wins) +- **`on(MessageClass, ...)`** — runs for one or more command types (one block per class, last-write-wins) - **`any`** — runs for all commands (multiple blocks allowed, executed in order) Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. @@ -282,6 +282,11 @@ class AppCommandContext < Sourced::CCC::CommandContext cmd.with_payload(created_by: app.current_user.id) end + # Same block for multiple command types + on EnrolStudent, DropStudent do |app, cmd| + cmd.with_metadata(campus: app.current_campus) + end + # Add metadata to every command any do |app, cmd| cmd.with_metadata( diff --git a/lib/sourced/ccc/command_context.rb b/lib/sourced/ccc/command_context.rb index 96716056..bad1a861 100644 --- a/lib/sourced/ccc/command_context.rb +++ b/lib/sourced/ccc/command_context.rb @@ -18,14 +18,14 @@ module CCC # cmd = ctx.build(type: 'my.cmd', payload: { name: 'hello' }) class CommandContext class << self - # Register a block to run when building a specific command type. + # Register a block to run when building specific command type(s). # The block receives the app scope and the command, and must return the (possibly modified) command. # - # @param message_class [Class] the command class to match + # @param message_classes [Class] one or more command classes to match # @yield [app, cmd] transformation block # @return [void] - def on(message_class, &block) - message_blocks[message_class] = block + def on(*message_classes, &block) + message_classes.each { |klass| message_blocks[klass] = block } end # Register a block to run for all commands. diff --git a/spec/sourced/ccc/command_context_spec.rb b/spec/sourced/ccc/command_context_spec.rb index 029a595c..2e23ac79 100644 --- a/spec/sourced/ccc/command_context_spec.rb +++ b/spec/sourced/ccc/command_context_spec.rb @@ -79,6 +79,17 @@ module CccContextTest expect(cmd.payload.value).to eq(11) end + it 'registers on block for multiple command types' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add, CccContextTest::Remove) { |_app, cmd| cmd.with_metadata(tagged: true) } + + ctx = klass.new + add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) + expect(add_cmd.metadata[:tagged]).to eq(true) + expect(remove_cmd.metadata[:tagged]).to eq(true) + end + it 'does not run on block for non-matching command type' do klass = Class.new(described_class) klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } From 2c06711621e3dfd3b17bdb0d8f6b0befd68b1406 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 17:26:27 +0000 Subject: [PATCH 071/115] Allow multiple `on` blocks per command type in CommandContext `on` now accumulates blocks instead of overwriting, so separate `on` calls for the same class all run in registration order. The internal registry is Hash>. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/README.md | 7 ++++++- lib/sourced/ccc/command_context.rb | 8 ++++---- spec/sourced/ccc/command_context_spec.rb | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 96b15c66..440ac03c 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -270,7 +270,7 @@ String keys are automatically symbolized, so `ctx.build('type' => '...', 'payloa Subclass `CommandContext` and register class-level hooks to enrich or transform commands at build time — e.g. injecting session data or adding metadata from the request scope. -- **`on(MessageClass, ...)`** — runs for one or more command types (one block per class, last-write-wins) +- **`on(MessageClass, ...)`** — runs for one or more command types. Multiple `on` calls for the same class accumulate (all blocks run in registration order). - **`any`** — runs for all commands (multiple blocks allowed, executed in order) Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. @@ -287,6 +287,11 @@ class AppCommandContext < Sourced::CCC::CommandContext cmd.with_metadata(campus: app.current_campus) end + # Additional block for EnrolStudent — both blocks run in order + on EnrolStudent do |app, cmd| + cmd.with_metadata(enrolment_source: 'web') + end + # Add metadata to every command any do |app, cmd| cmd.with_metadata( diff --git a/lib/sourced/ccc/command_context.rb b/lib/sourced/ccc/command_context.rb index bad1a861..db5821f8 100644 --- a/lib/sourced/ccc/command_context.rb +++ b/lib/sourced/ccc/command_context.rb @@ -25,7 +25,7 @@ class << self # @yield [app, cmd] transformation block # @return [void] def on(*message_classes, &block) - message_classes.each { |klass| message_blocks[klass] = block } + message_classes.each { |klass| (message_blocks[klass] ||= []) << block } end # Register a block to run for all commands. @@ -50,7 +50,7 @@ def any_blocks # @api private def inherited(subclass) super - message_blocks.each { |k, v| subclass.message_blocks[k] = v } + message_blocks.each { |k, v| subclass.message_blocks[k] = v.dup } any_blocks.each { |blk| subclass.any_blocks << blk } end end @@ -105,8 +105,8 @@ def build(*args) attr_reader :defaults, :scope, :app def run_pipeline(cmd) - block = self.class.message_blocks[cmd.class] - cmd = block.call(app, cmd) if block + blocks = self.class.message_blocks[cmd.class] + blocks&.each { |blk| cmd = blk.call(app, cmd) } self.class.any_blocks.each { |blk| cmd = blk.call(app, cmd) } cmd end diff --git a/spec/sourced/ccc/command_context_spec.rb b/spec/sourced/ccc/command_context_spec.rb index 2e23ac79..3f4775d1 100644 --- a/spec/sourced/ccc/command_context_spec.rb +++ b/spec/sourced/ccc/command_context_spec.rb @@ -90,6 +90,23 @@ module CccContextTest expect(remove_cmd.metadata[:tagged]).to eq(true) end + it 'accumulates multiple on blocks for the same command type' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add, CccContextTest::Remove) { |_app, cmd| cmd.with_metadata(first: true) } + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_metadata(second: true) } + + ctx = klass.new + # Add gets both blocks + add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(add_cmd.metadata[:first]).to eq(true) + expect(add_cmd.metadata[:second]).to eq(true) + + # Remove only gets the first block + remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) + expect(remove_cmd.metadata[:first]).to eq(true) + expect(remove_cmd.metadata).not_to have_key(:second) + end + it 'does not run on block for non-matching command type' do klass = Class.new(described_class) klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } From 36db6d11397d045c6025e65c2f8b3f499baf9f18 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sat, 14 Mar 2026 18:01:52 +0000 Subject: [PATCH 072/115] Run CCC::CommandContext hooks in context of instance So that custom class can have helper methods --- lib/sourced/ccc/README.md | 18 ++++++++++++- lib/sourced/ccc/command_context.rb | 9 ++++--- spec/sourced/ccc/command_context_spec.rb | 33 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 440ac03c..56f974f9 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -273,7 +273,7 @@ Subclass `CommandContext` and register class-level hooks to enrich or transform - **`on(MessageClass, ...)`** — runs for one or more command types. Multiple `on` calls for the same class accumulate (all blocks run in registration order). - **`any`** — runs for all commands (multiple blocks allowed, executed in order) -Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. +Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. Blocks are evaluated in the context of the `CommandContext` instance, so they can call private helper methods defined on the subclass. ```ruby class AppCommandContext < Sourced::CCC::CommandContext @@ -317,6 +317,22 @@ cmd.metadata[:request_id] # => set by the `any` hook `app` defaults to `nil`, so existing callers without hooks are unaffected. Hooks are inherited by subclasses. +Since blocks run in instance context, you can extract shared logic into private methods: + +```ruby +class AppCommandContext < Sourced::CCC::CommandContext + on CreateCourse do |app, cmd| + cmd.with_metadata(user_id: build_user_id(app)) + end + + private + + def build_user_id(app) + "user-#{app.session_id}" + end +end +``` + #### Scoping to a command subset By default, `CommandContext` looks up types in `CCC::Command.registry`. Pass a `scope:` to restrict lookups to a specific command subclass — attempts to build commands outside the scope raise `Sourced::UnknownMessageError`. diff --git a/lib/sourced/ccc/command_context.rb b/lib/sourced/ccc/command_context.rb index db5821f8..dabaa4c5 100644 --- a/lib/sourced/ccc/command_context.rb +++ b/lib/sourced/ccc/command_context.rb @@ -102,13 +102,14 @@ def build(*args) private + EMPTY_ARRAY = [].freeze + attr_reader :defaults, :scope, :app def run_pipeline(cmd) - blocks = self.class.message_blocks[cmd.class] - blocks&.each { |blk| cmd = blk.call(app, cmd) } - self.class.any_blocks.each { |blk| cmd = blk.call(app, cmd) } - cmd + blocks = self.class.message_blocks[cmd.class] || EMPTY_ARRAY + steps = blocks + self.class.any_blocks + steps.reduce(cmd) { |c, st| instance_exec(app, c, &st) } end end end diff --git a/spec/sourced/ccc/command_context_spec.rb b/spec/sourced/ccc/command_context_spec.rb index 3f4775d1..84ce357c 100644 --- a/spec/sourced/ccc/command_context_spec.rb +++ b/spec/sourced/ccc/command_context_spec.rb @@ -205,5 +205,38 @@ module CccContextTest cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) expect(cmd.payload.value).to eq(6) end + + it 'runs on blocks in the context of the instance' do + klass = Class.new(described_class) do + on(CccContextTest::Add) { |app, cmd| cmd.with_metadata(user_id: build_user_id(app)) } + + private + + def build_user_id(app) + "user-#{app.session_id}" + end + end + + app = double('app', session_id: '42') + ctx = klass.new(app: app) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:user_id]).to eq('user-42') + end + + it 'runs any blocks in the context of the instance' do + klass = Class.new(described_class) do + any { |app, cmd| cmd.with_metadata(source: request_source) } + + private + + def request_source + 'web' + end + end + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:source]).to eq('web') + end end end From fa9a31dc26a7574c6a8e76cb29337a8929abb647 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 14:52:33 +0000 Subject: [PATCH 073/115] Lazy offset discovery in CCC::Store#claim_next Replace bootstrap_offsets (which scanned ALL messages on every call) with a two-phase claim: fast path tries existing offsets first, then discovery scans forward from a per-consumer-group watermark to find new partitions. This avoids re-scanning earlier messages and bounds work per call via DISCOVERY_BATCH_SIZE. - Add discovery_position column to consumer_groups table - Add discover_new_partitions with watermark-based forward scanning - Add ensure_offset_for_partition for targeted offset creation - Extract create_offset_with_key_pairs shared helper - Update advance_offset to use ensure_offset_for_partition - Reset discovery_position on consumer group reset - Remove bootstrap_offsets method Co-Authored-By: Claude Opus 4.6 (1M context) --- .../migrations/001_create_ccc_tables.rb.erb | 1 + lib/sourced/ccc/store.rb | 175 +++++++++++++----- spec/sourced/ccc/store_spec.rb | 124 ++++++++++++- 3 files changed, 249 insertions(+), 51 deletions(-) diff --git a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb index 01979396..6dd5b92f 100644 --- a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb +++ b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb @@ -46,6 +46,7 @@ Sequel.migration do String :group_id, null: false, unique: true String :status, null: false, default: 'active' Integer :highest_position, null: false, default: 0 + Integer :discovery_position, null: false, default: 0 String :error_context String :retry_at String :created_at, null: false diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index cc9b1f2c..f4fbc4d0 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -66,6 +66,7 @@ class Store ACTIVE = 'active' STOPPED = 'stopped' FAILED = 'failed' + DISCOVERY_BATCH_SIZE = 100 # @return [Sequel::SQLite::Database] attr_reader :db @@ -87,6 +88,7 @@ def initialize(db, notifier: nil, logger: nil, prefix: 'sourced') @db = db @notifier = notifier || Sourced::InlineNotifier.new @logger = logger || Sourced.config.logger + Sequel.extension(:fiber_concurrency) @db.run('PRAGMA foreign_keys = ON') @db.run('PRAGMA journal_mode = WAL') @db.run('PRAGMA busy_timeout = 5000') @@ -179,6 +181,7 @@ def append(messages, guard: nil) end end + Console.info "AAA append #{messages.map(&:type).uniq}" notifier.notify_new_messages(messages.map(&:type).uniq) last_position @@ -457,6 +460,10 @@ def reset_consumer_group(group_id) return unless cg db[@offsets_table].where(consumer_group_id: cg[:id]).delete + db[@consumer_groups_table].where(id: cg[:id]).update( + discovery_position: 0, + updated_at: Time.now.iso8601 + ) end # Claim the next available partition for processing. @@ -489,9 +496,15 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: .first return nil unless cg - bootstrap_offsets(cg[:id], partition_by) - + # Phase 1: Fast path — try existing offsets claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + + # Phase 2: Discovery — scan forward from watermark, create new offsets + unless claimed + discover_new_partitions(cg[:id], partition_by, handled_types) + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end + return nil unless claimed key_pair_ids = db[@offset_key_pairs_table] @@ -635,15 +648,13 @@ def advance_offset(group_id, partition:, position:) cg = db[@consumer_groups_table].where(group_id: group_id).first return unless cg - partition_by = partition.keys.sort - bootstrap_offsets(cg[:id], partition_by) + offset_id = ensure_offset_for_partition(cg[:id], partition) + return unless offset_id - partition_key = build_partition_key(partition_by, partition) - offset = db[@offsets_table].where(consumer_group_id: cg[:id], partition_key: partition_key).first - return unless offset + offset = db[@offsets_table].where(id: offset_id).first return if offset[:last_position] >= position - db[@offsets_table].where(id: offset[:id]).update(last_position: position) + db[@offsets_table].where(id: offset_id).update(last_position: position) if position > cg[:highest_position] db[@consumer_groups_table].where(id: cg[:id]).update( @@ -765,63 +776,125 @@ def build_partition_key(partition_by, values) partition_by.sort.map { |attr| "#{attr}:#{values[attr]}" }.join('|') end - # Discover partition tuples via AND self-joins and create offset + key_pair rows. - # Only messages with ALL partition attributes create partition tuples. + # Scan a bounded window of messages forward from the consumer group's + # discovery_position watermark, find new partition tuples, create offsets + # for them, then advance the watermark. # # @param cg_id [Integer] consumer group internal ID # @param partition_by [Array] sorted attribute names + # @param handled_types [Array] message type strings # @return [void] - def bootstrap_offsets(cg_id, partition_by) - # Build AND self-join query to find all unique tuples + def discover_new_partitions(cg_id, partition_by, handled_types) + cg = db[@consumer_groups_table].where(id: cg_id).first + discovery_pos = cg[:discovery_position] + + types_list = handled_types.map { |t| db.literal(t) }.join(', ') + + # Build AND self-join query scoped to messages after the watermark joins = [] selects = [] + not_exists_joins = [] partition_by.each_with_index do |attr, i| joins << "JOIN #{@message_key_pairs_table} mkp#{i} ON m.position = mkp#{i}.message_position" joins << "JOIN #{@key_pairs_table} kp#{i} ON mkp#{i}.key_pair_id = kp#{i}.id AND kp#{i}.name = #{db.literal(attr)}" selects << "kp#{i}.id AS kp_id_#{i}, kp#{i}.value AS val_#{i}" + not_exists_joins << "JOIN #{@offset_key_pairs_table} okp#{i} ON o.id = okp#{i}.offset_id AND okp#{i}.key_pair_id = kp#{i}.id" end group_by = partition_by.each_index.map { |i| "kp#{i}.id" }.join(', ') sql = <<~SQL - SELECT #{selects.join(', ')} + SELECT #{selects.join(', ')}, + MAX(m.position) AS max_pos FROM #{@messages_table} m #{joins.join("\n")} + WHERE m.message_type IN (#{types_list}) + AND m.position > #{db.literal(discovery_pos)} + AND NOT EXISTS ( + SELECT 1 FROM #{@offsets_table} o + #{not_exists_joins.join("\n")} + WHERE o.consumer_group_id = #{db.literal(cg_id)} + ) GROUP BY #{group_by} + LIMIT #{DISCOVERY_BATCH_SIZE} SQL rows = db.fetch(sql).all - return if rows.empty? - db.transaction do - rows.each do |row| - # Build the values hash and collect key_pair_ids - values = {} - kp_ids = [] - partition_by.each_with_index do |attr, i| - values[attr] = row[:"val_#{i}"] - kp_ids << row[:"kp_id_#{i}"] + max_discovered_pos = 0 + + if rows.any? + db.transaction do + rows.each do |row| + values = {} + kp_ids = [] + partition_by.each_with_index do |attr, i| + values[attr] = row[:"val_#{i}"] + kp_ids << row[:"kp_id_#{i}"] + end + + create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) + max_discovered_pos = row[:max_pos] if row[:max_pos] > max_discovered_pos end + end + end + + # Advance watermark: to max discovered position, or to current max if nothing found + new_watermark = max_discovered_pos > 0 ? max_discovered_pos : (latest_position) + if new_watermark > discovery_pos + db[@consumer_groups_table].where(id: cg_id).update( + discovery_position: new_watermark, + updated_at: Time.now.iso8601 + ) + end + end - partition_key = build_partition_key(partition_by, values) + # Ensure an offset row exists for a specific partition (by attribute values). + # Resolves key_pair IDs from the key_pairs table; returns nil if any + # partition attribute has no corresponding key_pair (meaning no messages + # with that attribute value exist yet). + # + # @param cg_id [Integer] consumer group internal ID + # @param partition [Hash{String => String}] attribute names and values + # @return [Integer, nil] offset ID, or nil if key_pairs not found + def ensure_offset_for_partition(cg_id, partition) + partition_by = partition.keys.sort + kp_ids = [] + partition_by.each do |attr| + kp = db[@key_pairs_table].where(name: attr, value: partition[attr].to_s).first + return nil unless kp - # INSERT OR IGNORE the offset row - db.run(<<~SQL) - INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) - VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) - SQL + kp_ids << kp[:id] + end - offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + create_offset_with_key_pairs(cg_id, partition_by, partition, kp_ids) + end - # INSERT OR IGNORE the offset_key_pairs join rows - kp_ids.each do |kp_id| - db.run(<<~SQL) - INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) - VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) - SQL - end - end + # Create an offset row and its key_pair associations. Idempotent via INSERT OR IGNORE. + # + # @param cg_id [Integer] consumer group internal ID + # @param partition_by [Array] sorted attribute names + # @param values [Hash{String => String}] attribute values keyed by name + # @param kp_ids [Array] key_pair IDs + # @return [Integer] offset ID + def create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) + partition_key = build_partition_key(partition_by, values) + + db.run(<<~SQL) + INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) + VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) + SQL + + offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + + kp_ids.each do |kp_id| + db.run(<<~SQL) + INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) + VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) + SQL end + + offset_id end # Find the next unclaimed partition with pending messages and claim it. @@ -838,26 +911,30 @@ def bootstrap_offsets(cg_id, partition_by) def find_and_claim_partition(cg_id, handled_types, worker_id) types_list = handled_types.map { |t| db.literal(t) }.join(', ') + # Uses count-matching for AND semantics: a message is a candidate only + # when ALL of the offset's key_pairs appear in message_key_pairs for + # that message. This avoids the expensive NOT EXISTS + 4-table subquery + # and short-circuits immediately when any key_pair has no match. sql = <<~SQL SELECT o.id AS offset_id, o.partition_key, o.last_position, MIN(m.position) AS next_position FROM #{@offsets_table} o - JOIN #{@offset_key_pairs_table} okp ON o.id = okp.offset_id - JOIN #{@message_key_pairs_table} mkp ON okp.key_pair_id = mkp.key_pair_id - JOIN #{@messages_table} m ON mkp.message_position = m.position + CROSS JOIN #{@messages_table} m WHERE o.consumer_group_id = #{db.literal(cg_id)} AND o.claimed = 0 AND m.position > o.last_position AND m.message_type IN (#{types_list}) - AND NOT EXISTS ( - SELECT 1 - FROM #{@offset_key_pairs_table} okp2 - JOIN #{@key_pairs_table} kp_part ON okp2.key_pair_id = kp_part.id - JOIN #{@message_key_pairs_table} mkp2 ON mkp2.message_position = m.position - JOIN #{@key_pairs_table} kp_msg ON mkp2.key_pair_id = kp_msg.id - WHERE okp2.offset_id = o.id - AND kp_msg.name = kp_part.name - AND kp_msg.value != kp_part.value + AND ( + SELECT COUNT(*) + FROM #{@offset_key_pairs_table} okp + JOIN #{@message_key_pairs_table} mkp + ON mkp.key_pair_id = okp.key_pair_id + AND mkp.message_position = m.position + WHERE okp.offset_id = o.id + ) = ( + SELECT COUNT(*) + FROM #{@offset_key_pairs_table} + WHERE offset_id = o.id ) GROUP BY o.id ORDER BY next_position ASC diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 7e841181..460726cc 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -972,7 +972,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id) end - it 'bootstraps offsets for new partitions' do + it 'lazily discovers and creates offsets for new partitions' do store.append( CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) @@ -1252,6 +1252,126 @@ module CCCStoreTestMessages end end + describe '#claim_next (lazy discovery)' do + let(:group_id) { 'discovery-test' } + let(:handled_types) { ['store_test.device.registered', 'store_test.device.bound'] } + + before do + store.register_consumer_group(group_id) + end + + it 'advances discovery_position after discovering partitions' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + + cg = db[:sourced_consumer_groups].where(group_id: group_id).first + expect(cg[:discovery_position]).to be >= 1 + end + + it 'does not re-scan earlier messages on subsequent discovery calls' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + + # First claim discovers both partitions and claims one + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) + + # Second claim uses fast path (no discovery needed) + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2).not_to be_nil + expect(r2.partition_key).not_to eq(r1.partition_key) + end + + it 'discovers new partitions added after initial discovery' do + # First batch of messages + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) + + # No more work + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(result).to be_nil + + # New partition arrives + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ) + + # Should discover and claim the new partition + r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + expect(r2).not_to be_nil + expect(r2.partition_value).to eq({ 'device_id' => 'dev-2' }) + end + + it 'resets discovery_position when consumer group is reset' do + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) + + cg = db[:sourced_consumer_groups].where(group_id: group_id).first + expect(cg[:discovery_position]).to be > 0 + + store.reset_consumer_group(group_id) + + cg = db[:sourced_consumer_groups].where(group_id: group_id).first + expect(cg[:discovery_position]).to eq(0) + end + + it 'a new reactor catches up on old messages without explicit bootstrap' do + # Pre-populate with multiple partitions + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + ]) + + # New reactor with no prior offsets + new_group = 'new-reactor' + store.register_consumer_group(new_group) + + # Should discover and process all partitions + claimed_partitions = [] + 3.times do + r = store.claim_next(new_group, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + break unless r + + claimed_partitions << r.partition_key + store.ack(new_group, offset_id: r.offset_id, position: r.messages.last.position) + end + + expect(claimed_partitions.size).to eq(3) + expect(claimed_partitions).to contain_exactly('device_id:dev-1', 'device_id:dev-2', 'device_id:dev-3') + end + + it 'only discovers partitions matching handled_types' do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'Truck' }) + ]) + + # Only handles device events, not asset events + result = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-1') + expect(result).not_to be_nil + expect(result.partition_value).to eq({ 'device_id' => 'dev-1' }) + + # No more work for device_id partitions + store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) + result2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.device.registered'], worker_id: 'w-1') + expect(result2).to be_nil + end + end + describe '#claim_next (composite partition — conditional AND fetch)' do let(:group_id) { 'composite-test' } let(:handled_types) do @@ -1557,7 +1677,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id) end - it 'bootstraps and advances offset for a new partition' do + it 'creates and advances offset for a new partition' do store.append( CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) From 8c31b4937cddf8c5f9f6c8cc42c2357e339eed27 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 15:52:46 +0000 Subject: [PATCH 074/115] Replace CROSS JOIN with streaming iteration in find_and_claim_partition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CROSS JOIN on offsets × messages degraded quadratically as offsets accumulated (164ms/claim at 1000 partitions). Replace with iterating unclaimed offsets via Sequel cursor and checking each for pending messages via index-driven joins. Stops at the first match. Per-claim cost is now ~2ms at 1000 partitions (was 164ms). Memory usage is constant — rows stream from the SQLite cursor, not loaded into an array. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 61 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index f4fbc4d0..537f002c 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -911,37 +911,38 @@ def create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) def find_and_claim_partition(cg_id, handled_types, worker_id) types_list = handled_types.map { |t| db.literal(t) }.join(', ') - # Uses count-matching for AND semantics: a message is a candidate only - # when ALL of the offset's key_pairs appear in message_key_pairs for - # that message. This avoids the expensive NOT EXISTS + 4-table subquery - # and short-circuits immediately when any key_pair has no match. - sql = <<~SQL - SELECT o.id AS offset_id, o.partition_key, o.last_position, - MIN(m.position) AS next_position - FROM #{@offsets_table} o - CROSS JOIN #{@messages_table} m - WHERE o.consumer_group_id = #{db.literal(cg_id)} - AND o.claimed = 0 - AND m.position > o.last_position - AND m.message_type IN (#{types_list}) - AND ( - SELECT COUNT(*) - FROM #{@offset_key_pairs_table} okp - JOIN #{@message_key_pairs_table} mkp - ON mkp.key_pair_id = okp.key_pair_id - AND mkp.message_position = m.position - WHERE okp.offset_id = o.id - ) = ( - SELECT COUNT(*) - FROM #{@offset_key_pairs_table} - WHERE offset_id = o.id + # Iterate unclaimed offsets ordered by last_position (lowest = most behind). + # For each, check if a pending message exists with AND semantics. + # Stops at the first match. Uses Sequel's streaming `.each` — rows are + # fetched from the SQLite cursor one at a time, not loaded into memory. + row = nil + db[@offsets_table] + .where(consumer_group_id: cg_id, claimed: 0) + .select(:id, :partition_key, :last_position) + .order(:last_position) + .each do |offset| + + pending = db.fetch(<<~SQL).first + SELECT 1 + FROM #{@offset_key_pairs_table} okp + JOIN #{@message_key_pairs_table} mkp ON okp.key_pair_id = mkp.key_pair_id + JOIN #{@messages_table} m ON mkp.message_position = m.position + WHERE okp.offset_id = #{db.literal(offset[:id])} + AND m.position > #{db.literal(offset[:last_position])} + AND m.message_type IN (#{types_list}) + GROUP BY m.position + HAVING COUNT(*) = ( + SELECT COUNT(*) FROM #{@offset_key_pairs_table} WHERE offset_id = #{db.literal(offset[:id])} ) - GROUP BY o.id - ORDER BY next_position ASC - LIMIT 1 - SQL + LIMIT 1 + SQL + + if pending + row = { offset_id: offset[:id], partition_key: offset[:partition_key], last_position: offset[:last_position] } + break + end + end - row = db.fetch(sql).first return nil unless row now = Time.now.iso8601 @@ -951,7 +952,7 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) return nil if updated == 0 - { offset_id: row[:offset_id], partition_key: row[:partition_key], last_position: row[:last_position] } + row end # Fetch messages for a partition using conditional AND semantics. From e16b897db6571d234fb69682bd576dc9bf075b03 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 16:19:39 +0000 Subject: [PATCH 075/115] Order discovery results by min_pos to prevent skipping partitions Without ORDER BY, LIMIT on GROUP BY returned an arbitrary subset of partitions. The watermark advanced past undiscovered partitions, leaving them permanently invisible. Adding ORDER BY min_pos ASC ensures partitions are discovered in position order. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 537f002c..21f0e695 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -805,6 +805,7 @@ def discover_new_partitions(cg_id, partition_by, handled_types) sql = <<~SQL SELECT #{selects.join(', ')}, + MIN(m.position) AS min_pos, MAX(m.position) AS max_pos FROM #{@messages_table} m #{joins.join("\n")} @@ -816,6 +817,7 @@ def discover_new_partitions(cg_id, partition_by, handled_types) WHERE o.consumer_group_id = #{db.literal(cg_id)} ) GROUP BY #{group_by} + ORDER BY min_pos ASC LIMIT #{DISCOVERY_BATCH_SIZE} SQL From b8afa2479cd4596f560da8029df7b59cbd2ade1d Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 17:12:50 +0000 Subject: [PATCH 076/115] Use CTEs in discovery and fetch queries to halve key_pairs JOINs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit discover_new_partitions: pre-join message_key_pairs with key_pairs in per-attribute CTEs, so the main query self-joins on N CTEs instead of 2N tables. Cuts the 3-key discovery cost by ~7x. fetch_partition_messages: CTE pre-resolves partition attribute names, replacing the double-join on key_pairs in the count-match RHS with a simple IN (SELECT name FROM partition_attr_names). 3-key per-claim at 200 partitions: 7.28ms → 1.08ms. Key count scaling is now nearly flat: 0.79 / 0.98 / 1.08ms. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 21f0e695..624775c2 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -790,20 +790,29 @@ def discover_new_partitions(cg_id, partition_by, handled_types) types_list = handled_types.map { |t| db.literal(t) }.join(', ') - # Build AND self-join query scoped to messages after the watermark - joins = [] + # CTE per partition attribute: pre-joins message_key_pairs with key_pairs + # so the main query only self-joins on the CTEs (N joins instead of 2N). + ctes = [] selects = [] + joins = [] not_exists_joins = [] partition_by.each_with_index do |attr, i| - joins << "JOIN #{@message_key_pairs_table} mkp#{i} ON m.position = mkp#{i}.message_position" - joins << "JOIN #{@key_pairs_table} kp#{i} ON mkp#{i}.key_pair_id = kp#{i}.id AND kp#{i}.name = #{db.literal(attr)}" - selects << "kp#{i}.id AS kp_id_#{i}, kp#{i}.value AS val_#{i}" - not_exists_joins << "JOIN #{@offset_key_pairs_table} okp#{i} ON o.id = okp#{i}.offset_id AND okp#{i}.key_pair_id = kp#{i}.id" + ctes << <<~CTE + attr#{i} AS ( + SELECT mkp.message_position, kp.id AS kp_id, kp.value AS val + FROM #{@message_key_pairs_table} mkp + JOIN #{@key_pairs_table} kp ON mkp.key_pair_id = kp.id AND kp.name = #{db.literal(attr)} + ) + CTE + selects << "a#{i}.kp_id AS kp_id_#{i}, a#{i}.val AS val_#{i}" + joins << "JOIN attr#{i} a#{i} ON m.position = a#{i}.message_position" + not_exists_joins << "JOIN #{@offset_key_pairs_table} okp#{i} ON o.id = okp#{i}.offset_id AND okp#{i}.key_pair_id = a#{i}.kp_id" end - group_by = partition_by.each_index.map { |i| "kp#{i}.id" }.join(', ') + group_by = partition_by.each_index.map { |i| "a#{i}.kp_id" }.join(', ') sql = <<~SQL + WITH #{ctes.join(",\n")} SELECT #{selects.join(', ')}, MIN(m.position) AS min_pos, MAX(m.position) AS max_pos @@ -973,7 +982,14 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: kp_ids_list = key_pair_ids.map { |id| db.literal(id) }.join(', ') types_list = handled_types.map { |t| db.literal(t) }.join(', ') + # CTE pre-resolves which attribute names the partition's key_pairs cover. + # The main query uses this to count-match: a message qualifies when it + # matches as many partition key_pairs as it has attributes in common with + # the partition (AND semantics for shared attributes). sql = <<~SQL + WITH partition_attr_names AS ( + SELECT DISTINCT name FROM #{@key_pairs_table} WHERE id IN (#{kp_ids_list}) + ) SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at FROM #{@messages_table} m WHERE m.position > #{db.literal(last_position)} @@ -988,12 +1004,11 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: WHERE mkp.message_position = m.position AND mkp.key_pair_id IN (#{kp_ids_list}) ) = ( - SELECT COUNT(DISTINCT kp_part.name) + SELECT COUNT(DISTINCT kp_msg.name) FROM #{@message_key_pairs_table} mkp2 JOIN #{@key_pairs_table} kp_msg ON mkp2.key_pair_id = kp_msg.id - JOIN #{@key_pairs_table} kp_part ON kp_part.id IN (#{kp_ids_list}) - AND kp_part.name = kp_msg.name WHERE mkp2.message_position = m.position + AND kp_msg.name IN (SELECT name FROM partition_attr_names) ) ORDER BY m.position ASC SQL From c1831be91225edec0c7c65d7ff04ffb1b37fa733 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 17:22:45 +0000 Subject: [PATCH 077/115] Skip caught-up offsets in find_and_claim_partition Filter out offsets where last_position >= latest message position before iterating. In the steady-state "no work" case this avoids scanning offsets that can't possibly have pending messages. The streaming iteration is still O(N) in the worst case (all offsets behind, only one has matching messages), but this is acceptable: - Cold drain: first offset always matches (O(1) per call) - Steady state: notifier/catch-up-poller drives dispatch, so find_and_claim_partition is only called when work is likely - Needle-in-haystack: ~85ms at 1000 offsets, bounded by offset count Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 624775c2..402f10f0 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -922,13 +922,15 @@ def create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) def find_and_claim_partition(cg_id, handled_types, worker_id) types_list = handled_types.map { |t| db.literal(t) }.join(', ') - # Iterate unclaimed offsets ordered by last_position (lowest = most behind). - # For each, check if a pending message exists with AND semantics. - # Stops at the first match. Uses Sequel's streaming `.each` — rows are - # fetched from the SQLite cursor one at a time, not loaded into memory. + # Iterate unclaimed offsets that could have pending work. + # Ordered by last_position ASC so the most-behind partitions are checked + # first. Offsets at or past the latest message position are skipped entirely + # (they can't have pending work). Stops at the first match. + max_pos = latest_position row = nil db[@offsets_table] .where(consumer_group_id: cg_id, claimed: 0) + .where(Sequel.lit('last_position < ?', max_pos)) .select(:id, :partition_key, :last_position) .order(:last_position) .each do |offset| From dd4d40f5cb0f03d2e704eb651b0eaaac014ae693 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 17:32:37 +0000 Subject: [PATCH 078/115] Add timing logs to CCC::Router for profiling Log claim_next, read_history, handle_claim and transaction durations via Console.info for demo/profiling use. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/router.rb | 12 ++++++++++++ lib/sourced/ccc/store.rb | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index c764ff14..60298bfc 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -22,6 +22,7 @@ def register(reactor_class) def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) handled_types = reactor_class.handled_messages.map(&:type).uniq + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) claim = store.claim_next( reactor_class.group_id, partition_by: reactor_class.partition_keys.map(&:to_s), @@ -29,6 +30,8 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) worker_id: worker_id, batch_size: batch_size ) + t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Console.info "AAA #{reactor_class.name} claim_next=#{((t1-t0)*1000).round(1)}ms found=#{!!claim}" return false unless claim begin @@ -36,10 +39,16 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) if @needs_history[reactor_class] attrs = claim.partition_value.transform_keys(&:to_sym) conditions = reactor_class.context_for(attrs) + t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC) kwargs[:history] = store.read(conditions) + t3 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Console.info "AAA #{reactor_class.name} read_history=#{((t3-t2)*1000).round(1)}ms" end + t4 = Process.clock_gettime(Process::CLOCK_MONOTONIC) action_pairs = reactor_class.handle_claim(claim, **kwargs) + t5 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Console.info "AAA #{reactor_class.name} handle_claim=#{((t5-t4)*1000).round(1)}ms" if action_pairs == Actions::RETRY store.release(reactor_class.group_id, offset_id: claim.offset_id) @@ -81,6 +90,7 @@ def drain(limit = Float::INFINITY) def execute_actions(action_pairs, claim, group_id) after_sync_actions = [] + t_tx0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) store.db.transaction do last_position = nil Array(action_pairs).each do |(actions, source_message)| @@ -101,6 +111,8 @@ def execute_actions(action_pairs, claim, group_id) end end + t_tx1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Console.info "AAA transaction=#{((t_tx1-t_tx0)*1000).round(1)}ms" after_sync_actions.each(&:call) end end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 402f10f0..48925464 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -181,7 +181,7 @@ def append(messages, guard: nil) end end - Console.info "AAA append #{messages.map(&:type).uniq}" + Console.info "AAA append #{messages.map(&:type).uniq}", messages: messages.size notifier.notify_new_messages(messages.map(&:type).uniq) last_position From bc8f0290c7e774af93c00a9a67c78f22644586fb Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 17:46:25 +0000 Subject: [PATCH 079/115] Short-circuit idle polls in claim_next via types_max_pos check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When offsets exist and the latest message of the handled types is at or below the consumer group's highest_position, return nil immediately without iterating offsets. This makes idle polling O(1) regardless of offset count — two cheap indexed lookups instead of N per-offset queries. 1000 offsets all caught up: 233ms → 0.11ms (2000x improvement). Drain and active claim performance unchanged. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 48925464..c65db891 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -496,6 +496,19 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: .first return nil unless cg + # Short-circuit: if offsets exist and the latest message of the handled + # types is at or below highest_position, all work is done. O(1) via the + # message_type index — avoids the per-offset scan entirely on idle polls. + # Skipped when no offsets exist (e.g. after reset or first run) so that + # discovery can create them. + has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? + if has_offsets + types_max_pos = db[@messages_table] + .where(message_type: handled_types) + .max(:position) || 0 + return nil if types_max_pos <= cg[:highest_position] + end + # Phase 1: Fast path — try existing offsets claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) @@ -922,15 +935,9 @@ def create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) def find_and_claim_partition(cg_id, handled_types, worker_id) types_list = handled_types.map { |t| db.literal(t) }.join(', ') - # Iterate unclaimed offsets that could have pending work. - # Ordered by last_position ASC so the most-behind partitions are checked - # first. Offsets at or past the latest message position are skipped entirely - # (they can't have pending work). Stops at the first match. - max_pos = latest_position row = nil db[@offsets_table] .where(consumer_group_id: cg_id, claimed: 0) - .where(Sequel.lit('last_position < ?', max_pos)) .select(:id, :partition_key, :last_position) .order(:last_position) .each do |offset| From 6fc497ece0672fa0570ed1abc5f8815fa04a0a44 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 18:52:14 +0000 Subject: [PATCH 080/115] Add scaling benchmark with HTML charts Measures claim_next across orders of magnitude (10 to 1M partitions) using direct SQL seeding for large scales. Outputs CSV and generates an HTML file with Chart.js line charts on logarithmic axes. Four scenarios: idle poll, cold drain per-claim, warm claim per-claim, and incremental discovery. Adaptive iterations for large scales. Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/scaling_bench.rb | 440 +++++++++++++++++++++++++++++++++++++ bench/scaling_results.html | 278 +++++++++++++++++++++++ 2 files changed, 718 insertions(+) create mode 100644 bench/scaling_bench.rb create mode 100644 bench/scaling_results.html diff --git a/bench/scaling_bench.rb b/bench/scaling_bench.rb new file mode 100644 index 00000000..36122aac --- /dev/null +++ b/bench/scaling_bench.rb @@ -0,0 +1,440 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Benchmark: measures claim_next performance across orders of magnitude. +# Seeds data directly via SQL for large scales, then measures a sample. +# +# Usage: +# bundle exec ruby bench/scaling_bench.rb +# bundle exec ruby bench/scaling_bench.rb --scales 10,100,1000 +# bundle exec ruby bench/scaling_bench.rb --keys 1,2 +# bundle exec ruby bench/scaling_bench.rb --output results.html + +require 'bundler/setup' +require 'sequel' +require 'sourced' +require 'sourced/ccc' +require 'sourced/ccc/store' +require 'optparse' +require 'json' + +Sourced.config.logger = Logger.new(File::NULL) +Console.logger.off! + +# --- CLI options ----------------------------------------------------------- + +options = { + scales: [10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000], + keys: [1, 2, 3], + iterations: 2, + sample_size: 50, + output: 'bench/scaling_results.html' +} + +OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options]" + opts.on('--scales LIST', 'Comma-separated partition counts') { |v| options[:scales] = v.split(',').map(&:to_i) } + opts.on('--keys LIST', 'Comma-separated key counts') { |v| options[:keys] = v.split(',').map(&:to_i) } + opts.on('--iterations N', Integer, 'Iterations per measurement') { |v| options[:iterations] = v } + opts.on('--sample N', Integer, 'Claims to measure per scenario') { |v| options[:sample_size] = v } + opts.on('--output FILE', 'HTML output file') { |v| options[:output] = v } +end.parse! + +GROUP_ID = 'bench-group' +HANDLED_TYPES = ['scaling_bench.event'] +INCR_MAX_SCALE = 10_000 # skip incremental discovery above this + +BenchEvent = Sourced::CCC::Message.define('scaling_bench.event') do + attribute :k0, String + attribute :k1, String + attribute :k2, String + attribute :k3, String +end + +# --- Fast SQL seeding ------------------------------------------------------ + +def seed(count, key_count, caught_up: false) + db = Sequel.sqlite + db.run('PRAGMA cache_size = -64000') + db.run('PRAGMA synchronous = OFF') + db.run('PRAGMA journal_mode = MEMORY') + store = Sourced::CCC::Store.new(db) + store.install! + store.register_consumer_group(GROUP_ID) + + cg_id = db[:sourced_consumer_groups].where(group_id: GROUP_ID).get(:id) + now = Time.now.iso8601 + batch = [10_000, count].min + + (0...count).each_slice(batch) do |slice| + db.transaction do + # Messages + db[:sourced_messages].multi_insert( + slice.map { |i| + payload = (0...4).map { |k| "\"k#{k}\":\"v#{i}\"" }.join(',') + { message_id: "m-#{i}", message_type: 'scaling_bench.event', payload: "{#{payload}}", created_at: now } + } + ) + + # Key pairs + slice.each { |i| + (0...key_count).each { |k| db.run("INSERT OR IGNORE INTO sourced_key_pairs (name, value) VALUES ('k#{k}', 'v#{i}')") } + } + + # Message key pairs + slice.each { |i| + (0...key_count).each { |k| + db.run("INSERT INTO sourced_message_key_pairs (message_position, key_pair_id) SELECT #{i + 1}, id FROM sourced_key_pairs WHERE name = 'k#{k}' AND value = 'v#{i}'") + } + } + + if caught_up + # Offsets + db[:sourced_offsets].multi_insert( + slice.map { |i| + pk = (0...key_count).map { |k| "k#{k}:v#{i}" }.join('|') + { consumer_group_id: cg_id, partition_key: pk, last_position: i + 1, claimed: 0 } + } + ) + + # Offset key pairs — bulk via INSERT...SELECT + (0...key_count).each { |k| + db.run(<<~SQL) + INSERT OR IGNORE INTO sourced_offset_key_pairs (offset_id, key_pair_id) + SELECT o.id, kp.id + FROM sourced_offsets o + JOIN sourced_key_pairs kp ON kp.name = 'k#{k}' + AND kp.value = SUBSTR(o.partition_key, #{k > 0 ? "INSTR(o.partition_key, 'k#{k}:') + #{k.to_s.length + 2}" : '4'}, LENGTH(o.partition_key)) + WHERE o.consumer_group_id = #{cg_id} + AND o.id NOT IN (SELECT offset_id FROM sourced_offset_key_pairs) + SQL + } + end + end + $stderr.print '.' + end + + if caught_up + db[:sourced_consumer_groups].where(id: cg_id).update( + highest_position: count, discovery_position: count, updated_at: now + ) + end + + # Restore WAL mode for benchmarking + db.run('PRAGMA synchronous = FULL') + db.run('PRAGMA journal_mode = WAL') + + [db, store] +end + +# Simpler caught_up seeding: parse partition_key to match key_pairs +# For the bulk offset_key_pairs insert, extract the value from partition_key +# which has format "k0:v123" or "k0:v123|k1:v123" +def seed_caught_up_fast(count, key_count) + db = Sequel.sqlite + db.run('PRAGMA cache_size = -64000') + db.run('PRAGMA synchronous = OFF') + db.run('PRAGMA journal_mode = MEMORY') + store = Sourced::CCC::Store.new(db) + store.install! + store.register_consumer_group(GROUP_ID) + + cg_id = db[:sourced_consumer_groups].where(group_id: GROUP_ID).get(:id) + now = Time.now.iso8601 + batch = [10_000, count].min + + (0...count).each_slice(batch) do |slice| + db.transaction do + # Messages + db[:sourced_messages].multi_insert( + slice.map { |i| + payload = (0...4).map { |k| "\"k#{k}\":\"v#{i}\"" }.join(',') + { message_id: "m-#{i}", message_type: 'scaling_bench.event', payload: "{#{payload}}", created_at: now } + } + ) + + # Key pairs + message_key_pairs + slice.each { |i| + (0...key_count).each { |k| + db.run("INSERT OR IGNORE INTO sourced_key_pairs (name, value) VALUES ('k#{k}', 'v#{i}')") + db.run("INSERT INTO sourced_message_key_pairs (message_position, key_pair_id) SELECT #{i + 1}, id FROM sourced_key_pairs WHERE name = 'k#{k}' AND value = 'v#{i}'") + } + } + + # Offsets + db[:sourced_offsets].multi_insert( + slice.map { |i| + pk = (0...key_count).map { |k| "k#{k}:v#{i}" }.join('|') + { consumer_group_id: cg_id, partition_key: pk, last_position: i + 1, claimed: 0 } + } + ) + + # Offset key pairs — per-row but using subquery + slice.each { |i| + pk = (0...key_count).map { |k| "k#{k}:v#{i}" }.join('|') + (0...key_count).each { |k| + db.run("INSERT OR IGNORE INTO sourced_offset_key_pairs (offset_id, key_pair_id) SELECT o.id, kp.id FROM sourced_offsets o, sourced_key_pairs kp WHERE o.partition_key = '#{pk}' AND o.consumer_group_id = #{cg_id} AND kp.name = 'k#{k}' AND kp.value = 'v#{i}'") + } + } + end + $stderr.print '.' + end + + db[:sourced_consumer_groups].where(id: cg_id).update( + highest_position: count, discovery_position: count, updated_at: now + ) + db.run('PRAGMA synchronous = FULL') + db.run('PRAGMA journal_mode = WAL') + + [db, store] +end + +# --- Measurement helpers --------------------------------------------------- + +def measure + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + [Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0, result] +end + +def median(values) + sorted = values.sort + mid = sorted.size / 2 + sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0 +end + +def partition_keys(n) = (0...n).map { |i| "k#{i}" } + +def claim_once(store, key_count) + store.claim_next(GROUP_ID, partition_by: partition_keys(key_count), handled_types: HANDLED_TYPES, worker_id: 'w-1') +end + +# Adaptive iterations — fewer for large scales to keep total time reasonable +def effective_iterations(scale, base) + case scale + when 0..10_000 then base + when 10_001..100_000 then [base, 2].min + else 1 + end +end + +# --- Benchmark runner ------------------------------------------------------ + +results = [] + +options[:keys].each do |key_count| + options[:scales].each do |scale| + label = "keys=#{key_count} scale=#{format('%10d', scale)}" + $stderr.print "\n#{label}" + iters = effective_iterations(scale, options[:iterations]) + + row = { keys: key_count, scale: scale } + + # --- 1. Idle poll (all caught up) --- + $stderr.print " [idle" + idle_times = [] + iters.times do + _db, store = seed_caught_up_fast(scale, key_count) + polls = 5.times.map { measure { claim_once(store, key_count) }.first } + idle_times << median(polls) + end + row[:idle_poll_ms] = (median(idle_times) * 1000).round(4) + $stderr.print "=#{row[:idle_poll_ms]}ms]" + + # --- 2. Cold drain (sample first N claims) --- + $stderr.print " [cold" + sample = [options[:sample_size], scale].min + drain_times = [] + iters.times do + _db, store = seed(scale, key_count, caught_up: false) + times = [] + sample.times do + t, r = measure { claim_once(store, key_count) } + break unless r + store.ack(GROUP_ID, offset_id: r.offset_id, position: r.messages.last.position) + times << t + end + drain_times << median(times) if times.any? + end + row[:per_claim_cold_ms] = drain_times.any? ? (median(drain_times) * 1000).round(4) : 0 + $stderr.print "=#{row[:per_claim_cold_ms]}ms]" + + # --- 3. Warm claim (sample N claims with new messages) --- + $stderr.print " [warm" + warm_times = [] + iters.times do + _db, store = seed_caught_up_fast(scale, key_count) + msgs = (0...sample).map { |i| BenchEvent.new(payload: { k0: "v#{i}", k1: "v#{i}", k2: "v#{i}", k3: "v#{i}" }) } + store.append(msgs) + + times = [] + sample.times do + t, r = measure { claim_once(store, key_count) } + break unless r + store.ack(GROUP_ID, offset_id: r.offset_id, position: r.messages.last.position) + times << t + end + warm_times << median(times) if times.any? + end + row[:per_claim_warm_ms] = warm_times.any? ? (median(warm_times) * 1000).round(4) : 0 + $stderr.print "=#{row[:per_claim_warm_ms]}ms]" + + # --- 4. Incremental discovery (1 new partition) --- + if scale <= INCR_MAX_SCALE + $stderr.print " [incr" + incr_times = [] + iters.times do + _db, store = seed_caught_up_fast(scale, key_count) + store.append(BenchEvent.new(payload: { k0: 'vnew', k1: 'vnew', k2: 'vnew', k3: 'vnew' })) + t, _ = measure { claim_once(store, key_count) } + incr_times << t + end + row[:incremental_ms] = (median(incr_times) * 1000).round(4) + $stderr.print "=#{row[:incremental_ms]}ms]" + else + row[:incremental_ms] = nil + end + + results << row + end +end + +# --- CSV output ------------------------------------------------------------ + +$stderr.puts "\n" +puts "keys,scale,idle_poll_ms,per_claim_cold_ms,per_claim_warm_ms,incremental_ms" +results.each do |r| + incr = r[:incremental_ms] ? r[:incremental_ms].to_s : '' + puts "#{r[:keys]},#{r[:scale]},#{r[:idle_poll_ms]},#{r[:per_claim_cold_ms]},#{r[:per_claim_warm_ms]},#{incr}" +end + +# --- HTML chart output ----------------------------------------------------- + +chart_data = {} +options[:keys].each { |k| chart_data[k] = results.select { |r| r[:keys] == k } } + +def fmt_scale(s) + return "#{s / 1_000_000}M" if s >= 1_000_000 + return "#{s / 1_000}K" if s >= 1_000 + s.to_s +end + +html = <<~HTML + + + + CCC::Store#claim_next — Scaling Benchmark + + + + +

CCC::Store#claim_next — Scaling Benchmark

+

+ Generated #{Time.now.strftime('%Y-%m-%d %H:%M')} · + Keys: #{options[:keys].join(', ')} · + Scales: #{options[:scales].map { |s| fmt_scale(s) }.join(', ')} · + Iterations: #{options[:iterations]} (adaptive) · + Sample: #{options[:sample_size]} claims +

+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +

Raw Data

+ + + + + + #{results.map { |r| + scale_fmt = r[:scale].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + incr_fmt = r[:incremental_ms] ? r[:incremental_ms].to_s : '—' + "" + }.join("\n ")} + +
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms)
#{r[:keys]}#{scale_fmt}#{r[:idle_poll_ms]}#{r[:per_claim_cold_ms]}#{r[:per_claim_warm_ms]}#{incr_fmt}
+ + + + +HTML + +File.write(options[:output], html) +$stderr.puts "Chart written to #{options[:output]}" diff --git a/bench/scaling_results.html b/bench/scaling_results.html new file mode 100644 index 00000000..c637039f --- /dev/null +++ b/bench/scaling_results.html @@ -0,0 +1,278 @@ + + + + CCC::Store#claim_next — Scaling Benchmark + + + + +

CCC::Store#claim_next — Scaling Benchmark

+

+ Generated 2026-03-15 18:50 · + Keys: 1, 2, 3 · + Scales: 10, 100, 1K, 10K, 100K, 1M · + Iterations: 2 (adaptive) · + Sample: 50 claims +

+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +

Raw Data

+ + + + + + + + + + + + + + + + + + + + + + + + +
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms)
1100.25750.47880.37720.981
11000.1160.4350.48273.495
11,0000.11651.1561.3252119.5775
110,0000.11958.23489.63910931.7395
1100,0000.11886.9555103.1813—
11,000,0000.119907.1531089.092—
2100.1150.40530.44651.154
21000.12050.53480.58374.7125
21,0000.1111.83652.053191.048
210,0000.14114.827716.715818336.0155
2100,0000.1155162.5097180.0425—
21,000,0000.1171679.57751876.217—
3100.1180.41380.41321.1915
31000.11850.54580.63855.1345
31,0000.12152.21632.4238253.031
310,0000.119519.118220.528223965.6405
3100,0000.118208.8647227.058—
31,000,0000.1192165.0592372.594—
+ + + + From 999f77b78c208c147074925025d4e0ef5f0e6edb Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 18:55:27 +0000 Subject: [PATCH 081/115] Add scenario descriptions to scaling benchmark charts Each chart now explains what the scenario measures, when it happens in practice, and why it matters. Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/scaling_bench.rb | 15 +++ bench/scaling_results.html | 195 +++++++++++++++---------------------- 2 files changed, 93 insertions(+), 117 deletions(-) diff --git a/bench/scaling_bench.rb b/bench/scaling_bench.rb index 36122aac..4be72783 100644 --- a/bench/scaling_bench.rb +++ b/bench/scaling_bench.rb @@ -330,6 +330,8 @@ def fmt_scale(s) h1 { font-size: 1.3em; color: #333; } h2 { font-size: 1.1em; color: #555; margin-top: 2em; } .chart-container { background: white; border-radius: 8px; padding: 20px; margin: 16px 0; box-shadow: 0 1px 3px rgba(0,0,0,0.1); } + .chart-container p.desc { color: #666; font-size: 0.85em; margin: 0 0 12px 0; line-height: 1.5; } + .chart-container p.desc strong { color: #444; } canvas { max-height: 400px; } .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } @media (max-width: 800px) { .grid { grid-template-columns: 1fr; } } @@ -352,15 +354,28 @@ def fmt_scale(s)
+

Idle Poll — All partitions are fully caught up. No new messages exist. + This is what happens on every catch-up poll interval when the system is quiet. + Measures the cost of determining “nothing to do”.

+

Cold Drain — A new consumer group starts processing an existing log from scratch. + No offsets exist yet — each claim_next call must discover new partitions, create offsets, + and claim work. This is the per-claim cost during initial catch-up (sampled over #{options[:sample_size]} claims).

+

Warm Claim — All partitions have existing offsets (previously caught up), + then new messages arrive for some partitions. No discovery needed — offsets already exist. + This is the steady-state cost when the notifier or catch-up poller triggers processing.

+

Incremental Discovery — All existing partitions are caught up. + One message arrives for a brand-new partition (never seen before). Measures the cost of discovering + and claiming that single new partition against a backdrop of N existing offsets. + Skipped for scales > #{fmt_scale(INCR_MAX_SCALE)} due to prohibitive NOT EXISTS cost.

diff --git a/bench/scaling_results.html b/bench/scaling_results.html index c637039f..76a73249 100644 --- a/bench/scaling_results.html +++ b/bench/scaling_results.html @@ -8,6 +8,8 @@ h1 { font-size: 1.3em; color: #333; } h2 { font-size: 1.1em; color: #555; margin-top: 2em; } .chart-container { background: white; border-radius: 8px; padding: 20px; margin: 16px 0; box-shadow: 0 1px 3px rgba(0,0,0,0.1); } + .chart-container p.desc { color: #666; font-size: 0.85em; margin: 0 0 12px 0; line-height: 1.5; } + .chart-container p.desc strong { color: #444; } canvas { max-height: 400px; } .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } @media (max-width: 800px) { .grid { grid-template-columns: 1fr; } } @@ -21,24 +23,37 @@

CCC::Store#claim_next — Scaling Benchmark

- Generated 2026-03-15 18:50 · + Generated 2026-03-15 18:55 · Keys: 1, 2, 3 · - Scales: 10, 100, 1K, 10K, 100K, 1M · - Iterations: 2 (adaptive) · + Scales: 10, 100, 1K, 10K · + Iterations: 1 (adaptive) · Sample: 50 claims

+

Idle Poll — All partitions are fully caught up. No new messages exist. + This is what happens on every catch-up poll interval when the system is quiet. + Measures the cost of determining “nothing to do”.

+

Cold Drain — A new consumer group starts processing an existing log from scratch. + No offsets exist yet — each claim_next call must discover new partitions, create offsets, + and claim work. This is the per-claim cost during initial catch-up (sampled over 50 claims).

+

Warm Claim — All partitions have existing offsets (previously caught up), + then new messages arrive for some partitions. No discovery needed — offsets already exist. + This is the steady-state cost when the notifier or catch-up poller triggers processing.

+

Incremental Discovery — All existing partitions are caught up. + One message arrives for a brand-new partition (never seen before). Measures the cost of discovering + and claiming that single new partition against a backdrop of N existing offsets. + Skipped for scales > 10K due to prohibitive NOT EXISTS cost.

@@ -49,24 +64,18 @@

Raw Data

KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms) - 1100.25750.47880.37720.981 - 11000.1160.4350.48273.495 - 11,0000.11651.1561.3252119.5775 - 110,0000.11958.23489.63910931.7395 - 1100,0000.11886.9555103.1813— - 11,000,0000.119907.1531089.092— - 2100.1150.40530.44651.154 - 21000.12050.53480.58374.7125 - 21,0000.1111.83652.053191.048 - 210,0000.14114.827716.715818336.0155 - 2100,0000.1155162.5097180.0425— - 21,000,0000.1171679.57751876.217— - 3100.1180.41380.41321.1915 - 31000.11850.54580.63855.1345 - 31,0000.12152.21632.4238253.031 - 310,0000.119519.118220.528223965.6405 - 3100,0000.118208.8647227.058— - 31,000,0000.1192165.0592372.594— + 1100.1190.48150.40851.067 + 11000.1210.43950.48553.94 + 11,0000.1211.12251.28117.13 + 110,0000.138.0819.53110918.582 + 2100.1230.4240.40050.974 + 21000.1190.5130.57654.394 + 21,0000.1251.831.992192.275 + 210,0000.12715.086516.430518319.971 + 3100.1130.44150.44651.24 + 31000.1170.57650.65655.429 + 31,0000.1242.3822.446250.684 + 310,0000.12119.37520.81324177.391 @@ -76,150 +85,102 @@

Raw Data

{ "keys": 1, "scale": 10, - "idle_poll_ms": 0.2575, - "per_claim_cold_ms": 0.4788, - "per_claim_warm_ms": 0.3772, - "incremental_ms": 0.981 + "idle_poll_ms": 0.119, + "per_claim_cold_ms": 0.4815, + "per_claim_warm_ms": 0.4085, + "incremental_ms": 1.067 }, { "keys": 1, "scale": 100, - "idle_poll_ms": 0.116, - "per_claim_cold_ms": 0.435, - "per_claim_warm_ms": 0.4827, - "incremental_ms": 3.495 + "idle_poll_ms": 0.121, + "per_claim_cold_ms": 0.4395, + "per_claim_warm_ms": 0.4855, + "incremental_ms": 3.94 }, { "keys": 1, "scale": 1000, - "idle_poll_ms": 0.1165, - "per_claim_cold_ms": 1.156, - "per_claim_warm_ms": 1.3252, - "incremental_ms": 119.5775 + "idle_poll_ms": 0.121, + "per_claim_cold_ms": 1.1225, + "per_claim_warm_ms": 1.28, + "incremental_ms": 117.13 }, { "keys": 1, "scale": 10000, - "idle_poll_ms": 0.1195, - "per_claim_cold_ms": 8.2348, - "per_claim_warm_ms": 9.639, - "incremental_ms": 10931.7395 - }, - { - "keys": 1, - "scale": 100000, - "idle_poll_ms": 0.118, - "per_claim_cold_ms": 86.9555, - "per_claim_warm_ms": 103.1813, - "incremental_ms": null - }, - { - "keys": 1, - "scale": 1000000, - "idle_poll_ms": 0.119, - "per_claim_cold_ms": 907.153, - "per_claim_warm_ms": 1089.092, - "incremental_ms": null + "idle_poll_ms": 0.13, + "per_claim_cold_ms": 8.081, + "per_claim_warm_ms": 9.531, + "incremental_ms": 10918.582 } ], "2": [ { "keys": 2, "scale": 10, - "idle_poll_ms": 0.115, - "per_claim_cold_ms": 0.4053, - "per_claim_warm_ms": 0.4465, - "incremental_ms": 1.154 + "idle_poll_ms": 0.123, + "per_claim_cold_ms": 0.424, + "per_claim_warm_ms": 0.4005, + "incremental_ms": 0.974 }, { "keys": 2, "scale": 100, - "idle_poll_ms": 0.1205, - "per_claim_cold_ms": 0.5348, - "per_claim_warm_ms": 0.5837, - "incremental_ms": 4.7125 + "idle_poll_ms": 0.119, + "per_claim_cold_ms": 0.513, + "per_claim_warm_ms": 0.5765, + "incremental_ms": 4.394 }, { "keys": 2, "scale": 1000, - "idle_poll_ms": 0.111, - "per_claim_cold_ms": 1.8365, - "per_claim_warm_ms": 2.053, - "incremental_ms": 191.048 + "idle_poll_ms": 0.125, + "per_claim_cold_ms": 1.83, + "per_claim_warm_ms": 1.992, + "incremental_ms": 192.275 }, { "keys": 2, "scale": 10000, - "idle_poll_ms": 0.141, - "per_claim_cold_ms": 14.8277, - "per_claim_warm_ms": 16.7158, - "incremental_ms": 18336.0155 - }, - { - "keys": 2, - "scale": 100000, - "idle_poll_ms": 0.1155, - "per_claim_cold_ms": 162.5097, - "per_claim_warm_ms": 180.0425, - "incremental_ms": null - }, - { - "keys": 2, - "scale": 1000000, - "idle_poll_ms": 0.117, - "per_claim_cold_ms": 1679.5775, - "per_claim_warm_ms": 1876.217, - "incremental_ms": null + "idle_poll_ms": 0.127, + "per_claim_cold_ms": 15.0865, + "per_claim_warm_ms": 16.4305, + "incremental_ms": 18319.971 } ], "3": [ { "keys": 3, "scale": 10, - "idle_poll_ms": 0.118, - "per_claim_cold_ms": 0.4138, - "per_claim_warm_ms": 0.4132, - "incremental_ms": 1.1915 + "idle_poll_ms": 0.113, + "per_claim_cold_ms": 0.4415, + "per_claim_warm_ms": 0.4465, + "incremental_ms": 1.24 }, { "keys": 3, "scale": 100, - "idle_poll_ms": 0.1185, - "per_claim_cold_ms": 0.5458, - "per_claim_warm_ms": 0.6385, - "incremental_ms": 5.1345 + "idle_poll_ms": 0.117, + "per_claim_cold_ms": 0.5765, + "per_claim_warm_ms": 0.6565, + "incremental_ms": 5.429 }, { "keys": 3, "scale": 1000, - "idle_poll_ms": 0.1215, - "per_claim_cold_ms": 2.2163, - "per_claim_warm_ms": 2.4238, - "incremental_ms": 253.031 + "idle_poll_ms": 0.124, + "per_claim_cold_ms": 2.382, + "per_claim_warm_ms": 2.446, + "incremental_ms": 250.684 }, { "keys": 3, "scale": 10000, - "idle_poll_ms": 0.1195, - "per_claim_cold_ms": 19.1182, - "per_claim_warm_ms": 20.5282, - "incremental_ms": 23965.6405 - }, - { - "keys": 3, - "scale": 100000, - "idle_poll_ms": 0.118, - "per_claim_cold_ms": 208.8647, - "per_claim_warm_ms": 227.058, - "incremental_ms": null - }, - { - "keys": 3, - "scale": 1000000, - "idle_poll_ms": 0.119, - "per_claim_cold_ms": 2165.059, - "per_claim_warm_ms": 2372.594, - "incremental_ms": null + "idle_poll_ms": 0.121, + "per_claim_cold_ms": 19.375, + "per_claim_warm_ms": 20.813, + "incremental_ms": 24177.391 } ] }; From 1b3659ff8d0ba198bfb47891eb106b897878690c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 19:38:52 +0000 Subject: [PATCH 082/115] Skip offset scan when undiscovered messages exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When types_max_pos > discovery_position, new messages may belong to partitions that don't have offsets yet. Scanning all existing offsets to confirm "no work" is O(offsets) wasted queries. Skip straight to discovery instead. Also remove NOT EXISTS from discovery query — discover all partition tuples in the window and filter known ones in Ruby via a Set of partition_keys. INSERT OR IGNORE handles races. Incremental discovery at 10K offsets: 10.9s → 15ms (720x faster). Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/scaling_results.html | 153 ++++++++++++++++++++++--------------- lib/sourced/ccc/store.rb | 64 ++++++++++------ 2 files changed, 130 insertions(+), 87 deletions(-) diff --git a/bench/scaling_results.html b/bench/scaling_results.html index 76a73249..0798046e 100644 --- a/bench/scaling_results.html +++ b/bench/scaling_results.html @@ -23,10 +23,10 @@

CCC::Store#claim_next — Scaling Benchmark

- Generated 2026-03-15 18:55 · + Generated 2026-03-15 19:43 · Keys: 1, 2, 3 · - Scales: 10, 100, 1K, 10K · - Iterations: 1 (adaptive) · + Scales: 10, 100, 1K, 10K, 100K · + Iterations: 2 (adaptive) · Sample: 50 claims

@@ -64,18 +64,21 @@

Raw Data

KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms) - 1100.1190.48150.40851.067 - 11000.1210.43950.48553.94 - 11,0000.1211.12251.28117.13 - 110,0000.138.0819.53110918.582 - 2100.1230.4240.40050.974 - 21000.1190.5130.57654.394 - 21,0000.1251.831.992192.275 - 210,0000.12715.086516.430518319.971 - 3100.1130.44150.44651.24 - 31000.1170.57650.65655.429 - 31,0000.1242.3822.446250.684 - 310,0000.12119.37520.81324177.391 + 1100.2070.39230.3910.697 + 11000.1140.4370.47881.0375 + 11,0000.1111.28751.32052.5875 + 110,0000.123517.4339.63214.661 + 1100,0000.128135.8178102.9913— + 2100.1160.38820.45620.877 + 21000.13350.53270.58651.052 + 21,0000.11452.05732.0013.083 + 210,0000.113528.036516.443721.843 + 2100,0000.1145254.4025179.9813— + 3100.1160.42150.44131.014 + 31000.11350.56170.611.2725 + 31,0000.11852.37152.46153.4625 + 310,0000.12236.822820.76725.621 + 3100,0000.1245356.0062229.1015— @@ -85,102 +88,126 @@

Raw Data

{ "keys": 1, "scale": 10, - "idle_poll_ms": 0.119, - "per_claim_cold_ms": 0.4815, - "per_claim_warm_ms": 0.4085, - "incremental_ms": 1.067 + "idle_poll_ms": 0.207, + "per_claim_cold_ms": 0.3923, + "per_claim_warm_ms": 0.391, + "incremental_ms": 0.697 }, { "keys": 1, "scale": 100, - "idle_poll_ms": 0.121, - "per_claim_cold_ms": 0.4395, - "per_claim_warm_ms": 0.4855, - "incremental_ms": 3.94 + "idle_poll_ms": 0.114, + "per_claim_cold_ms": 0.437, + "per_claim_warm_ms": 0.4788, + "incremental_ms": 1.0375 }, { "keys": 1, "scale": 1000, - "idle_poll_ms": 0.121, - "per_claim_cold_ms": 1.1225, - "per_claim_warm_ms": 1.28, - "incremental_ms": 117.13 + "idle_poll_ms": 0.111, + "per_claim_cold_ms": 1.2875, + "per_claim_warm_ms": 1.3205, + "incremental_ms": 2.5875 }, { "keys": 1, "scale": 10000, - "idle_poll_ms": 0.13, - "per_claim_cold_ms": 8.081, - "per_claim_warm_ms": 9.531, - "incremental_ms": 10918.582 + "idle_poll_ms": 0.1235, + "per_claim_cold_ms": 17.433, + "per_claim_warm_ms": 9.632, + "incremental_ms": 14.661 + }, + { + "keys": 1, + "scale": 100000, + "idle_poll_ms": 0.128, + "per_claim_cold_ms": 135.8178, + "per_claim_warm_ms": 102.9913, + "incremental_ms": null } ], "2": [ { "keys": 2, "scale": 10, - "idle_poll_ms": 0.123, - "per_claim_cold_ms": 0.424, - "per_claim_warm_ms": 0.4005, - "incremental_ms": 0.974 + "idle_poll_ms": 0.116, + "per_claim_cold_ms": 0.3882, + "per_claim_warm_ms": 0.4562, + "incremental_ms": 0.877 }, { "keys": 2, "scale": 100, - "idle_poll_ms": 0.119, - "per_claim_cold_ms": 0.513, - "per_claim_warm_ms": 0.5765, - "incremental_ms": 4.394 + "idle_poll_ms": 0.1335, + "per_claim_cold_ms": 0.5327, + "per_claim_warm_ms": 0.5865, + "incremental_ms": 1.052 }, { "keys": 2, "scale": 1000, - "idle_poll_ms": 0.125, - "per_claim_cold_ms": 1.83, - "per_claim_warm_ms": 1.992, - "incremental_ms": 192.275 + "idle_poll_ms": 0.1145, + "per_claim_cold_ms": 2.0573, + "per_claim_warm_ms": 2.001, + "incremental_ms": 3.083 }, { "keys": 2, "scale": 10000, - "idle_poll_ms": 0.127, - "per_claim_cold_ms": 15.0865, - "per_claim_warm_ms": 16.4305, - "incremental_ms": 18319.971 + "idle_poll_ms": 0.1135, + "per_claim_cold_ms": 28.0365, + "per_claim_warm_ms": 16.4437, + "incremental_ms": 21.843 + }, + { + "keys": 2, + "scale": 100000, + "idle_poll_ms": 0.1145, + "per_claim_cold_ms": 254.4025, + "per_claim_warm_ms": 179.9813, + "incremental_ms": null } ], "3": [ { "keys": 3, "scale": 10, - "idle_poll_ms": 0.113, - "per_claim_cold_ms": 0.4415, - "per_claim_warm_ms": 0.4465, - "incremental_ms": 1.24 + "idle_poll_ms": 0.116, + "per_claim_cold_ms": 0.4215, + "per_claim_warm_ms": 0.4413, + "incremental_ms": 1.014 }, { "keys": 3, "scale": 100, - "idle_poll_ms": 0.117, - "per_claim_cold_ms": 0.5765, - "per_claim_warm_ms": 0.6565, - "incremental_ms": 5.429 + "idle_poll_ms": 0.1135, + "per_claim_cold_ms": 0.5617, + "per_claim_warm_ms": 0.61, + "incremental_ms": 1.2725 }, { "keys": 3, "scale": 1000, - "idle_poll_ms": 0.124, - "per_claim_cold_ms": 2.382, - "per_claim_warm_ms": 2.446, - "incremental_ms": 250.684 + "idle_poll_ms": 0.1185, + "per_claim_cold_ms": 2.3715, + "per_claim_warm_ms": 2.4615, + "incremental_ms": 3.4625 }, { "keys": 3, "scale": 10000, - "idle_poll_ms": 0.121, - "per_claim_cold_ms": 19.375, - "per_claim_warm_ms": 20.813, - "incremental_ms": 24177.391 + "idle_poll_ms": 0.122, + "per_claim_cold_ms": 36.8228, + "per_claim_warm_ms": 20.767, + "incremental_ms": 25.621 + }, + { + "keys": 3, + "scale": 100000, + "idle_poll_ms": 0.1245, + "per_claim_cold_ms": 356.0062, + "per_claim_warm_ms": 229.1015, + "incremental_ms": null } ] }; diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index c65db891..683c8c63 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -496,22 +496,25 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: .first return nil unless cg - # Short-circuit: if offsets exist and the latest message of the handled - # types is at or below highest_position, all work is done. O(1) via the - # message_type index — avoids the per-offset scan entirely on idle polls. - # Skipped when no offsets exist (e.g. after reset or first run) so that - # discovery can create them. + # Short-circuit: check the latest message position for handled types. + # If at or below highest_position and offsets exist, all work is done. + types_max_pos = db[@messages_table] + .where(message_type: handled_types) + .max(:position) || 0 + has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? - if has_offsets - types_max_pos = db[@messages_table] - .where(message_type: handled_types) - .max(:position) || 0 - return nil if types_max_pos <= cg[:highest_position] + return nil if has_offsets && types_max_pos <= cg[:highest_position] + + # Phase 1: Fast path — try existing offsets. + # Only worth scanning if discovery has already covered all messages + # (discovery_position >= types_max_pos). Otherwise new messages may + # belong to undiscovered partitions, and scanning caught-up offsets + # to confirm "no work" is O(offsets) wasted queries. + claimed = nil + if has_offsets && types_max_pos <= cg[:discovery_position] + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) end - # Phase 1: Fast path — try existing offsets - claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) - # Phase 2: Discovery — scan forward from watermark, create new offsets unless claimed discover_new_partitions(cg[:id], partition_by, handled_types) @@ -808,7 +811,6 @@ def discover_new_partitions(cg_id, partition_by, handled_types) ctes = [] selects = [] joins = [] - not_exists_joins = [] partition_by.each_with_index do |attr, i| ctes << <<~CTE attr#{i} AS ( @@ -819,11 +821,13 @@ def discover_new_partitions(cg_id, partition_by, handled_types) CTE selects << "a#{i}.kp_id AS kp_id_#{i}, a#{i}.val AS val_#{i}" joins << "JOIN attr#{i} a#{i} ON m.position = a#{i}.message_position" - not_exists_joins << "JOIN #{@offset_key_pairs_table} okp#{i} ON o.id = okp#{i}.offset_id AND okp#{i}.key_pair_id = a#{i}.kp_id" end group_by = partition_by.each_index.map { |i| "a#{i}.kp_id" }.join(', ') + # Discover all partition tuples in the window (no NOT EXISTS — fast). + # Duplicates are filtered in Ruby against known partition_keys, and + # INSERT OR IGNORE handles any remaining races. sql = <<~SQL WITH #{ctes.join(",\n")} SELECT #{selects.join(', ')}, @@ -833,11 +837,6 @@ def discover_new_partitions(cg_id, partition_by, handled_types) #{joins.join("\n")} WHERE m.message_type IN (#{types_list}) AND m.position > #{db.literal(discovery_pos)} - AND NOT EXISTS ( - SELECT 1 FROM #{@offsets_table} o - #{not_exists_joins.join("\n")} - WHERE o.consumer_group_id = #{db.literal(cg_id)} - ) GROUP BY #{group_by} ORDER BY min_pos ASC LIMIT #{DISCOVERY_BATCH_SIZE} @@ -845,11 +844,22 @@ def discover_new_partitions(cg_id, partition_by, handled_types) rows = db.fetch(sql).all + # Load known partition_keys for this consumer group to skip duplicates + known_keys = db[@offsets_table] + .where(consumer_group_id: cg_id) + .select_map(:partition_key) + .to_set + max_discovered_pos = 0 + new_rows = rows.reject do |row| + values = {} + partition_by.each_with_index { |attr, i| values[attr] = row[:"val_#{i}"] } + known_keys.include?(build_partition_key(partition_by, values)) + end - if rows.any? + if new_rows.any? db.transaction do - rows.each do |row| + new_rows.each do |row| values = {} kp_ids = [] partition_by.each_with_index do |attr, i| @@ -863,8 +873,14 @@ def discover_new_partitions(cg_id, partition_by, handled_types) end end - # Advance watermark: to max discovered position, or to current max if nothing found - new_watermark = max_discovered_pos > 0 ? max_discovered_pos : (latest_position) + # Advance watermark to the max position seen in the window (whether new or known). + # This ensures we don't re-scan these messages on the next discovery call. + max_window_pos = rows.any? ? rows.map { |r| r[:max_pos] }.max : 0 + new_watermark = [max_discovered_pos, max_window_pos, latest_position].select { |p| p > 0 }.min || discovery_pos + # If we found a full batch, advance only to the batch's max (more may follow). + # If we found fewer than a batch, we've scanned to the end — advance to latest. + new_watermark = latest_position if rows.size < DISCOVERY_BATCH_SIZE + if new_watermark > discovery_pos db[@consumer_groups_table].where(id: cg_id).update( discovery_position: new_watermark, From bd5af3d998142db18f2c9bfac4cc5807e6d7e7fc Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Sun, 15 Mar 2026 22:32:53 +0000 Subject: [PATCH 083/115] =?UTF-8?q?Remove=20known=5Fkeys=20Set=20from=20di?= =?UTF-8?q?scovery=20=E2=80=94=20rely=20on=20INSERT=20OR=20IGNORE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loading all partition_keys into a Ruby Set doesn't scale (20MB at 1M offsets). Not needed: create_offset_with_key_pairs already uses INSERT OR IGNORE with a unique index, so duplicate tuples are no-ops. Discovery is capped at DISCOVERY_BATCH_SIZE (100) tuples per call, bounding the cost of re-attempting known offsets. Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/scaling_results.html | 175 +++++++++++++++++++++---------------- lib/sourced/ccc/store.rb | 18 ++-- 2 files changed, 106 insertions(+), 87 deletions(-) diff --git a/bench/scaling_results.html b/bench/scaling_results.html index 0798046e..6c7765cd 100644 --- a/bench/scaling_results.html +++ b/bench/scaling_results.html @@ -23,10 +23,10 @@

CCC::Store#claim_next — Scaling Benchmark

- Generated 2026-03-15 19:43 · + Generated 2026-03-15 22:55 · Keys: 1, 2, 3 · - Scales: 10, 100, 1K, 10K, 100K · - Iterations: 2 (adaptive) · + Scales: 10, 100, 1K, 10K, 100K, 1M · + Iterations: 1 (adaptive) · Sample: 50 claims

@@ -64,21 +64,24 @@

Raw Data

KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms) - 1100.2070.39230.3910.697 - 11000.1140.4370.47881.0375 - 11,0000.1111.28751.32052.5875 - 110,0000.123517.4339.63214.661 - 1100,0000.128135.8178102.9913— - 2100.1160.38820.45620.877 - 21000.13350.53270.58651.052 - 21,0000.11452.05732.0013.083 - 210,0000.113528.036516.443721.843 - 2100,0000.1145254.4025179.9813— - 3100.1160.42150.44131.014 - 31000.11350.56170.611.2725 - 31,0000.11852.37152.46153.4625 - 310,0000.12236.822820.76725.621 - 3100,0000.1245356.0062229.1015— + 1100.1120.4750.37350.693 + 11000.1110.42950.4670.764 + 11,0000.1131.2391.2781.764 + 110,0000.11315.93959.39110.406 + 1100,0000.116134.379102.802— + 11,000,0000.1241371.13051093.0945— + 2100.1160.40250.40450.785 + 21000.1130.50.6111.012 + 21,0000.1152.1332.13252.488 + 210,0000.11727.043516.641517.072 + 2100,0000.121252.2725178.629— + 21,000,0000.142583.09551873.197— + 3100.1190.4340.40650.891 + 31000.1120.57350.661.211 + 31,0000.1212.3142.3982.942 + 310,0000.12135.30520.63320.734 + 3100,0000.129354.079225.447— + 31,000,0000.1883673.4612379.7215— @@ -88,41 +91,49 @@

Raw Data

{ "keys": 1, "scale": 10, - "idle_poll_ms": 0.207, - "per_claim_cold_ms": 0.3923, - "per_claim_warm_ms": 0.391, - "incremental_ms": 0.697 + "idle_poll_ms": 0.112, + "per_claim_cold_ms": 0.475, + "per_claim_warm_ms": 0.3735, + "incremental_ms": 0.693 }, { "keys": 1, "scale": 100, - "idle_poll_ms": 0.114, - "per_claim_cold_ms": 0.437, - "per_claim_warm_ms": 0.4788, - "incremental_ms": 1.0375 + "idle_poll_ms": 0.111, + "per_claim_cold_ms": 0.4295, + "per_claim_warm_ms": 0.467, + "incremental_ms": 0.764 }, { "keys": 1, "scale": 1000, - "idle_poll_ms": 0.111, - "per_claim_cold_ms": 1.2875, - "per_claim_warm_ms": 1.3205, - "incremental_ms": 2.5875 + "idle_poll_ms": 0.113, + "per_claim_cold_ms": 1.239, + "per_claim_warm_ms": 1.278, + "incremental_ms": 1.764 }, { "keys": 1, "scale": 10000, - "idle_poll_ms": 0.1235, - "per_claim_cold_ms": 17.433, - "per_claim_warm_ms": 9.632, - "incremental_ms": 14.661 + "idle_poll_ms": 0.113, + "per_claim_cold_ms": 15.9395, + "per_claim_warm_ms": 9.391, + "incremental_ms": 10.406 }, { "keys": 1, "scale": 100000, - "idle_poll_ms": 0.128, - "per_claim_cold_ms": 135.8178, - "per_claim_warm_ms": 102.9913, + "idle_poll_ms": 0.116, + "per_claim_cold_ms": 134.379, + "per_claim_warm_ms": 102.802, + "incremental_ms": null + }, + { + "keys": 1, + "scale": 1000000, + "idle_poll_ms": 0.124, + "per_claim_cold_ms": 1371.1305, + "per_claim_warm_ms": 1093.0945, "incremental_ms": null } ], @@ -131,40 +142,48 @@

Raw Data

"keys": 2, "scale": 10, "idle_poll_ms": 0.116, - "per_claim_cold_ms": 0.3882, - "per_claim_warm_ms": 0.4562, - "incremental_ms": 0.877 + "per_claim_cold_ms": 0.4025, + "per_claim_warm_ms": 0.4045, + "incremental_ms": 0.785 }, { "keys": 2, "scale": 100, - "idle_poll_ms": 0.1335, - "per_claim_cold_ms": 0.5327, - "per_claim_warm_ms": 0.5865, - "incremental_ms": 1.052 + "idle_poll_ms": 0.113, + "per_claim_cold_ms": 0.5, + "per_claim_warm_ms": 0.611, + "incremental_ms": 1.012 }, { "keys": 2, "scale": 1000, - "idle_poll_ms": 0.1145, - "per_claim_cold_ms": 2.0573, - "per_claim_warm_ms": 2.001, - "incremental_ms": 3.083 + "idle_poll_ms": 0.115, + "per_claim_cold_ms": 2.133, + "per_claim_warm_ms": 2.1325, + "incremental_ms": 2.488 }, { "keys": 2, "scale": 10000, - "idle_poll_ms": 0.1135, - "per_claim_cold_ms": 28.0365, - "per_claim_warm_ms": 16.4437, - "incremental_ms": 21.843 + "idle_poll_ms": 0.117, + "per_claim_cold_ms": 27.0435, + "per_claim_warm_ms": 16.6415, + "incremental_ms": 17.072 }, { "keys": 2, "scale": 100000, - "idle_poll_ms": 0.1145, - "per_claim_cold_ms": 254.4025, - "per_claim_warm_ms": 179.9813, + "idle_poll_ms": 0.121, + "per_claim_cold_ms": 252.2725, + "per_claim_warm_ms": 178.629, + "incremental_ms": null + }, + { + "keys": 2, + "scale": 1000000, + "idle_poll_ms": 0.14, + "per_claim_cold_ms": 2583.0955, + "per_claim_warm_ms": 1873.197, "incremental_ms": null } ], @@ -172,41 +191,49 @@

Raw Data

{ "keys": 3, "scale": 10, - "idle_poll_ms": 0.116, - "per_claim_cold_ms": 0.4215, - "per_claim_warm_ms": 0.4413, - "incremental_ms": 1.014 + "idle_poll_ms": 0.119, + "per_claim_cold_ms": 0.434, + "per_claim_warm_ms": 0.4065, + "incremental_ms": 0.891 }, { "keys": 3, "scale": 100, - "idle_poll_ms": 0.1135, - "per_claim_cold_ms": 0.5617, - "per_claim_warm_ms": 0.61, - "incremental_ms": 1.2725 + "idle_poll_ms": 0.112, + "per_claim_cold_ms": 0.5735, + "per_claim_warm_ms": 0.66, + "incremental_ms": 1.211 }, { "keys": 3, "scale": 1000, - "idle_poll_ms": 0.1185, - "per_claim_cold_ms": 2.3715, - "per_claim_warm_ms": 2.4615, - "incremental_ms": 3.4625 + "idle_poll_ms": 0.121, + "per_claim_cold_ms": 2.314, + "per_claim_warm_ms": 2.398, + "incremental_ms": 2.942 }, { "keys": 3, "scale": 10000, - "idle_poll_ms": 0.122, - "per_claim_cold_ms": 36.8228, - "per_claim_warm_ms": 20.767, - "incremental_ms": 25.621 + "idle_poll_ms": 0.121, + "per_claim_cold_ms": 35.305, + "per_claim_warm_ms": 20.633, + "incremental_ms": 20.734 }, { "keys": 3, "scale": 100000, - "idle_poll_ms": 0.1245, - "per_claim_cold_ms": 356.0062, - "per_claim_warm_ms": 229.1015, + "idle_poll_ms": 0.129, + "per_claim_cold_ms": 354.079, + "per_claim_warm_ms": 225.447, + "incremental_ms": null + }, + { + "keys": 3, + "scale": 1000000, + "idle_poll_ms": 0.188, + "per_claim_cold_ms": 3673.461, + "per_claim_warm_ms": 2379.7215, "incremental_ms": null } ] diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 683c8c63..b4014f62 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -844,22 +844,11 @@ def discover_new_partitions(cg_id, partition_by, handled_types) rows = db.fetch(sql).all - # Load known partition_keys for this consumer group to skip duplicates - known_keys = db[@offsets_table] - .where(consumer_group_id: cg_id) - .select_map(:partition_key) - .to_set - max_discovered_pos = 0 - new_rows = rows.reject do |row| - values = {} - partition_by.each_with_index { |attr, i| values[attr] = row[:"val_#{i}"] } - known_keys.include?(build_partition_key(partition_by, values)) - end - if new_rows.any? + if rows.any? db.transaction do - new_rows.each do |row| + rows.each do |row| values = {} kp_ids = [] partition_by.each_with_index do |attr, i| @@ -867,6 +856,9 @@ def discover_new_partitions(cg_id, partition_by, handled_types) kp_ids << row[:"kp_id_#{i}"] end + # INSERT OR IGNORE handles duplicates — no need to pre-filter. + # At most DISCOVERY_BATCH_SIZE (100) tuples per call, so the + # cost of re-attempting known offsets is bounded. create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) max_discovered_pos = row[:max_pos] if row[:max_pos] > max_discovered_pos end From 85c2e9b8d3a17867b5dda393d9ede627e19cf5b1 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 10:19:25 +0000 Subject: [PATCH 084/115] Add Store#append benchmark as baseline for eager offset creation Measures append latency across batch sizes (1-100), key counts (1-3), and pre-existing data volumes (0, 1K, 10K). Baseline per-message cost: ~0.1ms (1 key), ~0.15ms (2 keys), ~0.2ms (3 keys). Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/append_bench.rb | 266 +++++++++++++++++++++++++ bench/append_results.html | 402 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 668 insertions(+) create mode 100644 bench/append_bench.rb create mode 100644 bench/append_results.html diff --git a/bench/append_bench.rb b/bench/append_bench.rb new file mode 100644 index 00000000..f8bd15b6 --- /dev/null +++ b/bench/append_bench.rb @@ -0,0 +1,266 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Benchmark: measures Store#append performance across message counts +# and payload sizes. Baseline before eager offset creation. +# +# Usage: +# bundle exec ruby bench/append_bench.rb +# bundle exec ruby bench/append_bench.rb --counts 1,10,100 +# bundle exec ruby bench/append_bench.rb --keys 1,3 +# bundle exec ruby bench/append_bench.rb --output bench/append_results.html + +require 'bundler/setup' +require 'sequel' +require 'sourced' +require 'sourced/ccc' +require 'sourced/ccc/store' +require 'optparse' +require 'json' + +Sourced.config.logger = Logger.new(File::NULL) +Console.logger.off! + +# --- CLI options ----------------------------------------------------------- + +options = { + counts: [1, 10, 50, 100], # messages per append call + keys: [1, 2, 3], # payload attributes used as partition keys + iterations: 5, # iterations per measurement + pre_existing: [0, 1_000, 10_000], # pre-existing messages in the store + output: 'bench/append_results.html' +} + +OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options]" + opts.on('--counts LIST', 'Messages per append call') { |v| options[:counts] = v.split(',').map(&:to_i) } + opts.on('--keys LIST', 'Key counts') { |v| options[:keys] = v.split(',').map(&:to_i) } + opts.on('--iterations N', Integer, 'Iterations') { |v| options[:iterations] = v } + opts.on('--pre LIST', 'Pre-existing message counts') { |v| options[:pre_existing] = v.split(',').map(&:to_i) } + opts.on('--output FILE', 'HTML output') { |v| options[:output] = v } +end.parse! + +# --- Message definitions -------------------------------------------------- + +AppendBenchEvent = Sourced::CCC::Message.define('append_bench.event') do + attribute :k0, String + attribute :k1, String + attribute :k2, String +end + +# --- Helpers --------------------------------------------------------------- + +def new_store + db = Sequel.sqlite + store = Sourced::CCC::Store.new(db) + store.install! + [db, store] +end + +def seed_messages(db, count) + return if count == 0 + now = Time.now.iso8601 + (0...count).each_slice(10_000) do |slice| + db.transaction do + db[:sourced_messages].multi_insert( + slice.map { |i| + { message_id: "seed-#{i}", message_type: 'append_bench.seed', payload: '{"x":"y"}', created_at: now } + } + ) + end + end +end + +def build_messages(count, key_count, offset: 0) + count.times.map do |i| + n = offset + i + payload = {} + payload[:k0] = "v#{n}" if key_count >= 1 + payload[:k1] = "v#{n}" if key_count >= 2 + payload[:k2] = "v#{n}" if key_count >= 3 + AppendBenchEvent.new(payload: payload) + end +end + +def measure + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + [Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0, result] +end + +def median(values) + sorted = values.sort + mid = sorted.size / 2 + sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0 +end + +def fmt_scale(s) + return "#{s / 1_000_000}M" if s >= 1_000_000 + return "#{s / 1_000}K" if s >= 1_000 + s.to_s +end + +# --- Benchmark runner ------------------------------------------------------ + +results = [] + +options[:keys].each do |key_count| + options[:pre_existing].each do |pre| + options[:counts].each do |count| + label = "keys=#{key_count} pre=#{fmt_scale(pre)} count=#{count}" + $stderr.print " #{label}..." + + times = [] + per_msg_times = [] + + options[:iterations].times do + _db, store = new_store + seed_messages(_db, pre) + + msgs = build_messages(count, key_count) + + elapsed, _ = measure { store.append(msgs) } + times << elapsed + per_msg_times << elapsed / count + end + + row = { + keys: key_count, + pre_existing: pre, + count: count, + total_ms: (median(times) * 1000).round(3), + per_msg_ms: (median(per_msg_times) * 1000).round(3) + } + results << row + $stderr.puts " #{row[:total_ms]}ms total, #{row[:per_msg_ms]}ms/msg" + end + end +end + +# --- CSV output ------------------------------------------------------------ + +puts "keys,pre_existing,count,total_ms,per_msg_ms" +results.each do |r| + puts "#{r[:keys]},#{r[:pre_existing]},#{r[:count]},#{r[:total_ms]},#{r[:per_msg_ms]}" +end + +# --- HTML chart output ----------------------------------------------------- + +# Group by (keys, pre_existing) for charting +chart_data = {} +options[:keys].each do |k| + options[:pre_existing].each do |pre| + label = "#{k} key#{k > 1 ? 's' : ''}, #{fmt_scale(pre)} existing" + chart_data[label] = results.select { |r| r[:keys] == k && r[:pre_existing] == pre } + end +end + +colors = [ + '#2196F3', '#FF9800', '#4CAF50', '#9C27B0', '#F44336', + '#00BCD4', '#FF5722', '#8BC34A', '#3F51B5', '#CDDC39' +] + +html = <<~HTML + + + + CCC::Store#append — Benchmark + + + + +

CCC::Store#append — Benchmark

+

+ Generated #{Time.now.strftime('%Y-%m-%d %H:%M')} · + Keys: #{options[:keys].join(', ')} · + Batch sizes: #{options[:counts].join(', ')} · + Pre-existing: #{options[:pre_existing].map { |s| fmt_scale(s) }.join(', ')} · + Iterations: #{options[:iterations]} +

+ +
+
+

Total append time — Wall-clock time for a single + store.append(messages) call. Includes message insertion, key_pair + extraction/dedup, and message_key_pairs indexing. Varies with batch size + and number of payload attributes (keys).

+ +
+
+

Per-message cost — Total time divided by message count. + Shows the marginal cost of each additional message in the batch. Should be + roughly constant if append scales linearly with batch size.

+ +
+
+ +

Raw Data

+ + + + + + #{results.map { |r| + pre_fmt = r[:pre_existing].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + "" + }.join("\n ")} + +
KeysPre-existingBatch sizeTotal (ms)Per-msg (ms)
#{r[:keys]}#{pre_fmt}#{r[:count]}#{r[:total_ms]}#{r[:per_msg_ms]}
+ + + + +HTML + +File.write(options[:output], html) +$stderr.puts "\nChart written to #{options[:output]}" diff --git a/bench/append_results.html b/bench/append_results.html new file mode 100644 index 00000000..d956846e --- /dev/null +++ b/bench/append_results.html @@ -0,0 +1,402 @@ + + + + CCC::Store#append — Benchmark + + + + +

CCC::Store#append — Benchmark

+

+ Generated 2026-03-16 10:19 · + Keys: 1, 2, 3 · + Batch sizes: 1, 10, 50, 100 · + Pre-existing: 0, 1K, 10K · + Iterations: 5 +

+ +
+
+

Total append time — Wall-clock time for a single + store.append(messages) call. Includes message insertion, key_pair + extraction/dedup, and message_key_pairs indexing. Varies with batch size + and number of payload attributes (keys).

+ +
+
+

Per-message cost — Total time divided by message count. + Shows the marginal cost of each additional message in the batch. Should be + roughly constant if append scales linearly with batch size.

+ +
+
+ +

Raw Data

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeysPre-existingBatch sizeTotal (ms)Per-msg (ms)
1010.190.19
10101.0710.107
10505.3580.107
1010010.0920.101
11,00010.1940.194
11,000101.1810.118
11,000505.6710.113
11,00010011.1960.112
110,00010.2680.268
110,000101.2020.12
110,000505.2860.106
110,00010010.1780.102
2010.2210.221
20101.5470.155
20507.3080.146
2010015.6480.156
21,00010.2270.227
21,000101.6510.165
21,000507.6070.152
21,00010015.3640.154
210,00010.3670.367
210,000101.7260.173
210,000507.3760.148
210,00010015.3780.154
3010.2590.259
30102.0610.206
30509.9430.199
3010020.0030.2
31,00010.2690.269
31,000102.1530.215
31,0005010.4210.208
31,00010019.970.2
310,00010.3980.398
310,000102.2130.221
310,000509.7220.194
310,00010020.7540.208
+ + + + From c556cadca9bc3326a33086307bd1f6dcbd86ac7b Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 10:22:17 +0000 Subject: [PATCH 085/115] Optimise append: eliminate redundant SELECT queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Use Sequel's insert return value (last_insert_rowid) instead of a separate SELECT to get the message position. 2. Resolve key_pair_id via subquery in the message_key_pairs INSERT instead of a separate SELECT round-trip. Cuts per-message append cost by ~3x: 1 key: 0.10ms → 0.04ms 2 keys: 0.15ms → 0.05ms 3 keys: 0.20ms → 0.07ms Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/append_results.html | 218 +++++++++++++++++++------------------- lib/sourced/ccc/store.rb | 21 ++-- 2 files changed, 120 insertions(+), 119 deletions(-) diff --git a/bench/append_results.html b/bench/append_results.html index d956846e..f4dde74d 100644 --- a/bench/append_results.html +++ b/bench/append_results.html @@ -23,7 +23,7 @@

CCC::Store#append — Benchmark

- Generated 2026-03-16 10:19 · + Generated 2026-03-16 10:22 · Keys: 1, 2, 3 · Batch sizes: 1, 10, 50, 100 · Pre-existing: 0, 1K, 10K · @@ -52,42 +52,42 @@

Raw Data

KeysPre-existingBatch sizeTotal (ms)Per-msg (ms) - 1010.190.19 - 10101.0710.107 - 10505.3580.107 - 1010010.0920.101 - 11,00010.1940.194 - 11,000101.1810.118 - 11,000505.6710.113 - 11,00010011.1960.112 - 110,00010.2680.268 - 110,000101.2020.12 - 110,000505.2860.106 - 110,00010010.1780.102 - 2010.2210.221 - 20101.5470.155 - 20507.3080.146 - 2010015.6480.156 - 21,00010.2270.227 - 21,000101.6510.165 - 21,000507.6070.152 - 21,00010015.3640.154 - 210,00010.3670.367 - 210,000101.7260.173 - 210,000507.3760.148 - 210,00010015.3780.154 - 3010.2590.259 - 30102.0610.206 - 30509.9430.199 - 3010020.0030.2 - 31,00010.2690.269 - 31,000102.1530.215 - 31,0005010.4210.208 - 31,00010019.970.2 - 310,00010.3980.398 - 310,000102.2130.221 - 310,000509.7220.194 - 310,00010020.7540.208 + 1010.1150.115 + 10100.4660.047 + 10501.9880.04 + 101004.2570.043 + 11,00010.0940.094 + 11,000100.4620.046 + 11,000502.0310.041 + 11,0001003.9890.04 + 110,00010.1380.138 + 110,000100.530.053 + 110,000502.2910.046 + 110,0001004.4840.045 + 2010.1090.109 + 20100.5980.06 + 20502.8680.057 + 201005.4430.054 + 21,00010.1110.111 + 21,000100.6410.064 + 21,000502.7010.054 + 21,0001005.4450.054 + 210,00010.1760.176 + 210,000100.660.066 + 210,000502.960.059 + 210,0001005.4220.054 + 3010.1540.154 + 30100.7930.079 + 30503.8090.076 + 301006.7720.068 + 31,00010.1230.123 + 31,000100.7870.079 + 31,000503.4150.068 + 31,0001007.4270.074 + 310,00010.1790.179 + 310,000100.8120.081 + 310,000503.6690.073 + 310,0001006.8580.069 @@ -98,29 +98,29 @@

Raw Data

"keys": 1, "pre_existing": 0, "count": 1, - "total_ms": 0.19, - "per_msg_ms": 0.19 + "total_ms": 0.115, + "per_msg_ms": 0.115 }, { "keys": 1, "pre_existing": 0, "count": 10, - "total_ms": 1.071, - "per_msg_ms": 0.107 + "total_ms": 0.466, + "per_msg_ms": 0.047 }, { "keys": 1, "pre_existing": 0, "count": 50, - "total_ms": 5.358, - "per_msg_ms": 0.107 + "total_ms": 1.988, + "per_msg_ms": 0.04 }, { "keys": 1, "pre_existing": 0, "count": 100, - "total_ms": 10.092, - "per_msg_ms": 0.101 + "total_ms": 4.257, + "per_msg_ms": 0.043 } ], "1 key, 1K existing": [ @@ -128,29 +128,29 @@

Raw Data

"keys": 1, "pre_existing": 1000, "count": 1, - "total_ms": 0.194, - "per_msg_ms": 0.194 + "total_ms": 0.094, + "per_msg_ms": 0.094 }, { "keys": 1, "pre_existing": 1000, "count": 10, - "total_ms": 1.181, - "per_msg_ms": 0.118 + "total_ms": 0.462, + "per_msg_ms": 0.046 }, { "keys": 1, "pre_existing": 1000, "count": 50, - "total_ms": 5.671, - "per_msg_ms": 0.113 + "total_ms": 2.031, + "per_msg_ms": 0.041 }, { "keys": 1, "pre_existing": 1000, "count": 100, - "total_ms": 11.196, - "per_msg_ms": 0.112 + "total_ms": 3.989, + "per_msg_ms": 0.04 } ], "1 key, 10K existing": [ @@ -158,29 +158,29 @@

Raw Data

"keys": 1, "pre_existing": 10000, "count": 1, - "total_ms": 0.268, - "per_msg_ms": 0.268 + "total_ms": 0.138, + "per_msg_ms": 0.138 }, { "keys": 1, "pre_existing": 10000, "count": 10, - "total_ms": 1.202, - "per_msg_ms": 0.12 + "total_ms": 0.53, + "per_msg_ms": 0.053 }, { "keys": 1, "pre_existing": 10000, "count": 50, - "total_ms": 5.286, - "per_msg_ms": 0.106 + "total_ms": 2.291, + "per_msg_ms": 0.046 }, { "keys": 1, "pre_existing": 10000, "count": 100, - "total_ms": 10.178, - "per_msg_ms": 0.102 + "total_ms": 4.484, + "per_msg_ms": 0.045 } ], "2 keys, 0 existing": [ @@ -188,29 +188,29 @@

Raw Data

"keys": 2, "pre_existing": 0, "count": 1, - "total_ms": 0.221, - "per_msg_ms": 0.221 + "total_ms": 0.109, + "per_msg_ms": 0.109 }, { "keys": 2, "pre_existing": 0, "count": 10, - "total_ms": 1.547, - "per_msg_ms": 0.155 + "total_ms": 0.598, + "per_msg_ms": 0.06 }, { "keys": 2, "pre_existing": 0, "count": 50, - "total_ms": 7.308, - "per_msg_ms": 0.146 + "total_ms": 2.868, + "per_msg_ms": 0.057 }, { "keys": 2, "pre_existing": 0, "count": 100, - "total_ms": 15.648, - "per_msg_ms": 0.156 + "total_ms": 5.443, + "per_msg_ms": 0.054 } ], "2 keys, 1K existing": [ @@ -218,29 +218,29 @@

Raw Data

"keys": 2, "pre_existing": 1000, "count": 1, - "total_ms": 0.227, - "per_msg_ms": 0.227 + "total_ms": 0.111, + "per_msg_ms": 0.111 }, { "keys": 2, "pre_existing": 1000, "count": 10, - "total_ms": 1.651, - "per_msg_ms": 0.165 + "total_ms": 0.641, + "per_msg_ms": 0.064 }, { "keys": 2, "pre_existing": 1000, "count": 50, - "total_ms": 7.607, - "per_msg_ms": 0.152 + "total_ms": 2.701, + "per_msg_ms": 0.054 }, { "keys": 2, "pre_existing": 1000, "count": 100, - "total_ms": 15.364, - "per_msg_ms": 0.154 + "total_ms": 5.445, + "per_msg_ms": 0.054 } ], "2 keys, 10K existing": [ @@ -248,29 +248,29 @@

Raw Data

"keys": 2, "pre_existing": 10000, "count": 1, - "total_ms": 0.367, - "per_msg_ms": 0.367 + "total_ms": 0.176, + "per_msg_ms": 0.176 }, { "keys": 2, "pre_existing": 10000, "count": 10, - "total_ms": 1.726, - "per_msg_ms": 0.173 + "total_ms": 0.66, + "per_msg_ms": 0.066 }, { "keys": 2, "pre_existing": 10000, "count": 50, - "total_ms": 7.376, - "per_msg_ms": 0.148 + "total_ms": 2.96, + "per_msg_ms": 0.059 }, { "keys": 2, "pre_existing": 10000, "count": 100, - "total_ms": 15.378, - "per_msg_ms": 0.154 + "total_ms": 5.422, + "per_msg_ms": 0.054 } ], "3 keys, 0 existing": [ @@ -278,29 +278,29 @@

Raw Data

"keys": 3, "pre_existing": 0, "count": 1, - "total_ms": 0.259, - "per_msg_ms": 0.259 + "total_ms": 0.154, + "per_msg_ms": 0.154 }, { "keys": 3, "pre_existing": 0, "count": 10, - "total_ms": 2.061, - "per_msg_ms": 0.206 + "total_ms": 0.793, + "per_msg_ms": 0.079 }, { "keys": 3, "pre_existing": 0, "count": 50, - "total_ms": 9.943, - "per_msg_ms": 0.199 + "total_ms": 3.809, + "per_msg_ms": 0.076 }, { "keys": 3, "pre_existing": 0, "count": 100, - "total_ms": 20.003, - "per_msg_ms": 0.2 + "total_ms": 6.772, + "per_msg_ms": 0.068 } ], "3 keys, 1K existing": [ @@ -308,29 +308,29 @@

Raw Data

"keys": 3, "pre_existing": 1000, "count": 1, - "total_ms": 0.269, - "per_msg_ms": 0.269 + "total_ms": 0.123, + "per_msg_ms": 0.123 }, { "keys": 3, "pre_existing": 1000, "count": 10, - "total_ms": 2.153, - "per_msg_ms": 0.215 + "total_ms": 0.787, + "per_msg_ms": 0.079 }, { "keys": 3, "pre_existing": 1000, "count": 50, - "total_ms": 10.421, - "per_msg_ms": 0.208 + "total_ms": 3.415, + "per_msg_ms": 0.068 }, { "keys": 3, "pre_existing": 1000, "count": 100, - "total_ms": 19.97, - "per_msg_ms": 0.2 + "total_ms": 7.427, + "per_msg_ms": 0.074 } ], "3 keys, 10K existing": [ @@ -338,29 +338,29 @@

Raw Data

"keys": 3, "pre_existing": 10000, "count": 1, - "total_ms": 0.398, - "per_msg_ms": 0.398 + "total_ms": 0.179, + "per_msg_ms": 0.179 }, { "keys": 3, "pre_existing": 10000, "count": 10, - "total_ms": 2.213, - "per_msg_ms": 0.221 + "total_ms": 0.812, + "per_msg_ms": 0.081 }, { "keys": 3, "pre_existing": 10000, "count": 50, - "total_ms": 9.722, - "per_msg_ms": 0.194 + "total_ms": 3.669, + "per_msg_ms": 0.073 }, { "keys": 3, "pre_existing": 10000, "count": 100, - "total_ms": 20.754, - "per_msg_ms": 0.208 + "total_ms": 6.858, + "per_msg_ms": 0.069 } ] }; diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index b4014f62..af384c9c 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -156,7 +156,8 @@ def append(messages, guard: nil) payload_json = msg.payload ? JSON.dump(msg.payload.to_h) : '{}' metadata_json = msg.metadata.empty? ? nil : JSON.dump(msg.metadata) - db[@messages_table].insert( + # insert returns last_insert_rowid on SQLite — no need for a separate SELECT + last_position = db[@messages_table].insert( message_id: msg.id, message_type: msg.type, causation_id: msg.causation_id, @@ -166,17 +167,17 @@ def append(messages, guard: nil) created_at: msg.created_at.iso8601 ) - last_position = db[@messages_table].where(message_id: msg.id).get(:position) - - # Extract and index key pairs + # Upsert key pairs and link to message in 2 statements (was 3): + # 1. INSERT OR IGNORE the key_pair + # 2. INSERT message_key_pair with key_pair_id resolved via subquery msg.extracted_keys.each do |name, value| db.run("INSERT OR IGNORE INTO #{@key_pairs_table} (name, value) VALUES (#{db.literal(name)}, #{db.literal(value)})") - key_pair_id = db[@key_pairs_table].where(name: name, value: value).get(:id) - - db[@message_key_pairs_table].insert( - message_position: last_position, - key_pair_id: key_pair_id - ) + db.run(<<~SQL) + INSERT INTO #{@message_key_pairs_table} (message_position, key_pair_id) + SELECT #{db.literal(last_position)}, id + FROM #{@key_pairs_table} + WHERE name = #{db.literal(name)} AND value = #{db.literal(value)} + SQL end end end From 0153cfa3309ba275f28a084b7b040d44b24276b1 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 12:41:14 +0000 Subject: [PATCH 086/115] Fix claim_next short-circuit and add eager offset creation Replace highest_position-based short-circuit with min(last_position) across all offsets to avoid skipping unprocessed partitions during catch-up. Add eager offset creation path: register_consumer_group accepts partition_by, and append creates offsets inline for registered groups. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../migrations/001_create_ccc_tables.rb.erb | 1 + lib/sourced/ccc/router.rb | 5 +- lib/sourced/ccc/store.rb | 91 ++++++-- spec/sourced/ccc/router_spec.rb | 7 + spec/sourced/ccc/store_spec.rb | 201 ++++++++++++++++++ 5 files changed, 288 insertions(+), 17 deletions(-) diff --git a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb index 6dd5b92f..109d0b8b 100644 --- a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb +++ b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb @@ -47,6 +47,7 @@ Sequel.migration do String :status, null: false, default: 'active' Integer :highest_position, null: false, default: 0 Integer :discovery_position, null: false, default: 0 + String :partition_by String :error_context String :retry_at String :created_at, null: false diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index 60298bfc..b36e6954 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -15,7 +15,10 @@ def initialize(store:) def register(reactor_class) @reactors << reactor_class - store.register_consumer_group(reactor_class.group_id) + store.register_consumer_group( + reactor_class.group_id, + partition_by: reactor_class.partition_keys.map(&:to_s) + ) @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index af384c9c..ee8e5c4a 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'json' +require 'set' require 'sourced/inline_notifier' require 'sourced/ccc/installer' @@ -104,6 +105,11 @@ def initialize(db, notifier: nil, logger: nil, prefix: 'sourced') @offsets_table = @installer.offsets_table @offset_key_pairs_table = @installer.offset_key_pairs_table @workers_table = @installer.workers_table + + # Cache of registered consumer groups for eager offset creation in append. + # Populated by register_consumer_group. + # { group_id => { cg_id: Integer, partition_by: Array | nil } } + @registered_groups = {} end # Whether all required tables exist. @@ -180,6 +186,8 @@ def append(messages, guard: nil) SQL end end + + ensure_offsets_for_registered_groups(messages) end Console.info "AAA append #{messages.map(&:type).uniq}", messages: messages.size @@ -381,15 +389,25 @@ def messages_since(conditions, position) end # Register a consumer group. Idempotent. + # When +partition_by+ is provided, offsets are created eagerly during {#append} + # instead of lazily via discovery in {#claim_next}. # # @param group_id [String] unique identifier for the consumer group + # @param partition_by [Array, nil] attribute names defining partitions # @return [void] - def register_consumer_group(group_id) + def register_consumer_group(group_id, partition_by: nil) + partition_by_sorted = partition_by ? Array(partition_by).map(&:to_s).sort : nil + partition_by_json = partition_by_sorted ? JSON.dump(partition_by_sorted) : nil now = Time.now.iso8601 db.run(<<~SQL) - INSERT OR IGNORE INTO #{@consumer_groups_table} (group_id, status, highest_position, created_at, updated_at) - VALUES (#{db.literal(group_id)}, '#{ACTIVE}', 0, #{db.literal(now)}, #{db.literal(now)}) + INSERT INTO #{@consumer_groups_table} (group_id, status, highest_position, partition_by, created_at, updated_at) + VALUES (#{db.literal(group_id)}, '#{ACTIVE}', 0, #{db.literal(partition_by_json)}, #{db.literal(now)}, #{db.literal(now)}) + ON CONFLICT(group_id) DO UPDATE SET partition_by = #{db.literal(partition_by_json)}, updated_at = #{db.literal(now)} SQL + + # Cache for hot-path use in append + cg = db[@consumer_groups_table].where(group_id: group_id).first + @registered_groups[group_id] = { cg_id: cg[:id], partition_by: partition_by_sorted } end # Whether the consumer group exists and is active. @@ -498,28 +516,37 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: return nil unless cg # Short-circuit: check the latest message position for handled types. - # If at or below highest_position and offsets exist, all work is done. + # If the least-progressed offset is past types_max_pos, all work is done. types_max_pos = db[@messages_table] .where(message_type: handled_types) .max(:position) || 0 has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? - return nil if has_offsets && types_max_pos <= cg[:highest_position] + min_offset_pos = db[@offsets_table] + .where(consumer_group_id: cg[:id]) + .min(:last_position) + return nil if min_offset_pos && min_offset_pos >= types_max_pos - # Phase 1: Fast path — try existing offsets. - # Only worth scanning if discovery has already covered all messages - # (discovery_position >= types_max_pos). Otherwise new messages may - # belong to undiscovered partitions, and scanning caught-up offsets - # to confirm "no work" is O(offsets) wasted queries. claimed = nil - if has_offsets && types_max_pos <= cg[:discovery_position] - claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) - end + group_info = @registered_groups[group_id] - # Phase 2: Discovery — scan forward from watermark, create new offsets - unless claimed - discover_new_partitions(cg[:id], partition_by, handled_types) + if group_info&.fetch(:partition_by, nil) + # Eager path: offsets were created by append. Try fast claim first, + # fall back to discovery only for catch-up (new group against existing log). claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + unless claimed + discover_new_partitions(cg[:id], partition_by, handled_types) + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end + else + # Legacy path: lazy discovery + if has_offsets && types_max_pos <= cg[:discovery_position] + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end + unless claimed + discover_new_partitions(cg[:id], partition_by, handled_types) + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end end return nil unless claimed @@ -783,6 +810,38 @@ def resolve_group_id(group_id) group_id.respond_to?(:group_id) ? group_id.group_id : group_id end + # Create offsets eagerly for all registered consumer groups. + # Called inside the append transaction after messages and key_pairs are inserted. + # + # @param messages [Array] messages being appended + # @return [void] + def ensure_offsets_for_registered_groups(messages) + return if @registered_groups.empty? + + @registered_groups.each_value do |group_info| + partition_by = group_info[:partition_by] + next unless partition_by + + cg_id = group_info[:cg_id] + seen = Set.new + + messages.each do |msg| + keys = msg.extracted_keys.to_h # {"device_id"=>"dev-1", "name"=>"A"} + next unless partition_by.all? { |attr| keys.key?(attr) } + + values = partition_by.to_h { |attr| [attr, keys[attr]] } + pk = build_partition_key(partition_by, values) + next if seen.include?(pk) + seen << pk + + kp_ids = partition_by.map { |attr| + db[@key_pairs_table].where(name: attr, value: values[attr]).get(:id) + } + create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) + end + end + end + # Build canonical partition key string from attribute names and values. # Sorted by attribute name for deterministic uniqueness. # diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 41280eba..3d73398f 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -130,6 +130,13 @@ def self.handle_claim(claim) expect(router.instance_variable_get(:@needs_history)[RouterTestDecider]).to be true expect(router.instance_variable_get(:@needs_history)[RouterTestProjector]).to be false end + + it 'passes partition_keys to register_consumer_group' do + router.register(RouterTestDecider) + + row = db[:sourced_consumer_groups].where(group_id: 'router-test-decider').first + expect(JSON.parse(row[:partition_by])).to eq(['device_id']) + end end describe '#handle_next_for' do diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 460726cc..3d5db1c9 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1354,6 +1354,36 @@ module CCCStoreTestMessages expect(claimed_partitions).to contain_exactly('device_id:dev-1', 'device_id:dev-2', 'device_id:dev-3') end + it 'does not short-circuit remaining partitions after acking the one at max position' do + # Regression: highest_position short-circuit skipped unprocessed partitions + # when one partition's last message happened to be at types_max_pos. + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-4', name: 'D' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-5', name: 'E' }) + ]) + + new_group = 'multi-catch-up' + store.register_consumer_group(new_group) + + claimed_partitions = [] + 10.times do + r = store.claim_next(new_group, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') + break unless r + + claimed_partitions << r.partition_key + store.ack(new_group, offset_id: r.offset_id, position: r.messages.last.position) + end + + expect(claimed_partitions.size).to eq(5) + expect(claimed_partitions).to contain_exactly( + 'device_id:dev-1', 'device_id:dev-2', 'device_id:dev-3', + 'device_id:dev-4', 'device_id:dev-5' + ) + end + it 'only discovers partitions matching handled_types' do store.append([ CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), @@ -2001,4 +2031,175 @@ module CCCStoreTestMessages expect(r2.messages.map(&:position)).to eq(r1.messages.map(&:position)) end end + + describe 'eager offset creation' do + let(:group_id) { 'eager-test-group' } + + it 'creates offsets during append when partition_by is registered' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + offsets = db[:sourced_offsets].all + expect(offsets.size).to eq(1) + expect(offsets.first[:partition_key]).to eq('device_id:dev-1') + end + + it 'creates offsets for multiple partitions in a single append' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + + offsets = db[:sourced_offsets].order(:partition_key).all + expect(offsets.size).to eq(2) + expect(offsets.map { |o| o[:partition_key] }).to eq(['device_id:dev-1', 'device_id:dev-2']) + end + + it 'deduplicates offsets within the same append batch' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + ]) + + offsets = db[:sourced_offsets].all + expect(offsets.size).to eq(1) + end + + it 'is idempotent across multiple appends' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' })) + + offsets = db[:sourced_offsets].all + expect(offsets.size).to eq(1) + end + + it 'skips messages missing partition attributes' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + + # AssetRegistered has no device_id + store.append(CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' })) + + offsets = db[:sourced_offsets].all + expect(offsets).to be_empty + end + + it 'creates offsets for composite partitions' do + store.register_consumer_group(group_id, partition_by: [:course_name, :user_id]) + + store.append( + CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + ) + + offsets = db[:sourced_offsets].all + expect(offsets.size).to eq(1) + expect(offsets.first[:partition_key]).to eq('course_name:Algebra|user_id:joe') + end + + it 'skips messages with only partial composite partition attributes' do + store.register_consumer_group(group_id, partition_by: [:course_name, :user_id]) + + # CourseCreated only has course_name, not user_id + store.append(CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' })) + + offsets = db[:sourced_offsets].all + expect(offsets).to be_empty + end + + it 'does not create offsets when no consumer groups are registered' do + # No register_consumer_group call + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + offsets = db[:sourced_offsets].all + expect(offsets).to be_empty + end + + it 'does not create offsets when partition_by is nil (legacy group)' do + store.register_consumer_group(group_id) # no partition_by + + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + offsets = db[:sourced_offsets].all + expect(offsets).to be_empty + end + + it 'creates offsets for multiple registered consumer groups' do + store.register_consumer_group('group-a', partition_by: [:device_id]) + store.register_consumer_group('group-b', partition_by: [:device_id]) + + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + cg_a = db[:sourced_consumer_groups].where(group_id: 'group-a').first + cg_b = db[:sourced_consumer_groups].where(group_id: 'group-b').first + + offsets_a = db[:sourced_offsets].where(consumer_group_id: cg_a[:id]).all + offsets_b = db[:sourced_offsets].where(consumer_group_id: cg_b[:id]).all + + expect(offsets_a.size).to eq(1) + expect(offsets_b.size).to eq(1) + end + + it 'persists partition_by in consumer_groups table' do + store.register_consumer_group(group_id, partition_by: [:device_id, :name]) + + row = db[:sourced_consumer_groups].where(group_id: group_id).first + expect(JSON.parse(row[:partition_by])).to eq(['device_id', 'name']) + end + + it 'updates partition_by on re-registration' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + store.register_consumer_group(group_id, partition_by: [:asset_id]) + + row = db[:sourced_consumer_groups].where(group_id: group_id).first + expect(JSON.parse(row[:partition_by])).to eq(['asset_id']) + end + + it 'claim_next skips discovery when offsets already exist from append' do + store.register_consumer_group(group_id, partition_by: [:device_id]) + + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + # Offsets already exist from append — claim should work without discovery + result = store.claim_next( + group_id, + partition_by: 'device_id', + handled_types: ['store_test.device.registered'], + worker_id: 'w-1' + ) + + expect(result).not_to be_nil + expect(result.messages.size).to eq(1) + expect(result.partition_value).to eq({ 'device_id' => 'dev-1' }) + end + + it 'claim_next falls back to discovery for pre-existing messages' do + # Append BEFORE registering — no eager offsets + store.append( + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ) + + store.register_consumer_group(group_id, partition_by: [:device_id]) + + # claim_next should still find the message via discovery fallback + result = store.claim_next( + group_id, + partition_by: 'device_id', + handled_types: ['store_test.device.registered'], + worker_id: 'w-1' + ) + + expect(result).not_to be_nil + expect(result.messages.size).to eq(1) + end + end end From 6afd4a867b655edaa4b1a8ba9f72b7199c74312f Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 12:57:40 +0000 Subject: [PATCH 087/115] Scaling bench --- bench/scaling_bench.rb | 75 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/bench/scaling_bench.rb b/bench/scaling_bench.rb index 4be72783..bfa880ed 100644 --- a/bench/scaling_bench.rb +++ b/bench/scaling_bench.rb @@ -130,14 +130,18 @@ def seed(count, key_count, caught_up: false) # Simpler caught_up seeding: parse partition_key to match key_pairs # For the bulk offset_key_pairs insert, extract the value from partition_key # which has format "k0:v123" or "k0:v123|k1:v123" -def seed_caught_up_fast(count, key_count) +def seed_caught_up_fast(count, key_count, eager: false) db = Sequel.sqlite db.run('PRAGMA cache_size = -64000') db.run('PRAGMA synchronous = OFF') db.run('PRAGMA journal_mode = MEMORY') store = Sourced::CCC::Store.new(db) store.install! - store.register_consumer_group(GROUP_ID) + if eager + store.register_consumer_group(GROUP_ID, partition_by: partition_keys(key_count)) + else + store.register_consumer_group(GROUP_ID) + end cg_id = db[:sourced_consumer_groups].where(group_id: GROUP_ID).get(:id) now = Time.now.iso8601 @@ -295,6 +299,42 @@ def effective_iterations(scale, base) row[:incremental_ms] = nil end + # --- 5. Eager warm claim (offsets created by append, no discovery) --- + $stderr.print " [eager-warm" + eager_warm_times = [] + iters.times do + _db, store = seed_caught_up_fast(scale, key_count, eager: true) + msgs = (0...sample).map { |i| BenchEvent.new(payload: { k0: "v#{i}", k1: "v#{i}", k2: "v#{i}", k3: "v#{i}" }) } + store.append(msgs) + + times = [] + sample.times do + t, r = measure { claim_once(store, key_count) } + break unless r + store.ack(GROUP_ID, offset_id: r.offset_id, position: r.messages.last.position) + times << t + end + eager_warm_times << median(times) if times.any? + end + row[:eager_warm_ms] = eager_warm_times.any? ? (median(eager_warm_times) * 1000).round(4) : 0 + $stderr.print "=#{row[:eager_warm_ms]}ms]" + + # --- 6. Eager incremental (1 new partition, offset created by append) --- + if scale <= INCR_MAX_SCALE + $stderr.print " [eager-incr" + eager_incr_times = [] + iters.times do + _db, store = seed_caught_up_fast(scale, key_count, eager: true) + store.append(BenchEvent.new(payload: { k0: 'vnew', k1: 'vnew', k2: 'vnew', k3: 'vnew' })) + t, _ = measure { claim_once(store, key_count) } + eager_incr_times << t + end + row[:eager_incremental_ms] = (median(eager_incr_times) * 1000).round(4) + $stderr.print "=#{row[:eager_incremental_ms]}ms]" + else + row[:eager_incremental_ms] = nil + end + results << row end end @@ -302,10 +342,11 @@ def effective_iterations(scale, base) # --- CSV output ------------------------------------------------------------ $stderr.puts "\n" -puts "keys,scale,idle_poll_ms,per_claim_cold_ms,per_claim_warm_ms,incremental_ms" +puts "keys,scale,idle_poll_ms,per_claim_cold_ms,per_claim_warm_ms,incremental_ms,eager_warm_ms,eager_incremental_ms" results.each do |r| incr = r[:incremental_ms] ? r[:incremental_ms].to_s : '' - puts "#{r[:keys]},#{r[:scale]},#{r[:idle_poll_ms]},#{r[:per_claim_cold_ms]},#{r[:per_claim_warm_ms]},#{incr}" + eager_incr = r[:eager_incremental_ms] ? r[:eager_incremental_ms].to_s : '' + puts "#{r[:keys]},#{r[:scale]},#{r[:idle_poll_ms]},#{r[:per_claim_cold_ms]},#{r[:per_claim_warm_ms]},#{incr},#{r[:eager_warm_ms]},#{eager_incr}" end # --- HTML chart output ----------------------------------------------------- @@ -367,10 +408,16 @@ def fmt_scale(s)

Warm Claim — All partitions have existing offsets (previously caught up), - then new messages arrive for some partitions. No discovery needed — offsets already exist. + then new messages arrive for some partitions. Legacy path runs discovery to advance watermark. This is the steady-state cost when the notifier or catch-up poller triggers processing.

+
+

Eager Warm Claim — Same as Warm Claim, but with partition_by + registered. Offsets are created during append, so claim_next goes straight + to the fast path — no discovery CTE needed.

+ +

Incremental Discovery — All existing partitions are caught up. One message arrives for a brand-new partition (never seen before). Measures the cost of discovering @@ -378,18 +425,26 @@ def fmt_scale(s) Skipped for scales > #{fmt_scale(INCR_MAX_SCALE)} due to prohibitive NOT EXISTS cost.

+
+

Eager Incremental — Same scenario as Incremental, but with eager offset + creation. The offset is created during append, so claim_next finds it + on the fast path without running the discovery CTE. + Skipped for scales > #{fmt_scale(INCR_MAX_SCALE)}.

+ +

Raw Data

- + #{results.map { |r| scale_fmt = r[:scale].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse incr_fmt = r[:incremental_ms] ? r[:incremental_ms].to_s : '—' - "" + eager_incr_fmt = r[:eager_incremental_ms] ? r[:eager_incremental_ms].to_s : '—' + "" }.join("\n ")}
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms)
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm (ms)Eager Warm (ms)Incr (ms)Eager Incr (ms)
#{r[:keys]}#{scale_fmt}#{r[:idle_poll_ms]}#{r[:per_claim_cold_ms]}#{r[:per_claim_warm_ms]}#{incr_fmt}
#{r[:keys]}#{scale_fmt}#{r[:idle_poll_ms]}#{r[:per_claim_cold_ms]}#{r[:per_claim_warm_ms]}#{r[:eager_warm_ms]}#{incr_fmt}#{eager_incr_fmt}
@@ -444,8 +499,10 @@ def fmt_scale(s) makeChart('idlePoll', 'Idle Poll (all caught up, no work)', 'idle_poll_ms', 'ms'); makeChart('coldDrain', 'Cold Drain (per-claim cost, sampled)', 'per_claim_cold_ms', 'ms / claim'); - makeChart('warmClaim', 'Warm Claim (per-claim cost, sampled)', 'per_claim_warm_ms', 'ms / claim'); - makeChart('incremental', 'Incremental Discovery (1 new partition)', 'incremental_ms', 'ms'); + makeChart('warmClaim', 'Warm Claim — legacy (per-claim cost)', 'per_claim_warm_ms', 'ms / claim'); + makeChart('eagerWarm', 'Eager Warm Claim (per-claim cost)', 'eager_warm_ms', 'ms / claim'); + makeChart('incremental', 'Incremental Discovery — legacy (1 new partition)', 'incremental_ms', 'ms'); + makeChart('eagerIncremental', 'Eager Incremental (1 new partition)', 'eager_incremental_ms', 'ms'); From 7b6303fd0e12bf3bea3024891e3dbde6eedd72a4 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 12:58:02 +0000 Subject: [PATCH 088/115] Bench results --- bench/append_results.html | 216 +++++++++++++++++----------------- bench/scaling_results.html | 234 ++++++++++++++++++++----------------- 2 files changed, 234 insertions(+), 216 deletions(-) diff --git a/bench/append_results.html b/bench/append_results.html index f4dde74d..0167c234 100644 --- a/bench/append_results.html +++ b/bench/append_results.html @@ -23,7 +23,7 @@

CCC::Store#append — Benchmark

- Generated 2026-03-16 10:22 · + Generated 2026-03-16 11:27 · Keys: 1, 2, 3 · Batch sizes: 1, 10, 50, 100 · Pre-existing: 0, 1K, 10K · @@ -52,42 +52,42 @@

Raw Data

KeysPre-existingBatch sizeTotal (ms)Per-msg (ms) - 1010.1150.115 - 10100.4660.047 - 10501.9880.04 - 101004.2570.043 - 11,00010.0940.094 - 11,000100.4620.046 - 11,000502.0310.041 - 11,0001003.9890.04 - 110,00010.1380.138 - 110,000100.530.053 - 110,000502.2910.046 - 110,0001004.4840.045 - 2010.1090.109 - 20100.5980.06 - 20502.8680.057 - 201005.4430.054 - 21,00010.1110.111 - 21,000100.6410.064 - 21,000502.7010.054 - 21,0001005.4450.054 - 210,00010.1760.176 - 210,000100.660.066 - 210,000502.960.059 - 210,0001005.4220.054 - 3010.1540.154 - 30100.7930.079 - 30503.8090.076 - 301006.7720.068 - 31,00010.1230.123 - 31,000100.7870.079 - 31,000503.4150.068 - 31,0001007.4270.074 - 310,00010.1790.179 - 310,000100.8120.081 - 310,000503.6690.073 - 310,0001006.8580.069 + 1010.1590.159 + 10100.4390.044 + 10502.1210.042 + 101003.8140.038 + 11,00010.1350.135 + 11,000100.5260.053 + 11,000502.3790.048 + 11,0001004.7990.048 + 110,00010.1470.147 + 110,000100.5420.054 + 110,000502.2210.044 + 110,0001004.5760.046 + 2010.110.11 + 20100.5750.057 + 20502.8650.057 + 201005.7280.057 + 21,00010.120.12 + 21,000100.6930.069 + 21,000503.1430.063 + 21,0001006.4010.064 + 210,00010.1790.179 + 210,000100.6980.07 + 210,000503.1180.062 + 210,0001005.8530.059 + 3010.1420.142 + 30100.8410.084 + 30504.0550.081 + 301007.1390.071 + 31,00010.120.12 + 31,000100.7340.073 + 31,000503.4650.069 + 31,0001007.6050.076 + 310,00010.2030.203 + 310,000100.8720.087 + 310,000503.9050.078 + 310,0001007.310.073 @@ -98,29 +98,29 @@

Raw Data

"keys": 1, "pre_existing": 0, "count": 1, - "total_ms": 0.115, - "per_msg_ms": 0.115 + "total_ms": 0.159, + "per_msg_ms": 0.159 }, { "keys": 1, "pre_existing": 0, "count": 10, - "total_ms": 0.466, - "per_msg_ms": 0.047 + "total_ms": 0.439, + "per_msg_ms": 0.044 }, { "keys": 1, "pre_existing": 0, "count": 50, - "total_ms": 1.988, - "per_msg_ms": 0.04 + "total_ms": 2.121, + "per_msg_ms": 0.042 }, { "keys": 1, "pre_existing": 0, "count": 100, - "total_ms": 4.257, - "per_msg_ms": 0.043 + "total_ms": 3.814, + "per_msg_ms": 0.038 } ], "1 key, 1K existing": [ @@ -128,29 +128,29 @@

Raw Data

"keys": 1, "pre_existing": 1000, "count": 1, - "total_ms": 0.094, - "per_msg_ms": 0.094 + "total_ms": 0.135, + "per_msg_ms": 0.135 }, { "keys": 1, "pre_existing": 1000, "count": 10, - "total_ms": 0.462, - "per_msg_ms": 0.046 + "total_ms": 0.526, + "per_msg_ms": 0.053 }, { "keys": 1, "pre_existing": 1000, "count": 50, - "total_ms": 2.031, - "per_msg_ms": 0.041 + "total_ms": 2.379, + "per_msg_ms": 0.048 }, { "keys": 1, "pre_existing": 1000, "count": 100, - "total_ms": 3.989, - "per_msg_ms": 0.04 + "total_ms": 4.799, + "per_msg_ms": 0.048 } ], "1 key, 10K existing": [ @@ -158,29 +158,29 @@

Raw Data

"keys": 1, "pre_existing": 10000, "count": 1, - "total_ms": 0.138, - "per_msg_ms": 0.138 + "total_ms": 0.147, + "per_msg_ms": 0.147 }, { "keys": 1, "pre_existing": 10000, "count": 10, - "total_ms": 0.53, - "per_msg_ms": 0.053 + "total_ms": 0.542, + "per_msg_ms": 0.054 }, { "keys": 1, "pre_existing": 10000, "count": 50, - "total_ms": 2.291, - "per_msg_ms": 0.046 + "total_ms": 2.221, + "per_msg_ms": 0.044 }, { "keys": 1, "pre_existing": 10000, "count": 100, - "total_ms": 4.484, - "per_msg_ms": 0.045 + "total_ms": 4.576, + "per_msg_ms": 0.046 } ], "2 keys, 0 existing": [ @@ -188,29 +188,29 @@

Raw Data

"keys": 2, "pre_existing": 0, "count": 1, - "total_ms": 0.109, - "per_msg_ms": 0.109 + "total_ms": 0.11, + "per_msg_ms": 0.11 }, { "keys": 2, "pre_existing": 0, "count": 10, - "total_ms": 0.598, - "per_msg_ms": 0.06 + "total_ms": 0.575, + "per_msg_ms": 0.057 }, { "keys": 2, "pre_existing": 0, "count": 50, - "total_ms": 2.868, + "total_ms": 2.865, "per_msg_ms": 0.057 }, { "keys": 2, "pre_existing": 0, "count": 100, - "total_ms": 5.443, - "per_msg_ms": 0.054 + "total_ms": 5.728, + "per_msg_ms": 0.057 } ], "2 keys, 1K existing": [ @@ -218,29 +218,29 @@

Raw Data

"keys": 2, "pre_existing": 1000, "count": 1, - "total_ms": 0.111, - "per_msg_ms": 0.111 + "total_ms": 0.12, + "per_msg_ms": 0.12 }, { "keys": 2, "pre_existing": 1000, "count": 10, - "total_ms": 0.641, - "per_msg_ms": 0.064 + "total_ms": 0.693, + "per_msg_ms": 0.069 }, { "keys": 2, "pre_existing": 1000, "count": 50, - "total_ms": 2.701, - "per_msg_ms": 0.054 + "total_ms": 3.143, + "per_msg_ms": 0.063 }, { "keys": 2, "pre_existing": 1000, "count": 100, - "total_ms": 5.445, - "per_msg_ms": 0.054 + "total_ms": 6.401, + "per_msg_ms": 0.064 } ], "2 keys, 10K existing": [ @@ -248,29 +248,29 @@

Raw Data

"keys": 2, "pre_existing": 10000, "count": 1, - "total_ms": 0.176, - "per_msg_ms": 0.176 + "total_ms": 0.179, + "per_msg_ms": 0.179 }, { "keys": 2, "pre_existing": 10000, "count": 10, - "total_ms": 0.66, - "per_msg_ms": 0.066 + "total_ms": 0.698, + "per_msg_ms": 0.07 }, { "keys": 2, "pre_existing": 10000, "count": 50, - "total_ms": 2.96, - "per_msg_ms": 0.059 + "total_ms": 3.118, + "per_msg_ms": 0.062 }, { "keys": 2, "pre_existing": 10000, "count": 100, - "total_ms": 5.422, - "per_msg_ms": 0.054 + "total_ms": 5.853, + "per_msg_ms": 0.059 } ], "3 keys, 0 existing": [ @@ -278,29 +278,29 @@

Raw Data

"keys": 3, "pre_existing": 0, "count": 1, - "total_ms": 0.154, - "per_msg_ms": 0.154 + "total_ms": 0.142, + "per_msg_ms": 0.142 }, { "keys": 3, "pre_existing": 0, "count": 10, - "total_ms": 0.793, - "per_msg_ms": 0.079 + "total_ms": 0.841, + "per_msg_ms": 0.084 }, { "keys": 3, "pre_existing": 0, "count": 50, - "total_ms": 3.809, - "per_msg_ms": 0.076 + "total_ms": 4.055, + "per_msg_ms": 0.081 }, { "keys": 3, "pre_existing": 0, "count": 100, - "total_ms": 6.772, - "per_msg_ms": 0.068 + "total_ms": 7.139, + "per_msg_ms": 0.071 } ], "3 keys, 1K existing": [ @@ -308,29 +308,29 @@

Raw Data

"keys": 3, "pre_existing": 1000, "count": 1, - "total_ms": 0.123, - "per_msg_ms": 0.123 + "total_ms": 0.12, + "per_msg_ms": 0.12 }, { "keys": 3, "pre_existing": 1000, "count": 10, - "total_ms": 0.787, - "per_msg_ms": 0.079 + "total_ms": 0.734, + "per_msg_ms": 0.073 }, { "keys": 3, "pre_existing": 1000, "count": 50, - "total_ms": 3.415, - "per_msg_ms": 0.068 + "total_ms": 3.465, + "per_msg_ms": 0.069 }, { "keys": 3, "pre_existing": 1000, "count": 100, - "total_ms": 7.427, - "per_msg_ms": 0.074 + "total_ms": 7.605, + "per_msg_ms": 0.076 } ], "3 keys, 10K existing": [ @@ -338,29 +338,29 @@

Raw Data

"keys": 3, "pre_existing": 10000, "count": 1, - "total_ms": 0.179, - "per_msg_ms": 0.179 + "total_ms": 0.203, + "per_msg_ms": 0.203 }, { "keys": 3, "pre_existing": 10000, "count": 10, - "total_ms": 0.812, - "per_msg_ms": 0.081 + "total_ms": 0.872, + "per_msg_ms": 0.087 }, { "keys": 3, "pre_existing": 10000, "count": 50, - "total_ms": 3.669, - "per_msg_ms": 0.073 + "total_ms": 3.905, + "per_msg_ms": 0.078 }, { "keys": 3, "pre_existing": 10000, "count": 100, - "total_ms": 6.858, - "per_msg_ms": 0.069 + "total_ms": 7.31, + "per_msg_ms": 0.073 } ] }; diff --git a/bench/scaling_results.html b/bench/scaling_results.html index 6c7765cd..a70f70ca 100644 --- a/bench/scaling_results.html +++ b/bench/scaling_results.html @@ -23,10 +23,10 @@

CCC::Store#claim_next — Scaling Benchmark

- Generated 2026-03-15 22:55 · + Generated 2026-03-16 12:09 · Keys: 1, 2, 3 · - Scales: 10, 100, 1K, 10K, 100K, 1M · - Iterations: 1 (adaptive) · + Scales: 100, 1K, 10K, 100K, 1M · + Iterations: 2 (adaptive) · Sample: 50 claims

@@ -45,10 +45,16 @@

CCC::Store#claim_next — Scaling Benchmark

Warm Claim — All partitions have existing offsets (previously caught up), - then new messages arrive for some partitions. No discovery needed — offsets already exist. + then new messages arrive for some partitions. Legacy path runs discovery to advance watermark. This is the steady-state cost when the notifier or catch-up poller triggers processing.

+
+

Eager Warm Claim — Same as Warm Claim, but with partition_by + registered. Offsets are created during append, so claim_next goes straight + to the fast path — no discovery CTE needed.

+ +

Incremental Discovery — All existing partitions are caught up. One message arrives for a brand-new partition (never seen before). Measures the cost of discovering @@ -56,185 +62,195 @@

CCC::Store#claim_next — Scaling Benchmark

Skipped for scales > 10K due to prohibitive NOT EXISTS cost.

+
+

Eager Incremental — Same scenario as Incremental, but with eager offset + creation. The offset is created during append, so claim_next finds it + on the fast path without running the discovery CTE. + Skipped for scales > 10K.

+ +

Raw Data

- + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + +
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm /claim (ms)Incremental (ms)
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm (ms)Eager Warm (ms)Incr (ms)Eager Incr (ms)
1100.1120.4750.37350.693
11000.1110.42950.4670.764
11,0000.1131.2391.2781.764
110,0000.11315.93959.39110.406
1100,0000.116134.379102.802—
11,000,0000.1241371.13051093.0945—
2100.1160.40250.40450.785
21000.1130.50.6111.012
21,0000.1152.1332.13252.488
210,0000.11727.043516.641517.072
2100,0000.121252.2725178.629—
21,000,0000.142583.09551873.197—
3100.1190.4340.40650.891
31000.1120.57350.661.211
31,0000.1212.3142.3982.942
310,0000.12135.30520.63320.734
3100,0000.129354.079225.447—
31,000,0000.1883673.4612379.7215—
11000.15450.4430.48650.49250.80350.8365
11,0000.1131.34781.33231.3181.8661.438
110,0000.118516.40279.74959.65410.1729.476
1100,0000.1205133.7683102.8572102.696——
11,000,0000.1111377.7551090.8711093.1885——
21000.11250.56730.5760.57050.9880.62
21,0000.11851.96351.98282.02482.6192.2775
210,0000.1227.249716.44116.270516.916516.084
2100,0000.1155253.194179.9335179.7775——
21,000,0000.1162594.0711874.92851883.2475——
31000.1220.55830.64330.61851.220.6865
31,0000.11652.35582.39782.40682.91452.5075
310,0000.11435.423720.486720.383320.981520.2955
3100,0000.136356.1052226.6478226.9365——
31,000,0000.1183647.8922357.1292351.4825——
From 4af46ab6cd357144ff4e130a761b4b9189446a31 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 13:33:47 +0000 Subject: [PATCH 089/115] Fix idle poll regression: replace broken min/max short-circuit with cached watermark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The min(last_position) vs max(position) comparison was apples-to-oranges: a per-partition offset vs a cross-partition message max. After all partitions catch up, the short-circuit never fired, causing full offset scans + discovery on every idle poll. Replace with last_nil_types_max_pos: when claim_next finds no work, cache the current types_max_pos. Next poll compares in O(1) — if no new messages were appended, return nil instantly. Also batch key_pair lookups in ensure_offsets_for_registered_groups to eliminate N+1 queries, and move has_offsets query into legacy path only. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../migrations/001_create_ccc_tables.rb.erb | 1 + lib/sourced/ccc/store.rb | 31 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb index 109d0b8b..70ec29d4 100644 --- a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb +++ b/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb @@ -47,6 +47,7 @@ Sequel.migration do String :status, null: false, default: 'active' Integer :highest_position, null: false, default: 0 Integer :discovery_position, null: false, default: 0 + Integer :last_nil_types_max_pos, null: false, default: 0 String :partition_by String :error_context String :retry_at diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index ee8e5c4a..2c9fc2ee 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -481,6 +481,7 @@ def reset_consumer_group(group_id) db[@offsets_table].where(consumer_group_id: cg[:id]).delete db[@consumer_groups_table].where(id: cg[:id]).update( discovery_position: 0, + last_nil_types_max_pos: 0, updated_at: Time.now.iso8601 ) end @@ -515,17 +516,12 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: .first return nil unless cg - # Short-circuit: check the latest message position for handled types. - # If the least-progressed offset is past types_max_pos, all work is done. + # Short-circuit: no new messages since the last nil claim. types_max_pos = db[@messages_table] .where(message_type: handled_types) .max(:position) || 0 - has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? - min_offset_pos = db[@offsets_table] - .where(consumer_group_id: cg[:id]) - .min(:last_position) - return nil if min_offset_pos && min_offset_pos >= types_max_pos + return nil if types_max_pos <= cg[:last_nil_types_max_pos] claimed = nil group_info = @registered_groups[group_id] @@ -540,6 +536,7 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: end else # Legacy path: lazy discovery + has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? if has_offsets && types_max_pos <= cg[:discovery_position] claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) end @@ -549,7 +546,12 @@ def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: end end - return nil unless claimed + unless claimed + # Remember types_max_pos so next poll short-circuits instantly + db[@consumer_groups_table].where(id: cg[:id]) + .update(last_nil_types_max_pos: types_max_pos) + return nil + end key_pair_ids = db[@offset_key_pairs_table] .where(offset_id: claimed[:offset_id]) @@ -818,6 +820,15 @@ def resolve_group_id(group_id) def ensure_offsets_for_registered_groups(messages) return if @registered_groups.empty? + # Collect all partition attribute names across registered groups + attr_names = @registered_groups.each_value.flat_map { |gi| gi[:partition_by] || [] }.uniq + + # Pre-fetch relevant key_pair IDs in one query, keyed by "name:value" + kp_id_cache = {} + db[@key_pairs_table].where(name: attr_names).each do |row| + kp_id_cache["#{row[:name]}:#{row[:value]}"] = row[:id] + end + @registered_groups.each_value do |group_info| partition_by = group_info[:partition_by] next unless partition_by @@ -834,9 +845,7 @@ def ensure_offsets_for_registered_groups(messages) next if seen.include?(pk) seen << pk - kp_ids = partition_by.map { |attr| - db[@key_pairs_table].where(name: attr, value: values[attr]).get(:id) - } + kp_ids = partition_by.map { |attr| kp_id_cache["#{attr}:#{values[attr]}"] } create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) end end From 935c0a57d93bf44dc012d0aa1a55b6ace8b68f46 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 13:48:50 +0000 Subject: [PATCH 090/115] Add consumer group lifecycle hooks via Router Router now provides stop/reset/start_consumer_group methods that resolve a reactor class or group_id string, call the corresponding Store method, and invoke optional on_stop/on_reset/on_start callbacks on the reactor. Convenience delegators added to CCC module. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc.rb | 34 +++++++++ lib/sourced/ccc/README.md | 48 +++++++++++++ lib/sourced/ccc/consumer.rb | 26 +++++++ lib/sourced/ccc/router.rb | 69 ++++++++++++++++++ spec/sourced/ccc/router_spec.rb | 120 ++++++++++++++++++++++++++++++++ 5 files changed, 297 insertions(+) diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index 69a67100..f608b600 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -47,6 +47,40 @@ def self.router config.router end + # Stop a consumer group and invoke the reactor's +on_stop+ callback. + # Delegates to {Router#stop_consumer_group}. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @param message [String, nil] optional reason for stopping + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # @see Router#stop_consumer_group + def self.stop_consumer_group(reactor_or_id, message = nil) + config.router.stop_consumer_group(reactor_or_id, message) + end + + # Reset a consumer group and invoke the reactor's +on_reset+ callback. + # Delegates to {Router#reset_consumer_group}. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # @see Router#reset_consumer_group + def self.reset_consumer_group(reactor_or_id) + config.router.reset_consumer_group(reactor_or_id) + end + + # Start a consumer group and invoke the reactor's +on_start+ callback. + # Delegates to {Router#start_consumer_group}. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # @see Router#start_consumer_group + def self.start_consumer_group(reactor_or_id) + config.router.start_consumer_group(reactor_or_id) + end + # Reset the global configuration. For test teardown. def self.reset! @config = nil diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index 56f974f9..d1c10253 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -654,6 +654,54 @@ store.stop_consumer_group('CourseApp::CourseDecider') When retries are configured via `CCC.config.error_strategy`, failed consumer groups remain active but paused until their `retry_at` time. Once that time passes, they become claimable again automatically. +### Lifecycle hooks via Router + +The Router provides lifecycle methods that wrap the Store operations and invoke optional callbacks on the reactor class. This lets reactors run cleanup or setup logic when their consumer group is stopped, reset, or started. + +```ruby +# Accept a reactor class or a string group_id +Sourced::CCC.stop_consumer_group(CourseDecider, 'maintenance window') +Sourced::CCC.reset_consumer_group(CourseDecider) +Sourced::CCC.start_consumer_group(CourseDecider) + +# String group_id works too — the router resolves it to the registered class +Sourced::CCC.stop_consumer_group('CourseApp::CourseDecider') +``` + +These delegate to `Router#stop_consumer_group`, `Router#reset_consumer_group`, and `Router#start_consumer_group`, which: + +1. Resolve the argument to a registered reactor class (raising `ArgumentError` if the string doesn't match any registered reactor) +2. Call the corresponding `Store` method +3. Invoke the reactor's callback (`on_stop`, `on_reset`, `on_start`) + +#### Defining callbacks + +Override the no-op class methods on your reactor to hook into lifecycle events: + +```ruby +class CourseDecider < Sourced::CCC::Decider + partition_by :course_name + + # Called when the consumer group is stopped. + # `message` is the optional reason string passed to stop_consumer_group. + def self.on_stop(message = nil) + Rails.logger.info "CourseDecider stopped: #{message}" + end + + # Called when the consumer group is reset (offsets cleared). + def self.on_reset + Rails.cache.delete_matched('course_projections/*') + end + + # Called when the consumer group is started. + def self.on_start + Rails.logger.info 'CourseDecider started' + end +end +``` + +Reactors without custom callbacks work fine — the defaults are no-ops. + ## Monitoring `Store#stats` returns system-wide diagnostics for monitoring and debugging CCC deployments. diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb index 7ccc4b19..549e484e 100644 --- a/lib/sourced/ccc/consumer.rb +++ b/lib/sourced/ccc/consumer.rb @@ -85,6 +85,32 @@ def on_exception(exception, message, group) CCC.config.error_strategy.call(exception, message, group) end + # Called by {Router#stop_consumer_group} after the group is marked as stopped. + # Override in reactor classes to run cleanup logic on stop. + # + # @param message [String, nil] optional reason for stopping + # @return [void] + def on_stop(message = nil) + # no-op by default + end + + # Called by {Router#reset_consumer_group} after the group's offsets are cleared. + # Override in reactor classes to run cleanup logic on reset + # (e.g. clearing caches or projections). + # + # @return [void] + def on_reset + # no-op by default + end + + # Called by {Router#start_consumer_group} after the group is marked as active. + # Override in reactor classes to run setup logic on start. + # + # @return [void] + def on_start + # no-op by default + end + # Iterate messages collecting [actions, message] pairs. # On mid-batch failure, raises PartialBatchError with pairs collected so far. # If the first message fails, re-raises the original error. diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index b36e6954..015bb2ab 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -79,6 +79,63 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) end end + # Stop a consumer group and invoke the reactor's {Consumer#on_stop} callback. + # + # Marks the group as stopped in the store so workers will no longer claim + # work for it, then calls +on_stop+ on the reactor class. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @param message [String, nil] optional reason for stopping (persisted in the group's error_context) + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # + # @example Stop with a reactor class + # router.stop_consumer_group(CourseDecider, 'maintenance window') + # + # @example Stop with a string group_id + # router.stop_consumer_group('CourseDecider') + def stop_consumer_group(reactor_or_id, message = nil) + reactor_class = resolve_reactor_class(reactor_or_id) + store.stop_consumer_group(reactor_class.group_id, message) + reactor_class.on_stop(message) + end + + # Reset a consumer group and invoke the reactor's {Consumer#on_reset} callback. + # + # Clears all partition offsets and resets the discovery position to 0, + # so the group will reprocess messages from the beginning. Does not + # change the group's status (a stopped group remains stopped after reset). + # Then calls +on_reset+ on the reactor class. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # + # @example + # router.reset_consumer_group(CourseDecider) + def reset_consumer_group(reactor_or_id) + reactor_class = resolve_reactor_class(reactor_or_id) + store.reset_consumer_group(reactor_class.group_id) + reactor_class.on_reset + end + + # Start a consumer group and invoke the reactor's {Consumer#on_start} callback. + # + # Marks the group as active in the store so workers can claim work for it + # again, then calls +on_start+ on the reactor class. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # + # @example + # router.start_consumer_group(CourseDecider) + def start_consumer_group(reactor_or_id) + reactor_class = resolve_reactor_class(reactor_or_id) + store.start_consumer_group(reactor_class.group_id) + reactor_class.on_start + end + def drain(limit = Float::INFINITY) count = 0 loop do @@ -90,6 +147,18 @@ def drain(limit = Float::INFINITY) private + # Resolve a reactor class or group_id string to a registered reactor class. + # + # @param reactor_or_id [Class, String] a reactor class (returned as-is) or a +group_id+ string + # @return [Class] the matching registered reactor class + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + def resolve_reactor_class(reactor_or_id) + return reactor_or_id if reactor_or_id.is_a?(Module) + + @reactors.find { |r| r.group_id == reactor_or_id } || + raise(ArgumentError, "No reactor registered with group_id '#{reactor_or_id}'") + end + def execute_actions(action_pairs, claim, group_id) after_sync_actions = [] diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb index 3d73398f..a3d48be4 100644 --- a/spec/sourced/ccc/router_spec.rb +++ b/spec/sourced/ccc/router_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' require 'sourced/ccc' +require 'sourced/ccc/store' require 'sequel' module CCCRouterTestMessages @@ -363,6 +364,125 @@ def self.handle_claim(claim) end end + describe 'consumer group lifecycle' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + end + + describe '#stop_consumer_group' do + it 'stops group and calls on_stop with class' do + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be true + + router.stop_consumer_group(RouterTestDecider, 'maintenance') + + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false + end + + it 'stops group and calls on_stop with string group_id' do + router.stop_consumer_group('router-test-decider', 'maintenance') + + expect(store.consumer_group_active?('router-test-decider')).to be false + end + + it 'invokes on_stop callback on reactor class' do + allow(RouterTestDecider).to receive(:on_stop) + + router.stop_consumer_group(RouterTestDecider, 'going down') + + expect(RouterTestDecider).to have_received(:on_stop).with('going down') + end + + it 'passes nil message when none given' do + allow(RouterTestDecider).to receive(:on_stop) + + router.stop_consumer_group(RouterTestDecider) + + expect(RouterTestDecider).to have_received(:on_stop).with(nil) + end + end + + describe '#reset_consumer_group' do + it 'resets group offsets and calls on_reset with class' do + # Append a message and drain so offsets are advanced + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + router.drain + + router.reset_consumer_group(RouterTestDecider) + + # Offsets should be cleared (group can re-process messages) + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:discovery_position]).to eq(0) + end + + it 'resets group with string group_id' do + store.append( + CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + router.drain + + router.reset_consumer_group('router-test-decider') + + row = db[:sourced_consumer_groups].where(group_id: 'router-test-decider').first + expect(row[:discovery_position]).to eq(0) + end + + it 'invokes on_reset callback on reactor class' do + allow(RouterTestDecider).to receive(:on_reset) + + router.reset_consumer_group(RouterTestDecider) + + expect(RouterTestDecider).to have_received(:on_reset) + end + end + + describe '#start_consumer_group' do + it 'starts group and calls on_start with class' do + router.stop_consumer_group(RouterTestDecider) + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false + + router.start_consumer_group(RouterTestDecider) + + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be true + end + + it 'starts group with string group_id' do + router.stop_consumer_group('router-test-decider') + + router.start_consumer_group('router-test-decider') + + expect(store.consumer_group_active?('router-test-decider')).to be true + end + + it 'invokes on_start callback on reactor class' do + allow(RouterTestDecider).to receive(:on_start) + + router.start_consumer_group(RouterTestDecider) + + expect(RouterTestDecider).to have_received(:on_start) + end + end + + describe 'resolve_reactor_class' do + it 'raises ArgumentError for unregistered group_id' do + expect { + router.stop_consumer_group('unknown-group') + }.to raise_error(ArgumentError, /No reactor registered with group_id 'unknown-group'/) + end + end + + describe 'no-op default callbacks' do + it 'works fine when reactor does not override callbacks' do + # RouterTestProjector has no custom callbacks — should not raise + expect { router.stop_consumer_group(RouterTestProjector) }.not_to raise_error + expect { router.reset_consumer_group(RouterTestProjector) }.not_to raise_error + expect { router.start_consumer_group(RouterTestProjector) }.not_to raise_error + end + end + end + describe 'simple Consumer reactor (no Decider/Projector)' do before do router.register(RouterTestAuditReactor) From 939910ac6282de22d3084a8948023c70f1ee54db Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Tue, 17 Mar 2026 14:44:34 +0000 Subject: [PATCH 091/115] Make CCC::Store#read_all from_position inclusive Changed from_position to use >= (asc) and <= (desc) instead of strict > / <. Updated the internal fetcher to offset by 1 to prevent duplicates during auto-pagination via to_enum. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 6 +++--- spec/sourced/ccc/store_spec.rb | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 2c9fc2ee..63a6ca74 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -264,7 +264,7 @@ def update_schedule! # @example Next page (using the last position from the previous page) # messages = store.read_all(from_position: 20, limit: 20) # - # @param from_position [Integer] return messages after this position (default 0) + # @param from_position [Integer] return messages from this position, inclusive (default 0) # @param limit [Integer] max number of messages to return (default 50) # @return [ReadAllResult] messages and last global position def read_all(from_position: nil, limit: 50, order: :asc) @@ -272,14 +272,14 @@ def read_all(from_position: nil, limit: 50, order: :asc) ds = db[@messages_table] if from_position - ds = desc ? ds.where { position < from_position } : ds.where { position > from_position } + ds = desc ? ds.where { position <= from_position } : ds.where { position >= from_position } end messages = ds.order(desc ? Sequel.desc(:position) : :position) .limit(limit) .map { |row| deserialize(row) } - fetcher = ->(pos) { read_all(from_position: pos, limit: limit, order: order) } + fetcher = ->(pos) { read_all(from_position: desc ? pos - 1 : pos + 1, limit: limit, order: order) } ReadAllResult.new(messages: messages, last_position: latest_position, fetcher: fetcher) end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 3d5db1c9..37ba725a 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -347,11 +347,12 @@ module CCCStoreTestMessages expect(result1.messages.map(&:position)).to eq([1, 2]) expect(result1.last_position).to eq(5) - result2 = store.read_all(from_position: result1.messages.last.position, limit: 2) - expect(result2.messages.map(&:position)).to eq([3, 4]) + # from_position is inclusive, so position 2 is included + result2 = store.read_all(from_position: 2, limit: 2) + expect(result2.messages.map(&:position)).to eq([2, 3]) - result3 = store.read_all(from_position: result2.messages.last.position, limit: 2) - expect(result3.messages.map(&:position)).to eq([5]) + result3 = store.read_all(from_position: 4, limit: 2) + expect(result3.messages.map(&:position)).to eq([4, 5]) end it 'returns empty messages with last_position 0 for an empty store' do @@ -381,20 +382,21 @@ module CCCStoreTestMessages expect(result.last_position).to eq(5) end - it 'paginates in descending order using from_position' do + it 'paginates in descending order using from_position (inclusive)' do result1 = store.read_all(order: :desc, limit: 2) expect(result1.messages.map(&:position)).to eq([5, 4]) - result2 = store.read_all(from_position: result1.messages.last.position, order: :desc, limit: 2) - expect(result2.messages.map(&:position)).to eq([3, 2]) + # from_position is inclusive, so position 4 is included + result2 = store.read_all(from_position: 4, order: :desc, limit: 2) + expect(result2.messages.map(&:position)).to eq([4, 3]) - result3 = store.read_all(from_position: result2.messages.last.position, order: :desc, limit: 2) - expect(result3.messages.map(&:position)).to eq([1]) + result3 = store.read_all(from_position: 2, order: :desc, limit: 2) + expect(result3.messages.map(&:position)).to eq([2, 1]) end - it 'returns [] when no messages before from_position' do + it 'returns only the first message when from_position is 1 (inclusive)' do result = store.read_all(from_position: 1, order: :desc) - expect(result.messages).to eq([]) + expect(result.messages.map(&:position)).to eq([1]) expect(result.last_position).to eq(5) end end From 985568ad845b886359b946bf1da9ceaae2e5608c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Tue, 17 Mar 2026 16:52:13 +0000 Subject: [PATCH 092/115] Add CCC::Store#read_offsets for paginated offset inspection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds observability for consumer group partitions — lists offsets with group filtering, cursor-based pagination, and claim status fields. Includes OffsetsResult type with Enumerable, destructuring, and to_enum auto-pagination. Documents in CCC README. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/README.md | 74 +++++++++++++++++ lib/sourced/ccc/store.rb | 66 +++++++++++++++ spec/sourced/ccc/store_spec.rb | 144 +++++++++++++++++++++++++++++++++ 3 files changed, 284 insertions(+) diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md index d1c10253..3df14ed8 100644 --- a/lib/sourced/ccc/README.md +++ b/lib/sourced/ccc/README.md @@ -746,6 +746,80 @@ stats.groups.each do |g| end ``` +### `Store#read_offsets` — inspecting partition offsets + +`read_offsets` lists individual consumer group offsets with optional filtering and cursor-based pagination. Useful for inspecting the progress and claim status of each partition. + +```ruby +result = store.read_offsets +result.offsets # => array of offset hashes +result.total_count # => total number of matching offsets (ignoring pagination) +``` + +#### Parameters + +| Parameter | Type | Default | Description | +|-------------|----------------|---------|----------------------------------------------------------| +| `group_id:` | `String`, `nil` | `nil` | Filter by consumer group. `nil` returns all groups. | +| `limit:` | `Integer` | `50` | Max offsets per page. | +| `from_id:` | `Integer`, `nil`| `nil` | Cursor — return offsets with `id >= from_id` (inclusive). | + +#### Offset hash fields + +Each offset in the result is a Hash with: + +| Key | Type | Description | +|------------------|---------------|------------------------------------------------------| +| `:id` | `Integer` | Offset primary key (used as pagination cursor) | +| `:group_name` | `String` | Consumer group identifier | +| `:group_status` | `String` | `"active"`, `"stopped"`, or `"failed"` | +| `:partition_key` | `String` | Partition identifier (e.g. `"device_id:dev-1"`) | +| `:last_position` | `Integer` | Highest acked position for this partition | +| `:claimed` | `Boolean` | Whether a worker currently holds this partition | +| `:claimed_at` | `String`, `nil`| ISO8601 timestamp of the claim | +| `:claimed_by` | `String`, `nil`| Worker ID holding the claim | + +#### Filtering by group + +```ruby +result = store.read_offsets(group_id: 'CourseDecider') +result.offsets.each do |o| + puts "#{o[:partition_key]}: position #{o[:last_position]}, claimed=#{o[:claimed]}" +end +``` + +#### Pagination + +```ruby +# First page +page1 = store.read_offsets(limit: 20) + +# Next page using cursor +page2 = store.read_offsets(limit: 20, from_id: page1.offsets.last[:id] + 1) +``` + +#### Auto-pagination with `to_enum` + +`OffsetsResult#to_enum` returns a lazy `Enumerator` that fetches subsequent pages automatically. + +```ruby +# Iterate all offsets in pages of 20 +store.read_offsets(limit: 20).to_enum.each do |offset| + puts "#{offset[:group_name]} / #{offset[:partition_key]}: #{offset[:last_position]}" +end + +# Works with Enumerable methods +behind = store.read_offsets(limit: 50).to_enum.lazy.select { |o| + o[:last_position] < store.latest_position - 100 +}.to_a +``` + +#### Array destructuring + +```ruby +offsets, total_count = store.read_offsets(group_id: 'CourseDecider') +``` + ## Testing CCC ships with RSpec helpers for Given-When-Then testing of deciders and projectors. The helpers call `handle_batch` directly — no store, router, or consumer group setup needed. diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 63a6ca74..6ecce2a9 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -59,6 +59,29 @@ def to_enum Stats = Data.define(:max_position, :groups) + OffsetsResult = Data.define(:offsets, :total_count, :fetcher) do + include Enumerable + + def to_ary = [offsets, total_count] + + # Iterates offsets in the current page. + def each(&block) = offsets.each(&block) + + # Returns an Enumerator that lazily paginates through all offsets, + # fetching subsequent pages as needed. + def to_enum + Enumerator.new do |y| + result = self + loop do + break if result.offsets.empty? + + result.offsets.each { |o| y << o } + result = result.fetcher.call(result.offsets.last[:id] + 1) + end + end + end + end + # SQLite-backed store for CCC's flat, globally-ordered message log. # Provides message storage with automatic key-pair indexing, # consumer group management, and partition-based offset tracking @@ -762,6 +785,49 @@ def stats Stats.new(max_position: latest_position, groups: groups) end + # List offsets with optional group filtering and cursor-based pagination. + # + # @param group_id [String, nil] filter by consumer group (nil = all groups) + # @param limit [Integer] max offsets per page (default 50) + # @param from_id [Integer, nil] cursor — return offsets with id >= from_id (inclusive) + # @return [CCC::OffsetsResult] paginated offsets with total_count and fetcher for auto-pagination + def read_offsets(group_id: nil, limit: 50, from_id: nil) + dataset = db[@offsets_table].join(@consumer_groups_table, id: :consumer_group_id) + .select( + Sequel[@offsets_table][:id], + Sequel[@consumer_groups_table][:group_id].as(:group_name), + Sequel[@consumer_groups_table][:status].as(:group_status), + Sequel[@offsets_table][:partition_key], + Sequel[@offsets_table][:last_position], + Sequel[@offsets_table][:claimed], + Sequel[@offsets_table][:claimed_at], + Sequel[@offsets_table][:claimed_by] + ) + .order(Sequel[@offsets_table][:id]) + + count_dataset = db[@offsets_table].join(@consumer_groups_table, id: :consumer_group_id) + + if group_id + dataset = dataset.where(Sequel[@consumer_groups_table][:group_id] => group_id) + count_dataset = count_dataset.where(Sequel[@consumer_groups_table][:group_id] => group_id) + end + + if from_id + dataset = dataset.where(Sequel[@offsets_table][:id] >= from_id) + end + + total_count = count_dataset.count + offsets = dataset.limit(limit).all + + offsets.each do |o| + o[:claimed] = o[:claimed] == 1 + end + + fetcher = ->(next_from_id) { read_offsets(group_id: group_id, limit: limit, from_id: next_from_id) } + + OffsetsResult.new(offsets: offsets, total_count: total_count, fetcher: fetcher) + end + # Fetch all messages sharing the same correlation_id as the given message. # Useful for tracing causal chains (command -> events -> reactions). # diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 37ba725a..f99d07b6 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -1946,6 +1946,150 @@ module CCCStoreTestMessages end end + describe '#read_offsets' do + it 'returns empty result when no offsets exist' do + result = store.read_offsets + expect(result).to be_a(Sourced::CCC::OffsetsResult) + expect(result.offsets).to eq([]) + expect(result.total_count).to eq(0) + end + + it 'returns all offsets across groups with group_name and group_status' do + store.register_consumer_group('group-a') + store.register_consumer_group('group-b') + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + ]) + + # Claim to bootstrap offsets in both groups + store.claim_next('group-a', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + store.claim_next('group-b', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-2') + + result = store.read_offsets + expect(result.total_count).to be >= 2 + + group_names = result.offsets.map { |o| o[:group_name] }.uniq.sort + expect(group_names).to include('group-a', 'group-b') + + result.offsets.each do |o| + expect(o).to have_key(:id) + expect(o).to have_key(:group_name) + expect(o).to have_key(:group_status) + expect(o).to have_key(:partition_key) + expect(o).to have_key(:last_position) + expect(o).to have_key(:claimed) + expect(o).to have_key(:claimed_at) + expect(o).to have_key(:claimed_by) + expect(o[:group_status]).to eq('active') + end + end + + it 'filters by group_id when provided' do + store.register_consumer_group('group-a') + store.register_consumer_group('group-b') + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ]) + + store.claim_next('group-a', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + store.claim_next('group-b', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-2') + + result = store.read_offsets(group_id: 'group-a') + expect(result.offsets).to all(satisfy { |o| o[:group_name] == 'group-a' }) + expect(result.total_count).to eq(result.offsets.size) + end + + it 'paginates with from_id and limit (inclusive)' do + store.register_consumer_group('group-a') + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + ]) + + # Claim each to bootstrap offsets + 3.times do + store.claim_next('group-a', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + end + + # Get first page of 2 + page1 = store.read_offsets(group_id: 'group-a', limit: 2) + expect(page1.offsets.size).to eq(2) + expect(page1.total_count).to eq(3) + + # Get second page starting from next id (inclusive) + page2 = store.read_offsets(group_id: 'group-a', limit: 2, from_id: page1.offsets.last[:id] + 1) + expect(page2.offsets.size).to eq(1) + + # No overlap between pages + page1_ids = page1.offsets.map { |o| o[:id] } + page2_ids = page2.offsets.map { |o| o[:id] } + expect(page1_ids & page2_ids).to be_empty + end + + it 'to_enum iterates across pages' do + store.register_consumer_group('group-a') + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + ]) + + 3.times do + store.claim_next('group-a', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + end + + # Paginate with limit: 1 to force multiple fetches + result = store.read_offsets(group_id: 'group-a', limit: 1) + all_offsets = result.to_enum.to_a + expect(all_offsets.size).to eq(3) + expect(all_offsets.map { |o| o[:id] }).to eq(all_offsets.map { |o| o[:id] }.sort) + end + + it 'includes claim status fields' do + store.register_consumer_group('group-a') + + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + ]) + + # Claim creates an offset that is claimed + claim = store.claim_next('group-a', partition_by: 'device_id', + handled_types: ['store_test.device.registered'], worker_id: 'w-1') + + result = store.read_offsets(group_id: 'group-a') + offset = result.offsets.first + + expect(offset[:claimed]).to be true + expect(offset[:claimed_by]).to eq('w-1') + + # Ack releases the claim + store.ack('group-a', offset_id: claim.offset_id, position: claim.messages.last.position) + + result = store.read_offsets(group_id: 'group-a') + offset = result.offsets.first + + expect(offset[:claimed]).to be false + end + + it 'supports array destructuring' do + offsets, total = store.read_offsets + expect(offsets).to eq([]) + expect(total).to eq(0) + end + end + describe '#read_correlation_batch' do it 'returns all messages sharing the same correlation_id, ordered by position' do # Create a command (source of the correlation chain) From d5680075462c79be8a46bd4dbfa5d8a313ac0a24 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 18 Mar 2026 16:12:00 +0000 Subject: [PATCH 093/115] Remove logging --- lib/sourced/ccc/router.rb | 12 ------------ lib/sourced/ccc/store.rb | 1 - 2 files changed, 13 deletions(-) diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb index 015bb2ab..84d1274d 100644 --- a/lib/sourced/ccc/router.rb +++ b/lib/sourced/ccc/router.rb @@ -25,7 +25,6 @@ def register(reactor_class) def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) handled_types = reactor_class.handled_messages.map(&:type).uniq - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) claim = store.claim_next( reactor_class.group_id, partition_by: reactor_class.partition_keys.map(&:to_s), @@ -33,8 +32,6 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) worker_id: worker_id, batch_size: batch_size ) - t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - Console.info "AAA #{reactor_class.name} claim_next=#{((t1-t0)*1000).round(1)}ms found=#{!!claim}" return false unless claim begin @@ -42,16 +39,10 @@ def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) if @needs_history[reactor_class] attrs = claim.partition_value.transform_keys(&:to_sym) conditions = reactor_class.context_for(attrs) - t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC) kwargs[:history] = store.read(conditions) - t3 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - Console.info "AAA #{reactor_class.name} read_history=#{((t3-t2)*1000).round(1)}ms" end - t4 = Process.clock_gettime(Process::CLOCK_MONOTONIC) action_pairs = reactor_class.handle_claim(claim, **kwargs) - t5 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - Console.info "AAA #{reactor_class.name} handle_claim=#{((t5-t4)*1000).round(1)}ms" if action_pairs == Actions::RETRY store.release(reactor_class.group_id, offset_id: claim.offset_id) @@ -162,7 +153,6 @@ def resolve_reactor_class(reactor_or_id) def execute_actions(action_pairs, claim, group_id) after_sync_actions = [] - t_tx0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) store.db.transaction do last_position = nil Array(action_pairs).each do |(actions, source_message)| @@ -183,8 +173,6 @@ def execute_actions(action_pairs, claim, group_id) end end - t_tx1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - Console.info "AAA transaction=#{((t_tx1-t_tx0)*1000).round(1)}ms" after_sync_actions.each(&:call) end end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 6ecce2a9..17d51d1f 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -213,7 +213,6 @@ def append(messages, guard: nil) ensure_offsets_for_registered_groups(messages) end - Console.info "AAA append #{messages.map(&:type).uniq}", messages: messages.size notifier.notify_new_messages(messages.map(&:type).uniq) last_position From f3af89a98bd70e2f9faabaa9c43f17e5f5837fc7 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 18 Mar 2026 17:15:36 +0000 Subject: [PATCH 094/115] Rename from_position: to after_position: in CCC::Store#read call stack The parameter uses exclusive (>) semantics, while read_all's from_position: uses inclusive (>=). Rename throughout the call stack to eliminate ambiguity: read, read_partition, query_messages, max_position_for, condition_position_subqueries, and check_conflicts. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 52 +++++++++++++++++----------------- spec/sourced/ccc/store_spec.rb | 14 ++++----- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index 17d51d1f..a0ee8c28 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -309,18 +309,18 @@ def read_all(from_position: nil, limit: 50, order: :asc) # (message_type AND key_name AND key_value). Multiple conditions are OR'd. # # @param conditions [QueryCondition, Array] query conditions - # @param from_position [Integer, nil] only return messages after this position + # @param after_position [Integer, nil] only return messages after this position (exclusive) # @param limit [Integer, nil] max number of messages to return # @return [ReadResult] messages and a guard - def read(conditions, from_position: nil, limit: nil) + def read(conditions, after_position: nil, limit: nil) conditions = Array(conditions) if conditions.empty? - guard = ConsistencyGuard.new(conditions:, last_position: from_position || latest_position) + guard = ConsistencyGuard.new(conditions:, last_position: after_position || latest_position) return ReadResult.new(messages: [], guard:) end - messages = query_messages(conditions, from_position:, limit:) - last_position = messages.any? ? messages.last.position : (from_position || latest_position) + messages = query_messages(conditions, after_position:, limit:) + last_position = messages.any? ? messages.last.position : (after_position || latest_position) guard = ConsistencyGuard.new(conditions:, last_position:) ReadResult.new(messages:, guard:) end @@ -351,7 +351,7 @@ def read(conditions, from_position: nil, limit: nil) # result = store.read_partition( # { device_id: 'dev-1' }, # handled_types: ['device.registered'], - # from_position: 42 + # after_position: 42 # ) # # Only returns messages with position > 42 # @@ -366,9 +366,9 @@ def read(conditions, from_position: nil, limit: nil) # # @param partition_attrs [Hash{Symbol|String => String}] partition attribute values # @param handled_types [Array] message type strings to include - # @param from_position [Integer] fetch messages after this position (default 0) + # @param after_position [Integer] fetch messages after this position (exclusive, default 0) # @return [ReadResult] messages and a guard for optimistic concurrency - def read_partition(partition_attrs, handled_types:, from_position: 0) + def read_partition(partition_attrs, handled_types:, after_position: 0) # Resolve key_pair_ids for each partition attribute key_pair_ids = partition_attrs.filter_map do |name, value| db[@key_pairs_table].where(name: name.to_s, value: value.to_s).get(:id) @@ -376,11 +376,11 @@ def read_partition(partition_attrs, handled_types:, from_position: 0) # If any key pair doesn't exist in the store, no messages can match if key_pair_ids.size < partition_attrs.size - guard = ConsistencyGuard.new(conditions: [], last_position: from_position) + guard = ConsistencyGuard.new(conditions: [], last_position: after_position) return ReadResult.new(messages: [], guard:) end - messages = fetch_partition_messages(key_pair_ids, from_position, handled_types) + messages = fetch_partition_messages(key_pair_ids, after_position, handled_types) # Build guard conditions from handled_types, scoped to partition attrs. # These use OR semantics so the guard detects any concurrent write @@ -394,7 +394,7 @@ def read_partition(partition_attrs, handled_types:, from_position: 0) # The guard's last_position must cover the full OR-context, not just # the AND-filtered messages. Otherwise a message that passes the OR # conditions but was excluded by AND filtering would look like a conflict. - last_pos = max_position_for(guard_conditions, from_position: from_position) + last_pos = max_position_for(guard_conditions, after_position: after_position) guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) ReadResult.new(messages: messages, guard: guard) @@ -407,7 +407,7 @@ def read_partition(partition_attrs, handled_types:, from_position: 0) # @param position [Integer] check for messages after this position # @return [ReadResult] def messages_since(conditions, position) - read(conditions, from_position: position) + read(conditions, after_position: position) end # Register a consumer group. Idempotent. @@ -1173,11 +1173,11 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: # Attributes within each condition are AND'd; conditions are OR'd. # # @param conditions [Array] - # @param from_position [Integer, nil] + # @param after_position [Integer, nil] only include messages after this position (exclusive) # @param limit [Integer, nil] # @return [Array] - def query_messages(conditions, from_position: nil, limit: nil) - subqueries = condition_position_subqueries(conditions, from_position: from_position) + def query_messages(conditions, after_position: nil, limit: nil) + subqueries = condition_position_subqueries(conditions, after_position: after_position) return [] if subqueries.empty? union = subqueries.join(" UNION ") @@ -1201,25 +1201,25 @@ def query_messages(conditions, from_position: nil, limit: nil) def check_conflicts(conditions, after_position) return [] if conditions.empty? - query_messages(conditions, from_position: after_position) + query_messages(conditions, after_position:) end # Max position among messages matching the given conditions. # Attributes within each condition are AND'd; conditions are OR'd. - # Returns from_position (or latest_position) if no matches. + # Returns after_position (or latest_position) if no matches. # # @param conditions [Array] - # @param from_position [Integer, nil] + # @param after_position [Integer, nil] only consider messages after this position (exclusive) # @return [Integer] - def max_position_for(conditions, from_position: nil) - return from_position || latest_position if conditions.empty? + def max_position_for(conditions, after_position: nil) + return after_position || latest_position if conditions.empty? - subqueries = condition_position_subqueries(conditions, from_position: from_position) - return from_position || latest_position if subqueries.empty? + subqueries = condition_position_subqueries(conditions, after_position: after_position) + return after_position || latest_position if subqueries.empty? union = subqueries.join(" UNION ") row = db.fetch("SELECT MAX(position) AS max_pos FROM (#{union})").first - row[:max_pos] || from_position || latest_position + row[:max_pos] || after_position || latest_position end # Build per-condition position subqueries with AND-within/OR-across semantics. @@ -1227,9 +1227,9 @@ def max_position_for(conditions, from_position: nil) # Each subquery selects positions where the message matches ALL attrs in the condition. # # @param conditions [Array] - # @param from_position [Integer, nil] only include positions after this + # @param after_position [Integer, nil] only include positions after this (exclusive) # @return [Array] SQL subquery strings (empty if no conditions can match) - def condition_position_subqueries(conditions, from_position: nil) + def condition_position_subqueries(conditions, after_position: nil) all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq return [] if all_lookups.empty? @@ -1239,7 +1239,7 @@ def condition_position_subqueries(conditions, from_position: nil) key_pair_index = {} key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } - position_filter = from_position ? "AND m.position > #{db.literal(from_position)}" : "" + position_filter = after_position ? "AND m.position > #{db.literal(after_position)}" : "" conditions.filter_map do |c| kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index f99d07b6..ae654df9 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -489,17 +489,17 @@ module CCCStoreTestMessages ) end - it 'filters with from_position' do + it 'filters with after_position' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) - # dev-1 registered is at position 1, so from_position: 1 should return nothing - results, _guard = store.read([cond], from_position: 1) + # dev-1 registered is at position 1, so after_position: 1 should return nothing + results, _guard = store.read([cond], after_position: 1) expect(results).to be_empty - # from_position: 0 should return it - results, _guard = store.read([cond], from_position: 0) + # after_position: 0 should return it + results, _guard = store.read([cond], after_position: 0) expect(results.size).to eq(1) end @@ -575,12 +575,12 @@ module CCCStoreTestMessages expect(guard.last_position).to eq(store.latest_position) end - it 'guard last_position falls back to from_position when no results and from_position given' do + it 'guard last_position falls back to after_position when no results and after_position given' do cond = Sourced::CCC::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'nonexistent' } ) - _results, guard = store.read([cond], from_position: 2) + _results, guard = store.read([cond], after_position: 2) expect(guard.last_position).to eq(2) end end From af56568b515a382a144a04d3d8e587a505ee5159 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 18 Mar 2026 18:21:37 +0000 Subject: [PATCH 095/115] Add CCC::Message::Registry#all for recursive enumeration Returns an Enumerator that yields all registered message classes, walking this registry's own lookup then recursing into subclass registries (Command, Event, etc.). Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/message.rb | 11 +++++++++++ spec/sourced/ccc/message_spec.rb | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 6592de16..07818e3e 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -76,6 +76,17 @@ def [](key) nil end + # All registered message classes across this registry and subclass registries. + # + # @return [Enumerator] if no block given + # @yield [Class] each registered message class + def all(&block) + return enum_for(:all) unless block + + lookup.each_value(&block) + subclasses.each { |c| c.registry.all(&block) } + end + private attr_reader :lookup, :message_class diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index e146d892..4b1bf562 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -311,6 +311,38 @@ module CCCTestMessages expect(keys).to include('device.registered', 'asset.registered', 'system.updated') end end + + describe '#all' do + let!(:test_cmd) { Sourced::CCC::Command.define('test.reg_all_cmd') { attribute :name, String } } + let!(:test_evt) { Sourced::CCC::Event.define('test.reg_all_evt') { attribute :name, String } } + + it 'returns an Enumerator when no block given' do + expect(Sourced::CCC::Message.registry.all).to be_a(Enumerator) + end + + it 'includes classes from subclass registries' do + all = Sourced::CCC::Message.registry.all.to_a + expect(all).to include(test_cmd) + expect(all).to include(test_evt) + end + + it 'includes classes registered directly on Message' do + all = Sourced::CCC::Message.registry.all.to_a + expect(all).to include(CCCTestMessages::DeviceRegistered) + end + + it 'yields each class when block given' do + yielded = [] + Sourced::CCC::Message.registry.all { |c| yielded << c } + expect(yielded).to include(test_cmd, test_evt) + end + + it 'scoped to a subclass registry only includes that branch' do + cmd_all = Sourced::CCC::Command.registry.all.to_a + expect(cmd_all).to include(test_cmd) + expect(cmd_all).not_to include(test_evt) + end + end end describe 'Payload' do From c184ac5d3643023a564c1ecf187acb7a306773e4 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Tue, 14 Apr 2026 17:37:23 +0100 Subject: [PATCH 096/115] Add conditions filter to CCC::Store#read_all Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/ccc/store.rb | 33 +++++++++------ spec/sourced/ccc/store_spec.rb | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 13 deletions(-) diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index a0ee8c28..a1203c14 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -286,23 +286,28 @@ def update_schedule! # @example Next page (using the last position from the previous page) # messages = store.read_all(from_position: 20, limit: 20) # + # @example Filtered by conditions (OR semantics, same as #read) + # messages = store.read_all(conditions: [cond1, cond2], limit: 20) + # # @param from_position [Integer] return messages from this position, inclusive (default 0) + # @param conditions [Array, nil] optional conditions to filter by # @param limit [Integer] max number of messages to return (default 50) # @return [ReadAllResult] messages and last global position - def read_all(from_position: nil, limit: 50, order: :asc) + def read_all(from_position: nil, conditions: [], limit: 50, order: :asc) desc = order == :desc - ds = db[@messages_table] + conditions = Array(conditions).compact + after_position = from_position ? from_position - 1 : nil - if from_position - ds = desc ? ds.where { position <= from_position } : ds.where { position >= from_position } + if conditions.any? + messages = query_messages(conditions, after_position:, limit:, order:) + else + ds = db[@messages_table] + ds = desc ? ds.where { position <= from_position } : ds.where { position >= from_position } if from_position + messages = ds.order(desc ? Sequel.desc(:position) : :position).limit(limit).map { |row| deserialize(row) } end - messages = ds.order(desc ? Sequel.desc(:position) : :position) - .limit(limit) - .map { |row| deserialize(row) } - - fetcher = ->(pos) { read_all(from_position: desc ? pos - 1 : pos + 1, limit: limit, order: order) } - ReadAllResult.new(messages: messages, last_position: latest_position, fetcher: fetcher) + fetcher = ->(pos) { read_all(from_position: desc ? pos - 1 : pos + 1, conditions:, limit:, order:) } + ReadAllResult.new(messages:, last_position: latest_position, fetcher:) end # Query messages by conditions. Each condition matches on @@ -1168,25 +1173,27 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: db.fetch(sql).map { |row| deserialize(row) } end - # Core query logic shared by {#read} and {#check_conflicts}. + # Core query logic shared by {#read}, {#read_all}, and {#check_conflicts}. # Resolves key_pair IDs from conditions, then queries messages. # Attributes within each condition are AND'd; conditions are OR'd. # # @param conditions [Array] # @param after_position [Integer, nil] only include messages after this position (exclusive) # @param limit [Integer, nil] + # @param order [:asc, :desc] position order (default :asc) # @return [Array] - def query_messages(conditions, after_position: nil, limit: nil) + def query_messages(conditions, after_position: nil, limit: nil, order: :asc) subqueries = condition_position_subqueries(conditions, after_position: after_position) return [] if subqueries.empty? union = subqueries.join(" UNION ") + direction = order == :desc ? "DESC" : "ASC" sql = <<~SQL SELECT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at FROM #{@messages_table} m WHERE m.position IN (#{union}) - ORDER BY m.position + ORDER BY m.position #{direction} SQL sql += " LIMIT #{db.literal(limit)}" if limit diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index ae654df9..85dbeae1 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -438,6 +438,80 @@ module CCCStoreTestMessages expect(first_three).to eq([1, 2, 3]) end end + + context 'with conditions' do + before do + store.append([ + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }), + CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), + CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'Sensor B' }) + ]) + end + + it 'filters messages matching conditions' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'dev-1' } + ) + result = store.read_all(conditions: [cond]) + expect(result.messages.size).to eq(1) + expect(result.messages.first.payload.device_id).to eq('dev-1') + end + + it 'ORs multiple conditions' do + cond1 = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'dev-1' } + ) + cond2 = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.asset.registered', + attrs: { asset_id: 'asset-1' } + ) + result = store.read_all(conditions: [cond1, cond2]) + types = result.messages.map(&:type) + expect(types).to eq(['store_test.device.registered', 'store_test.asset.registered']) + end + + it 'combines conditions with from_position' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'dev-2' } + ) + # dev-2 registered is at position 4 + result = store.read_all(conditions: [cond], from_position: 4) + expect(result.messages.size).to eq(1) + expect(result.messages.first.payload.device_id).to eq('dev-2') + + # from_position past it returns nothing + result = store.read_all(conditions: [cond], from_position: 5) + expect(result.messages).to be_empty + end + + it 'returns empty messages when conditions match nothing' do + cond = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'nonexistent' } + ) + result = store.read_all(conditions: [cond]) + expect(result.messages).to be_empty + end + + it 'paginates with to_enum' do + cond1 = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'dev-1' } + ) + cond2 = Sourced::CCC::QueryCondition.new( + message_type: 'store_test.device.registered', + attrs: { device_id: 'dev-2' } + ) + # DeviceRegistered messages at positions 1 and 4 + result = store.read_all(conditions: [cond1, cond2], limit: 1) + positions = result.to_enum.map(&:position) + expect(positions).to eq([1, 4]) + end + end end describe '#read' do From c2a54a449c09a606972c9af774266f8851fa4c4b Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Tue, 14 Apr 2026 17:54:33 +0100 Subject: [PATCH 097/115] Encapsulate Dispatcher setup and spawning into class level CCC::Dispatcher.spawn_into(task) --- lib/sourced/ccc/dispatcher.rb | 16 ++++++++++++++++ lib/sourced/ccc/falcon/service.rb | 14 +------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb index d268b4fc..bf09a1fa 100644 --- a/lib/sourced/ccc/dispatcher.rb +++ b/lib/sourced/ccc/dispatcher.rb @@ -92,6 +92,20 @@ def build_group_id_lookup(reactors) # @return [Array] worker instances managed by this dispatcher attr_reader :workers + def self.spawn_into(task) + config = CCC.config + dispatcher = CCC::Dispatcher.new( + router: CCC.router, + worker_count: config.worker_count, + batch_size: config.batch_size, + max_drain_rounds: config.max_drain_rounds, + catchup_interval: config.catchup_interval, + housekeeping_interval: config.housekeeping_interval, + claim_ttl_seconds: config.claim_ttl_seconds, + logger: config.logger + ).spawn_into(task) + end + # @param router [CCC::Router] the CCC router providing reactors and store # @param worker_count [Integer] number of worker fibers to spawn (default 2) # @param batch_size [Integer] max messages per claim (default 50) @@ -185,6 +199,8 @@ def spawn_into(task) @workers.each do |w| task.send(s) { w.run } end + + self end # Stop all components and close the work queue. diff --git a/lib/sourced/ccc/falcon/service.rb b/lib/sourced/ccc/falcon/service.rb index d2116082..e749fee4 100644 --- a/lib/sourced/ccc/falcon/service.rb +++ b/lib/sourced/ccc/falcon/service.rb @@ -19,21 +19,9 @@ def run(instance, evaluator) server = evaluator.make_server(@bound_endpoint) - config = CCC.config - @dispatcher = CCC::Dispatcher.new( - router: CCC.router, - worker_count: config.worker_count, - batch_size: config.batch_size, - max_drain_rounds: config.max_drain_rounds, - catchup_interval: config.catchup_interval, - housekeeping_interval: config.housekeeping_interval, - claim_ttl_seconds: config.claim_ttl_seconds, - logger: config.logger - ) - Async do |task| server.run - @dispatcher.spawn_into(task) + CCC::Dispatcher.spawn_into(task) task.children.each(&:wait) end From 003548dd70695d5aaf096d1e84e24b208487c311 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Tue, 14 Apr 2026 17:59:40 +0100 Subject: [PATCH 098/115] Remove redundant names --- lib/sourced/ccc/dispatcher.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb index bf09a1fa..a4138c85 100644 --- a/lib/sourced/ccc/dispatcher.rb +++ b/lib/sourced/ccc/dispatcher.rb @@ -139,11 +139,11 @@ def initialize( @workers = worker_count.times.map do |i| Worker.new( work_queue: @work_queue, - router: router, + router:, name: "worker-#{i}", - batch_size: batch_size, - max_drain_rounds: max_drain_rounds, - logger: logger + batch_size:, + max_drain_rounds:, + logger: ) end @@ -153,15 +153,15 @@ def initialize( @catchup_poller = CatchUpPoller.new( work_queue: @work_queue, - reactors: reactors, + reactors:, interval: catchup_interval, - logger: logger + logger: ) @scheduled_message_poller = ScheduledMessagePoller.new( store: router.store, interval: catchup_interval, - logger: logger + logger: ) @stale_claim_reaper = StaleClaimReaper.new( @@ -169,7 +169,7 @@ def initialize( interval: housekeeping_interval, ttl_seconds: claim_ttl_seconds, worker_ids_provider: -> { @workers.map(&:name) }, - logger: logger + logger: ) end From 23a34335e834b1dea6925d471573b92c2bd5099d Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 10:48:15 +0100 Subject: [PATCH 099/115] PositionedMessage#to_message and Message.=== --- lib/sourced/ccc/message.rb | 16 ++++++++++++ lib/sourced/ccc/store.rb | 5 ++++ spec/sourced/ccc/message_spec.rb | 45 ++++++++++++++++++++++++++++++++ spec/sourced/ccc/store_spec.rb | 25 ++++++++++++++++++ 4 files changed, 91 insertions(+) diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb index 07818e3e..e8bc2c16 100644 --- a/lib/sourced/ccc/message.rb +++ b/lib/sourced/ccc/message.rb @@ -154,6 +154,22 @@ def initialize(attrs = {}) super(attrs) end + # Identity implementation of the +to_message+ contract — see + # {.===} and {CCC::PositionedMessage#to_message}. + def to_message = self + + # Make +case/when+ transparent to {CCC::PositionedMessage} (or any + # wrapper implementing +#to_message+). Ruby's default +Module#===+ + # is implemented in C and ignores +is_a?+ overrides, so wrapped + # messages would otherwise fall through the +else+ branch. + def self.===(other) + return true if super + return false unless other.respond_to?(:to_message) + + unwrapped = other.to_message + !unwrapped.equal?(other) && super(unwrapped) + end + def with_metadata(meta = {}) return self if meta.empty? diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb index a1203c14..eca30e4b 100644 --- a/lib/sourced/ccc/store.rb +++ b/lib/sourced/ccc/store.rb @@ -22,6 +22,11 @@ def class = __getobj__.class def is_a?(klass) = __getobj__.is_a?(klass) || super def kind_of?(klass) = is_a?(klass) def instance_of?(klass) = __getobj__.instance_of?(klass) + + # Unwrap to the underlying {CCC::Message}. Part of the +to_message+ + # contract honoured by {CCC::Message.===} so that +case/when+ works + # transparently across wrapped and unwrapped messages. + def to_message = __getobj__ end # Returned by {Store#claim_next} with everything needed to process and ack a partition. diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb index 4b1bf562..9930b49f 100644 --- a/spec/sourced/ccc/message_spec.rb +++ b/spec/sourced/ccc/message_spec.rb @@ -384,6 +384,51 @@ module CCCTestMessages end end + describe '#to_message' do + it 'returns self — identity implementation of the to_message contract' do + msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + expect(msg.to_message).to equal(msg) + end + end + + describe '.===' do + let(:msg) { CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) } + + it 'matches unwrapped instances like Module#===' do + expect(CCCTestMessages::DeviceRegistered === msg).to be true + expect(CCCTestMessages::AssetRegistered === msg).to be false + end + + it 'matches messages wrapped in a to_message-aware delegator' do + wrapper = Class.new(SimpleDelegator) do + def to_message = __getobj__ + end.new(msg) + + expect(CCCTestMessages::DeviceRegistered === wrapper).to be true + expect(CCCTestMessages::AssetRegistered === wrapper).to be false + end + + it 'makes case/when transparent across wrapped and unwrapped messages' do + classify = ->(m) do + case m + when CCCTestMessages::DeviceRegistered then :device + when CCCTestMessages::AssetRegistered then :asset + else :unknown + end + end + + wrapper = Sourced::CCC::PositionedMessage.new(msg, 1) + expect(classify.call(msg)).to eq(:device) + expect(classify.call(wrapper)).to eq(:device) + end + + it 'returns false for arbitrary non-message objects without looping' do + expect(CCCTestMessages::DeviceRegistered === 'string').to be false + expect(CCCTestMessages::DeviceRegistered === 42).to be false + expect(CCCTestMessages::DeviceRegistered === Object.new).to be false + end + end + describe Sourced::CCC::ConsistencyGuard do it 'is a Data struct with conditions and last_position' do conditions = [Sourced::CCC::QueryCondition.new(message_type: 'device.registered', attrs: { device_id: 'dev-1' })] diff --git a/spec/sourced/ccc/store_spec.rb b/spec/sourced/ccc/store_spec.rb index 85dbeae1..a92848c1 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/sourced/ccc/store_spec.rb @@ -369,6 +369,31 @@ module CCCStoreTestMessages expect(result.messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) end + describe 'PositionedMessage#to_message' do + it 'unwraps to the underlying CCC::Message instance' do + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + wrapped = store.read_all.messages.first + inner = wrapped.to_message + + expect(wrapped).to be_a(Sourced::CCC::PositionedMessage) + expect(inner).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(inner).not_to be_a(Sourced::CCC::PositionedMessage) + expect(inner.payload.device_id).to eq('dev-1') + end + + it 'allows case/when against wrapped messages via CCC::Message.===' do + store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + + wrapped = store.read_all.messages.first + matched = case wrapped + when CCCStoreTestMessages::DeviceRegistered then :device + else :other + end + expect(matched).to eq(:device) + end + end + context 'with order: :desc' do before do 5.times do |i| From 109ee36ac5059ec37256c4185440b6c773e4c005 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 11:40:34 +0100 Subject: [PATCH 100/115] Make all #handle_batch signatures the same so that GWT helpers work for any reactor --- lib/sourced/ccc/decider.rb | 2 +- lib/sourced/ccc/projector.rb | 2 +- lib/sourced/ccc/testing/rspec.rb | 159 ++++++++++++------------- spec/sourced/ccc/testing/rspec_spec.rb | 41 +++---- 4 files changed, 94 insertions(+), 110 deletions(-) diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb index 87b9468c..dc72adca 100644 --- a/lib/sourced/ccc/decider.rb +++ b/lib/sourced/ccc/decider.rb @@ -33,7 +33,7 @@ def command(message_class, &block) define_method(Sourced.message_method_name('ccc_decide', message_class.to_s), &block) end - def handle_batch(partition_values, new_messages, history:) + def handle_batch(partition_values, new_messages, history:, replaying: false) instance = new(partition_values) instance.evolve(history.messages) diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb index ec7609e7..a0bab8d5 100644 --- a/lib/sourced/ccc/projector.rb +++ b/lib/sourced/ccc/projector.rb @@ -49,7 +49,7 @@ def initialize(partition_values = {}) # Projector variant that evolves only the claimed messages on top of stored state. class StateStored < self class << self - def handle_batch(partition_values, new_messages, replaying: false) + def handle_batch(partition_values, new_messages, history: nil, replaying: false) instance = new(partition_values) instance.evolve(new_messages) build_action_pairs(instance, new_messages, replaying: replaying) diff --git a/lib/sourced/ccc/testing/rspec.rb b/lib/sourced/ccc/testing/rspec.rb index 5b354076..27f350d5 100644 --- a/lib/sourced/ccc/testing/rspec.rb +++ b/lib/sourced/ccc/testing/rspec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'sourced/ccc' +require 'sourced/ccc/store' module Sourced module CCC @@ -10,7 +11,11 @@ module RSpec # Entry point for CCC reactor GWT tests. # - # @param reactor_class [Class] a CCC::Decider or CCC::Projector subclass + # Works with any reactor that responds to the standard + # handle_batch(partition_values, new_messages, history:, replaying:) + # contract (Deciders, Projectors, DurableWorkflows). + # + # @param reactor_class [Class] a CCC reactor class # @param partition_attrs [Hash] partition key-value pairs (e.g. device_id: 'd1') # @return [GWT] # @@ -20,14 +25,20 @@ module RSpec # .when(BindDevice, device_id: 'd1', asset_id: 'a1') # .then(DeviceBound, device_id: 'd1', asset_id: 'a1') # - # @example Projector + # @example Projector — assert state via block # with_reactor(MyProjector, list_id: 'L1') # .given(ItemAdded, list_id: 'L1', name: 'Apple') - # .then { |state| expect(state[:items]).to eq(['Apple']) } + # .then { |r| expect(r.state[:items]).to eq(['Apple']) } def with_reactor(reactor_class, **partition_attrs) GWT.new(reactor_class, **partition_attrs) end + # Uniform result yielded to +.then+ / +.then!+ block callbacks. + # Gives access to both the reactor's produced action pairs / messages + # (what deciders typically assert on) and the evolved state (what + # projectors typically assert on). + RunResult = Data.define(:pairs, :messages, :state) + class MessageMatcher def initialize(expected_messages) @expected_messages = Array(expected_messages) @@ -69,75 +80,62 @@ def failure_message class GWT def initialize(reactor_class, **partition_attrs) @reactor_class = reactor_class - @partition_attrs = partition_attrs @partition_values = partition_attrs @given_messages = [] - @when_message = nil + @when_messages = [] @asserted = false end - # Accumulate history/context messages. - # For Deciders: these become the history (ReadResult). - # For Projectors: these are evolved onto the instance. + # Accumulate history / context messages. These become +history.messages+ + # passed to the reactor's +handle_batch+. # - # @param klass [Class] message class - # @param payload [Hash] payload attributes + # @param klass_or_instance [Class, CCC::Message] + # @param payload [Hash] # @return [self] def given(klass_or_instance = nil, **payload) raise 'test case already asserted' if @asserted - msg = build_message(klass_or_instance, **payload) - @given_messages << msg + @given_messages << build_message(klass_or_instance, **payload) self end alias_method :and, :given - # Set the command to decide on (Deciders only). + # The batch of new messages to process via +handle_batch+. Can be + # called multiple times to supply several messages. # - # @param klass [Class] command class - # @param payload [Hash] payload attributes + # @param klass_or_instance [Class, CCC::Message] + # @param payload [Hash] # @return [self] def when(klass_or_instance = nil, **payload) raise 'test case already asserted' if @asserted - raise ArgumentError, '.when is not supported for Projectors' if projector? - @when_message = build_message(klass_or_instance, **payload) + @when_messages << build_message(klass_or_instance, **payload) self end # Assert expected outcomes. # - # For Deciders: - # - Pass message class + payload pairs to assert produced messages - # - Pass [] or NONE to assert no messages - # - Pass an Exception class (+ optional message) to assert invariant violation - # - Pass a block to receive action pairs for custom assertions - # - # For Projectors: - # - Requires a block that receives the evolved state + # - Pass message class + payload pairs (or instances) to assert + # produced messages. + # - Pass +[]+ or +NONE+ to assert no messages. + # - Pass an Exception class (+ optional message) to assert the + # reactor raised. + # - Pass a block to receive a {RunResult} for custom assertions. # # @return [self] def then(*expected, **payload, &block) run_then(false, *expected, **payload, &block) end - # Like #then, but runs sync actions before yielding state (Projectors) - # or before extracting messages (Deciders). + # Like #then, but runs Sync / AfterSync actions before computing + # the result yielded to the block (or before extracting messages). def then!(*expected, **payload, &block) run_then(true, *expected, **payload, &block) end private - def decider? - @reactor_class < CCC::Decider - end - - def projector? - @reactor_class < CCC::Projector - end - def build_message(klass_or_instance, **payload) if klass_or_instance.is_a?(CCC::Message) klass_or_instance @@ -154,43 +152,28 @@ def run_then(sync, *expected, **payload, &block) expected = [expected[0].new(payload: payload)] end - if decider? - run_decider_then(sync, *expected, &block) - elsif projector? - run_projector_then(sync, *expected, &block) - else - raise ArgumentError, "unsupported reactor type: #{@reactor_class}" - end - - self - end - - def run_decider_then(sync, *expected, &block) # Exception expectation if expected.size >= 1 && exception_expectation?(expected[0]) expect_exception(expected[0], expected[1]) - return + return self end - pairs = run_decider + pairs = run_handle_batch if sync pairs.each do |actions, _| Array(actions).select { |a| a.is_a?(CCC::Actions::Sync) || a.is_a?(CCC::Actions::AfterSync) - }.each(&:call) # collect_actions produces both types; filter from pairs + }.each(&:call) end end if block_given? - block.call(pairs) - return + block.call(RunResult.new(pairs: pairs, messages: extract_messages(pairs), state: compute_state(sync: sync))) + return self end - # Extract messages from action pairs actual_messages = extract_messages(pairs) - - # Build expected messages expected_msgs = build_expected(*expected) if expected_msgs.empty? @@ -199,41 +182,58 @@ def run_decider_then(sync, *expected, &block) "Expected no messages, but got #{actual_messages.size}: #{actual_messages.inspect}" ) end - return + return self end matcher = MessageMatcher.new(expected_msgs) unless matcher.matches?(actual_messages) ::RSpec::Expectations.fail_with(matcher.failure_message) end + + self end - def run_projector_then(sync, *_expected, &block) - raise ArgumentError, '.then for Projectors requires a block' unless block_given? + def run_handle_batch + guard = CCC::ConsistencyGuard.new(conditions: [], last_position: 0) + history = CCC::ReadResult.new(messages: @given_messages, guard: guard) + @reactor_class.handle_batch( + @partition_values, + @when_messages, + history: history, + replaying: false + ) + end + # Build an instance and evolve it with all known messages so the + # caller can assert on state regardless of reactor type. For reactors + # whose +handle_batch+ evolves its own instance (Decider, Projector, + # DurableWorkflow), this is an independent, predictable computation. + # When +sync+ is true, also runs the reactor's Sync / AfterSync + # blocks against this instance so their state mutations are visible. + def compute_state(sync: false) instance = @reactor_class.new(@partition_values) + return nil unless instance.respond_to?(:evolve) - if @reactor_class < CCC::Projector::EventSourced - # EventSourced: evolve from full history - instance.evolve(@given_messages) - else - # StateStored: evolve from given messages - instance.evolve(@given_messages) - end - - if sync - instance.collect_actions( - state: instance.state, messages: @given_messages, replaying: false - ).each(&:call) - end - - block.call(instance.state) + messages = @given_messages + @when_messages + instance.evolve(messages) + run_sync_on(instance, messages) if sync + instance.state end - def run_decider - guard = ConsistencyGuard.new(conditions: [], last_position: 0) - history = ReadResult.new(messages: @given_messages, guard: guard) - @reactor_class.handle_batch(@partition_values, [@when_message], history: history) + # Invoke Sync / AfterSync blocks against +instance+. Per-block + # kwarg signatures vary by reactor type (deciders expect +events:+, + # projectors expect +replaying:+); we inspect each block's + # parameters and pass only what it declares. + def run_sync_on(instance, messages) + all_args = { state: instance.state, messages: messages, events: [], replaying: false } + klass = instance.class + blocks = [] + blocks.concat(klass.sync_blocks) if klass.respond_to?(:sync_blocks) + blocks.concat(klass.after_sync_blocks) if klass.respond_to?(:after_sync_blocks) + blocks.each do |block| + wanted = block.parameters.select { |type, _| type == :keyreq || type == :key }.map(&:last) + instance.instance_exec(**all_args.slice(*wanted), &block) + end end def extract_messages(pairs) @@ -263,11 +263,8 @@ def exception_expectation?(arg) end def expect_exception(exception_class, message = nil) - guard = ConsistencyGuard.new(conditions: [], last_position: 0) - history = ReadResult.new(messages: @given_messages, guard: guard) - begin - @reactor_class.handle_batch(@partition_values, [@when_message], history: history) + run_handle_batch rescue exception_class => e if message && e.message != message ::RSpec::Expectations.fail_with( diff --git a/spec/sourced/ccc/testing/rspec_spec.rb b/spec/sourced/ccc/testing/rspec_spec.rb index dd43d93e..072ba33b 100644 --- a/spec/sourced/ccc/testing/rspec_spec.rb +++ b/spec/sourced/ccc/testing/rspec_spec.rb @@ -179,9 +179,9 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced with_reactor(GWTTestDecider, device_id: 'd1') .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then { |pairs| - expect(pairs).to be_a(Array) - actions, _source = pairs.first + .then { |r| + expect(r.pairs).to be_a(Array) + actions, _source = r.pairs.first append_actions = Array(actions).select { |a| a.respond_to?(:messages) } expect(append_actions).not_to be_empty } @@ -198,8 +198,8 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced with_reactor(GWTTestDecider, device_id: 'd1') .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then! { |pairs| - expect(pairs).to be_a(Array) + .then! { |r| + expect(r.pairs).to be_a(Array) } end @@ -228,26 +228,26 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced it 'given events → then block asserts evolved state' do with_reactor(GWTTestStateStoredProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then { |state| expect(state[:items]).to eq(['Apple']) } + .then { |r| expect(r.state[:items]).to eq(['Apple']) } end it 'given multiple events → then block sees cumulative state' do with_reactor(GWTTestStateStoredProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') - .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } + .then { |r| expect(r.state[:items]).to eq(['Apple', 'Banana']) } end it 'then! runs sync actions before yielding state' do with_reactor(GWTTestStateStoredProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |state| expect(state[:synced]).to be true } + .then! { |r| expect(r.state[:synced]).to be true } end it 'then! runs after_sync actions before yielding state' do with_reactor(GWTTestStateStoredProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |state| expect(state[:after_synced]).to be true } + .then! { |r| expect(r.state[:after_synced]).to be true } end it 'given events with archive → state reflects removal' do @@ -255,14 +255,7 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') .and(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') .and(CCCGWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') - .then { |state| expect(state[:items]).to eq(['Banana']) } - end - - it '.when raises ArgumentError' do - expect { - with_reactor(GWTTestStateStoredProjector, list_id: 'L1') - .when(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - }.to raise_error(ArgumentError, '.when is not supported for Projectors') + .then { |r| expect(r.state[:items]).to eq(['Banana']) } end end @@ -271,33 +264,27 @@ class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') - .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } + .then { |r| expect(r.state[:items]).to eq(['Apple', 'Banana']) } end it 'given events with archive → state reflects removal' do with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') .and(CCCGWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') - .then { |state| expect(state[:items]).to eq([]) } + .then { |r| expect(r.state[:items]).to eq([]) } end it 'then! runs sync actions before yielding state' do with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |state| expect(state[:synced]).to be true } + .then! { |r| expect(r.state[:synced]).to be true } end it 'then! runs after_sync actions before yielding state' do with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |state| expect(state[:after_synced]).to be true } + .then! { |r| expect(r.state[:after_synced]).to be true } end - it '.when raises ArgumentError' do - expect { - with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') - .when(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - }.to raise_error(ArgumentError, '.when is not supported for Projectors') - end end end From 9985ffad456aedbb169215b7f50e2764c44a4473 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 10:52:11 +0100 Subject: [PATCH 101/115] Re-implement DurableWorkflow for CCC --- lib/sourced/ccc.rb | 1 + lib/sourced/ccc/durable_workflow.rb | 397 ++++++++++++++++++++++ spec/sourced/ccc/durable_workflow_spec.rb | 361 ++++++++++++++++++++ 3 files changed, 759 insertions(+) create mode 100644 lib/sourced/ccc/durable_workflow.rb create mode 100644 spec/sourced/ccc/durable_workflow_spec.rb diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb index f608b600..73d87418 100644 --- a/lib/sourced/ccc.rb +++ b/lib/sourced/ccc.rb @@ -271,3 +271,4 @@ def self.load(reactor_class, store: nil, **values) require 'sourced/ccc/command_context' require 'sourced/ccc/topology' require 'sourced/ccc/supervisor' +require 'sourced/ccc/durable_workflow' diff --git a/lib/sourced/ccc/durable_workflow.rb b/lib/sourced/ccc/durable_workflow.rb new file mode 100644 index 00000000..4d72e4be --- /dev/null +++ b/lib/sourced/ccc/durable_workflow.rb @@ -0,0 +1,397 @@ +# frozen_string_literal: true + +require 'securerandom' + +module Sourced + module CCC + # Stream-less port of {Sourced::DurableWorkflow}. + # + # A workflow instance is identified by a +workflow_id+ string, which doubles + # as the partition key. All lifecycle events (WorkflowStarted, StepStarted, + # StepFailed, StepComplete, ContextUpdated, WaitStarted, WaitEnded, + # WorkflowComplete, WorkflowFailed) carry +workflow_id+ as their first + # payload attribute so {CCC::Message#extracted_keys} indexes them for + # partition queries. + # + # The +durable+ / +wait+ / +context+ / +execute+ DSL mirrors + # {Sourced::DurableWorkflow} 1:1. The step-memoisation mechanism + # (@lookup + catch(:halt)) is unchanged; only the + # persistence and dispatch layer differ. + class DurableWorkflow + extend CCC::Consumer + include CCC::Evolve + + partition_by :workflow_id + + UnknownMessageError = Class.new(StandardError) + + # Stable hash-based key for a given (method, args) pair. + def self.step_key(step_name, args) + [step_name, args].hash.to_s + end + + def self.inherited(child) + super + child.partition_by(:workflow_id) + cname = child.name.to_s.gsub(/::/, '.') + .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') + .gsub(/([a-z\d])([A-Z])/, '\1_\2') + .tr('-', '_') + .downcase + + child.const_set(:WorkflowStarted, CCC::Event.define("#{cname}.workflow.started") do + attribute :workflow_id, String + attribute :args, Sourced::Types::Array.default([].freeze) + end) + child.const_set(:ContextUpdated, CCC::Event.define("#{cname}.context.updated") do + attribute :workflow_id, String + attribute :context, Sourced::Types::Any + end) + child.const_set(:WorkflowComplete, CCC::Event.define("#{cname}.workflow.complete") do + attribute :workflow_id, String + attribute :output, Sourced::Types::Any + end) + child.const_set(:WorkflowFailed, CCC::Event.define("#{cname}.workflow.failed") do + attribute :workflow_id, String + end) + child.const_set(:StepStarted, CCC::Event.define("#{cname}.step.started") do + attribute :workflow_id, String + attribute :key, String + attribute :step_name, Sourced::Types::Lax::Symbol + attribute :args, Sourced::Types::Array.default([].freeze) + end) + child.const_set(:StepFailed, CCC::Event.define("#{cname}.step.failed") do + attribute :workflow_id, String + attribute :key, String + attribute :step_name, Sourced::Types::Lax::Symbol + attribute :error_message, String + attribute :error_class, String + attribute :backtrace, Sourced::Types::Array[String] + end) + child.const_set(:StepComplete, CCC::Event.define("#{cname}.step.complete") do + attribute :workflow_id, String + attribute :key, String + attribute :step_name, Sourced::Types::Lax::Symbol + attribute :output, Sourced::Types::Any + end) + child.const_set(:WaitStarted, CCC::Event.define("#{cname}.wait.started") do + attribute :workflow_id, String + attribute :count, Integer + attribute :at, Sourced::Types::Forms::Time + end) + child.const_set(:WaitEnded, CCC::Event.define("#{cname}.wait.ended") do + attribute :workflow_id, String + end) + + # Register all event classes so: + # - Router claims them on our consumer group (`handled_messages`). + # - `context_for(workflow_id:)` builds OR conditions for the partition + # read (one per event type, via `Message.to_conditions`). + [ + child::WorkflowStarted, child::ContextUpdated, child::WorkflowComplete, + child::WorkflowFailed, child::StepStarted, child::StepFailed, + child::StepComplete, child::WaitStarted, child::WaitEnded + ].each do |klass| + child.handled_messages_for_evolve << klass unless child.handled_messages_for_evolve.include?(klass) + end + end + + # Message types this consumer claims. Same set as evolve types because + # every workflow event both advances state and re-triggers the workflow. + def self.handled_messages + handled_messages_for_evolve + end + + # Define the initial context hash. Block receives no arguments. + def self.context(&block) + define_method :initial_context, &block + end + + # Wrap a method so the runtime memoises its result across workflow + # re-entries. Mirrors {Sourced::DurableWorkflow.durable}. + def self.durable(method_name, retries: nil) + source_method = :"__durable_source_#{method_name}" + alias_method source_method, method_name + define_method method_name do |*args| + key = self.class.step_key(method_name, args) + cached = @lookup[key] + + case cached&.status + when :complete + cached.output + when :started + begin + output = send(source_method, *args) + @new_events << self.class::StepComplete.new( + payload: { workflow_id: id, key:, step_name: method_name, output: } + ) + throw :halt + rescue StandardError => e + @new_events << self.class::StepFailed.new( + payload: { + workflow_id: id, + key:, + step_name: method_name, + error_message: e.inspect, + error_class: e.class.to_s, + backtrace: e.backtrace + } + ) + if retries && cached.attempts == retries + @new_events << self.class::WorkflowFailed.new(payload: { workflow_id: id }) + end + throw :halt + end + when :failed + @new_events << self.class::StepStarted.new( + payload: { workflow_id: id, key:, step_name: method_name, args: } + ) + throw :halt + when nil + @new_events << self.class::StepStarted.new( + payload: { workflow_id: id, key:, step_name: method_name, args: } + ) + throw :halt + end + end + end + + Step = Struct.new(:status, :backtrace, :output, :attempts) do + def self.build + new(:started, [], nil, 0) + end + + def start + self.status = :started + self.attempts += 1 + end + + def fail_with(backtrace) + self.status = :failed + self.backtrace = backtrace + self + end + + def complete_with(output) + self.status = :complete + self.output = output + self + end + end + + # Kick off a new workflow instance. Appends a WorkflowStarted event and + # returns a {Waiter} that can poll for completion. + # + # @param args [Array] positional args passed to the workflow's #execute + # @param store [CCC::Store] defaults to CCC.store + # @return [Waiter] + def self.execute(*args, store: CCC.store) + workflow_id = "workflow-#{SecureRandom.uuid}" + evt = self::WorkflowStarted.new(payload: { workflow_id:, args: }) + store.append([evt]) + Waiter.new(self, workflow_id, store:) + end + + # Router entry point. Processes one claimed message at a time: for each + # message, replay history up to and including that position, then run + # {#__handle} to derive the next action. + def self.handle_claim(claim, history:) + workflow_id = claim.partition_value['workflow_id'] + ordered = history.messages.sort_by { |m| m.respond_to?(:position) ? m.position : 0 } + guard = history.guard + + each_with_partial_ack(claim.messages) do |msg| + msg_pos = msg.respond_to?(:position) ? msg.position : nil + prior = if msg_pos + ordered.take_while { |m| (m.respond_to?(:position) ? m.position : 0) <= msg_pos } + else + ordered + end + instance = new([workflow_id]) + instance.__replay(prior) + actions = instance.__handle(msg, guard: guard) + [actions, msg] + end + end + + # Direct handler used by unit tests and by {Waiter}. Mirrors + # {Sourced::DurableWorkflow.handle}: +history+ should already contain + # +message+ as its last element. + def self.handle(message, history:) + from(history).__handle(message) + end + + # Rebuild a workflow instance by replaying +history+. + def self.from(history) + new.__replay(history) + end + + # Load a workflow instance for +workflow_id+ from the store. + def self.load(workflow_id, store: CCC.store) + _inst, _rr = CCC.load(self, store:, workflow_id: workflow_id) + end + + # Polls the store for terminal workflow events. + class Waiter + attr_reader :workflow_id, :instance + + def initialize(klass, workflow_id, store: CCC.store) + @klass = klass + @workflow_id = workflow_id + @store = store + @instance = klass.new([workflow_id]) + end + + def wait(timeout: nil) + deadline = timeout ? Time.now + timeout : nil + until @instance.status == :complete || @instance.status == :failed + raise 'DurableWorkflow wait timed out' if deadline && Time.now > deadline + + sleep 0.05 + load + end + @instance + end + + def load + handled_types = @klass.handled_messages_for_evolve.map(&:type).uniq + result = @store.read_partition({ workflow_id: @workflow_id }, handled_types:) + @instance = @klass.new([@workflow_id]) + @instance.__replay(result.messages) + @instance + end + end + + attr_reader :id, :context, :args, :output, :status + + # +partition_values+ may be: + # - an Array like +['wf-id']+ (from {.handle_claim}) + # - a Hash like +{ workflow_id: 'wf-id' }+ (from {CCC.load}) + # - a String +'wf-id'+ + # - nil + def initialize(partition_values = nil) + @id = case partition_values + when Array then partition_values.first + when Hash then partition_values[:workflow_id] || partition_values['workflow_id'] + when String then partition_values + else nil + end + @status = :new + @args = [] + @output = nil + @lookup = {} + @new_events = [] + @wait_count = 0 + @waiters = [] + @context = initial_context + end + + def initial_context = nil + + def __replay(history) + Array(history).each { |m| __evolve(m) } + self + end + + # Override CCC::Evolve#evolve so {CCC.load} (which calls +instance.evolve+) + # applies workflow events via our manual dispatcher. + def evolve(messages) + __replay(messages) + end + + def __evolve(event) + case event + when self.class::ContextUpdated + @context = deep_dup(event.payload.context) + when self.class::WorkflowStarted + @id ||= event.payload.workflow_id + @args = event.payload.args + @status = :started + when self.class::WorkflowFailed + @status = :failed + when self.class::StepStarted + (@lookup[event.payload.key] ||= Step.build).start + when self.class::StepFailed + @lookup[event.payload.key].fail_with(event.payload.backtrace) + when self.class::WaitStarted + @waiters[event.payload.count] = true + @waiting = true + when self.class::WaitEnded + @waiting = false + when self.class::StepComplete + @lookup[event.payload.key].complete_with(event.payload.output) + when self.class::WorkflowComplete + @status = :complete + @output = event.payload.output + else + raise UnknownMessageError, "No idea how to handle #{event.inspect}" + end + end + + # Decide the next action given +message+. State is assumed to already + # reflect +message+ (caller replayed it). + def __handle(message, guard: nil) + return Actions::OK if @status == :complete || @status == :failed + + if message.is_a?(self.class::WaitStarted) + evt = self.class::WaitEnded.new(payload: { workflow_id: id }) + return Actions::Schedule.new([evt], at: message.payload.at) + end + + @initial_context = deep_dup(@context) + + completed = false + output = nil + + catch(:halt) do + output = execute(*@args) + completed = true + end + + if @context != @initial_context + @new_events << self.class::ContextUpdated.new( + payload: { workflow_id: id, context: deep_dup(@context) } + ) + end + + if completed + @new_events << self.class::WorkflowComplete.new( + payload: { workflow_id: id, output: } + ) + end + + events = @new_events + @new_events = [] + return Actions::OK if events.empty? + + Actions::Append.new(events, guard: guard) + end + + private + + def wait(seconds) + @wait_count += 1 + + if @waiters[@wait_count] + seconds + else + @new_events << self.class::WaitStarted.new( + payload: { workflow_id: id, count: @wait_count, at: Time.now + seconds } + ) + throw :halt + end + end + + def deep_dup(value) + case value + when Hash + value.each.with_object({}) { |(k, v), h| h[k] = deep_dup(v) } + when Array + value.map { |v| deep_dup(v) } + else + value.dup rescue value + end + end + end + end +end diff --git a/spec/sourced/ccc/durable_workflow_spec.rb b/spec/sourced/ccc/durable_workflow_spec.rb new file mode 100644 index 00000000..00d97a3b --- /dev/null +++ b/spec/sourced/ccc/durable_workflow_spec.rb @@ -0,0 +1,361 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'sourced/ccc' +require 'sourced/ccc/store' +require 'sequel' + +module CCCDurableTests + FilledStringArray = Sourced::Types::Array[String].with(size: 1..) + + class IPResolver + def self.resolve = '11.111.111' + end + + class Geolocator + def self.locate(_ip) = 'London, UK' + end + + class Task < Sourced::CCC::DurableWorkflow + def execute(name) + ip = get_ip + location = geolocate(ip) + "Hello #{name}, your IP is #{ip} and its location is #{location}" + end + + durable def get_ip + IPResolver.resolve + end + + durable def geolocate(ip) + Geolocator.locate(ip) + end + end + + class AnotherTask < Sourced::CCC::DurableWorkflow + end + + class Doubler + def self.double(num) = num * 2 + end + + class MultiArgTask < Sourced::CCC::DurableWorkflow + def execute + double(2) + double(4) + double(2) + end + + durable def double(num) + Doubler.double(num) + end + end + + class Retryable < Sourced::CCC::DurableWorkflow + def execute + compute + end + + def compute + raise 'nope' + end + + durable :compute, retries: 2 + end + + class WithContext < Sourced::CCC::DurableWorkflow + context do + { index: 0, results: [] } + end + + def execute + @numbers = [1, 2, 3, 4, 5] + iterate + context[:results] + end + + durable def iterate + index = context[:index] + @numbers[index..].each do |n| + raise 'oopsie!' if n == 3 && index == 0 + + context[:index] += 1 + context[:results] << n * 2 + end + end + end + + class WithDelay < Sourced::CCC::DurableWorkflow + def execute + name = get_name + wait 10 + notify(name) + end + + durable def get_name + 'Joe' + end + + durable def notify(_name) + true + end + end +end + +RSpec.describe Sourced::CCC::DurableWorkflow do + let(:workflow_id) { 'durable-test-1' } + let(:name) { 'Joe' } + + def make_started(klass, args: [], workflow_id: 'durable-test-1') + klass::WorkflowStarted.new(payload: { workflow_id:, args: }) + end + + context 'with happy path' do + it 'starts and produces new messages until completing workflow' do + started = make_started(CCCDurableTests::Task, args: [name]) + history = [started] + + until history.last.is_a?(CCCDurableTests::Task::WorkflowComplete) + next_action = CCCDurableTests::Task.handle(history.last, history:) + expect(next_action).to be_a Sourced::CCC::Actions::Append + history += next_action.messages + end + + expect(history.map(&:class)).to eq([ + CCCDurableTests::Task::WorkflowStarted, + CCCDurableTests::Task::StepStarted, + CCCDurableTests::Task::StepComplete, + CCCDurableTests::Task::StepStarted, + CCCDurableTests::Task::StepComplete, + CCCDurableTests::Task::WorkflowComplete + ]) + + last = history.last + expect(last.payload.output).to eq('Hello Joe, your IP is 11.111.111 and its location is London, UK') + expect(last.payload.workflow_id).to eq(workflow_id) + end + end + + context 'with failed steps' do + it 'produces StepFailed message' do + expect(CCCDurableTests::IPResolver).to receive(:resolve).and_raise('Network Error!') + + started = make_started(CCCDurableTests::Task, args: [name]) + history = [started] + + until history.last.is_a?(CCCDurableTests::Task::StepFailed) + next_action = CCCDurableTests::Task.handle(history.last, history:) + expect(next_action).to be_a Sourced::CCC::Actions::Append + history += next_action.messages + end + + expect(history.map(&:class)).to eq([ + CCCDurableTests::Task::WorkflowStarted, + CCCDurableTests::Task::StepStarted, + CCCDurableTests::Task::StepFailed + ]) + expect(history.last.payload.error_class).to eq('RuntimeError') + expect(CCCDurableTests::FilledStringArray).to be === history.last.payload.backtrace + end + end + + context 'with previously successful step' do + it 'does not invoke step again, using cached result instead' do + get_ip_key = Sourced::CCC::DurableWorkflow.step_key(:get_ip, []) + geolocate_key = Sourced::CCC::DurableWorkflow.step_key(:geolocate, ['11.111.111']) + + history = [ + CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id:, args: [name] }), + CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, args: [] }), + CCCDurableTests::Task::StepComplete.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, output: '11.111.111' }), + CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: geolocate_key, step_name: :geolocate, args: ['11.111.111'] }) + ] + + expect(CCCDurableTests::IPResolver).not_to receive(:resolve) + expect(CCCDurableTests::Geolocator).to receive(:locate).with('11.111.111').and_return('Santiago, Chile') + + until history.last.is_a?(CCCDurableTests::Task::WorkflowComplete) + next_action = CCCDurableTests::Task.handle(history.last, history:) + expect(next_action).to be_a Sourced::CCC::Actions::Append + history += next_action.messages + end + + task = CCCDurableTests::Task.from(history) + expect(task.status).to eq(:complete) + expect(task.output).to eq('Hello Joe, your IP is 11.111.111 and its location is Santiago, Chile') + end + end + + context 'when workflow is finally failed' do + it 'does not try again' do + get_ip_key = Sourced::CCC::DurableWorkflow.step_key(:get_ip, []) + + history = [ + CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id:, args: [name] }), + CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, args: [] }), + CCCDurableTests::Task::StepFailed.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, error_class: 'NetworkError', error_message: 'foo', backtrace: [] }), + CCCDurableTests::Task::WorkflowFailed.new(payload: { workflow_id: }) + ] + + step_started = CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, args: [] }) + next_action = CCCDurableTests::Task.handle(step_started, history:) + expect(next_action).to eq(Sourced::CCC::Actions::OK) + end + end + + context 'with a different workflow handling irrelevant messages' do + it 'blows up' do + started = make_started(CCCDurableTests::AnotherTask, args: [name]) + history = [started] + + expect { + CCCDurableTests::Task.handle(history.last, history:) + }.to raise_error(Sourced::CCC::DurableWorkflow::UnknownMessageError) + end + end + + describe 'caching method calls by signature' do + it 'only invokes methods with the same arguments once per workflow' do + started = make_started(CCCDurableTests::MultiArgTask) + history = [started] + + allow(CCCDurableTests::Doubler).to receive(:double).and_call_original + + until history.last.is_a?(CCCDurableTests::MultiArgTask::WorkflowComplete) + next_action = CCCDurableTests::MultiArgTask.handle(history.last, history:) + expect(next_action).to be_a Sourced::CCC::Actions::Append + history += next_action.messages + end + + expect(history.last.payload.output).to eq(16) + expect(CCCDurableTests::Doubler).to have_received(:double).with(2).once + expect(CCCDurableTests::Doubler).to have_received(:double).with(4).once + end + end + + describe 'limited retries' do + it 'retries the configured number of times until it fails the workflow' do + started = make_started(CCCDurableTests::Retryable) + history = [started] + + 6.times do + next_action = CCCDurableTests::Retryable.handle(history.last, history:) + history += next_action.messages if next_action.respond_to?(:messages) + end + + task = CCCDurableTests::Retryable.from(history) + expect(task.status).to eq(:failed) + + expect(history.map(&:class)).to eq([ + CCCDurableTests::Retryable::WorkflowStarted, + CCCDurableTests::Retryable::StepStarted, + CCCDurableTests::Retryable::StepFailed, + CCCDurableTests::Retryable::StepStarted, + CCCDurableTests::Retryable::StepFailed, + CCCDurableTests::Retryable::WorkflowFailed + ]) + end + end + + context 'with context preserved across failures' do + it 'tracks context changes in event history' do + started = make_started(CCCDurableTests::WithContext) + history = [started] + + until history.last.is_a?(CCCDurableTests::WithContext::WorkflowComplete) + next_action = CCCDurableTests::WithContext.handle(history.last, history:) + history += next_action.messages + end + + task = CCCDurableTests::WithContext.from(history) + expect(task.output).to eq([2, 4, 6, 8, 10]) + + expect(history.map(&:class)).to eq([ + CCCDurableTests::WithContext::WorkflowStarted, + CCCDurableTests::WithContext::StepStarted, + CCCDurableTests::WithContext::StepFailed, + CCCDurableTests::WithContext::ContextUpdated, + CCCDurableTests::WithContext::StepStarted, + CCCDurableTests::WithContext::StepComplete, + CCCDurableTests::WithContext::ContextUpdated, + CCCDurableTests::WithContext::WorkflowComplete + ]) + + ctx_events = history.select { |m| m.is_a?(CCCDurableTests::WithContext::ContextUpdated) } + expect(ctx_events[0].payload.context).to eq(index: 2, results: [2, 4]) + expect(ctx_events[1].payload.context).to eq(index: 5, results: [2, 4, 6, 8, 10]) + end + end + + describe '#wait' do + it 'schedules a WaitStarted and WaitEnded combo' do + now = Time.now + + Timecop.freeze(now) do + started = make_started(CCCDurableTests::WithDelay) + history = [started] + + next_action = CCCDurableTests::WithDelay.handle(history.last, history:) + history += next_action.messages # StepStarted + + next_action = CCCDurableTests::WithDelay.handle(history.last, history:) + history += next_action.messages # StepComplete + + next_action = CCCDurableTests::WithDelay.handle(history.last, history:) + history += next_action.messages # WaitStarted + + expect(history.last).to be_a(CCCDurableTests::WithDelay::WaitStarted) + expect(history.last.payload.at).to eq(now + 10) + + next_action = CCCDurableTests::WithDelay.handle(history.last, history:) + expect(next_action).to be_a(Sourced::CCC::Actions::Schedule) + expect(next_action.at).to eq(now + 10) + expect(next_action.messages.first).to be_a(CCCDurableTests::WithDelay::WaitEnded) + expect(next_action.messages.first.payload.workflow_id).to eq('durable-test-1') + + history << next_action.messages.first + + until history.last.is_a?(CCCDurableTests::WithDelay::WorkflowComplete) + next_action = CCCDurableTests::WithDelay.handle(history.last, history:) + history += next_action.messages + end + + expect(history.map(&:class)).to eq([ + CCCDurableTests::WithDelay::WorkflowStarted, + CCCDurableTests::WithDelay::StepStarted, + CCCDurableTests::WithDelay::StepComplete, + CCCDurableTests::WithDelay::WaitStarted, + CCCDurableTests::WithDelay::WaitEnded, + CCCDurableTests::WithDelay::StepStarted, + CCCDurableTests::WithDelay::StepComplete, + CCCDurableTests::WithDelay::WorkflowComplete + ]) + end + end + end + + describe 'end-to-end via store + router' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::CCC::Store.new(db) } + let(:router) { Sourced::CCC::Router.new(store:) } + + before { store.install! } + + it 'drains two concurrent workflows to completion' do + router.register(CCCDurableTests::Task) + + wf1_id = "wf-#{SecureRandom.uuid}" + wf2_id = "wf-#{SecureRandom.uuid}" + store.append([CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id: wf1_id, args: ['Alice'] })]) + store.append([CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id: wf2_id, args: ['Bob'] })]) + + router.drain + + wf1, = Sourced::CCC.load(CCCDurableTests::Task, store:, workflow_id: wf1_id) + wf2, = Sourced::CCC.load(CCCDurableTests::Task, store:, workflow_id: wf2_id) + + expect(wf1.status).to eq(:complete) + expect(wf1.output).to include('Alice') + expect(wf2.status).to eq(:complete) + expect(wf2.output).to include('Bob') + end + end +end From 235b365ea8811881c5889fdd90939c9d2f17238e Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 11:41:35 +0100 Subject: [PATCH 102/115] Implement CCC::DurableWorkflow#handle_batch and rewrite tests to use GWT helpers --- lib/sourced/ccc/durable_workflow.rb | 39 ++++++++-------- spec/sourced/ccc/durable_workflow_spec.rb | 55 ++++++++++------------- 2 files changed, 44 insertions(+), 50 deletions(-) diff --git a/lib/sourced/ccc/durable_workflow.rb b/lib/sourced/ccc/durable_workflow.rb index 4d72e4be..54837a47 100644 --- a/lib/sourced/ccc/durable_workflow.rb +++ b/lib/sourced/ccc/durable_workflow.rb @@ -192,24 +192,27 @@ def self.execute(*args, store: CCC.store) Waiter.new(self, workflow_id, store:) end - # Router entry point. Processes one claimed message at a time: for each - # message, replay history up to and including that position, then run - # {#__handle} to derive the next action. + # Router entry point. Drops claim.messages from the read history (the + # router's +store.read(conditions)+ returns the full partition, including + # messages being claimed) and delegates to {.handle_batch}. def self.handle_claim(claim, history:) - workflow_id = claim.partition_value['workflow_id'] - ordered = history.messages.sort_by { |m| m.respond_to?(:position) ? m.position : 0 } - guard = history.guard - - each_with_partial_ack(claim.messages) do |msg| - msg_pos = msg.respond_to?(:position) ? msg.position : nil - prior = if msg_pos - ordered.take_while { |m| (m.respond_to?(:position) ? m.position : 0) <= msg_pos } - else - ordered - end - instance = new([workflow_id]) - instance.__replay(prior) - actions = instance.__handle(msg, guard: guard) + claim_positions = claim.messages.map { |m| m.position if m.respond_to?(:position) }.compact.to_set + prior = history.messages.reject { |m| m.respond_to?(:position) && claim_positions.include?(m.position) } + prior_history = ReadResult.new(messages: prior, guard: history.guard) + values = claim.partition_value.transform_keys(&:to_sym) + handle_batch(values, claim.messages, history: prior_history) + end + + # GWT-compatible entry point. +history.messages+ must be disjoint from + # +new_messages+ — the caller owns that distinction. + def self.handle_batch(partition_values, new_messages, history:, replaying: false) + workflow_id = partition_values[:workflow_id] + instance = new([workflow_id]) + instance.__replay(history.messages) + + each_with_partial_ack(new_messages) do |msg| + instance.__evolve(msg) + actions = instance.__handle(msg, guard: history.guard) [actions, msg] end end @@ -272,7 +275,7 @@ def load def initialize(partition_values = nil) @id = case partition_values when Array then partition_values.first - when Hash then partition_values[:workflow_id] || partition_values['workflow_id'] + when Hash then partition_values[:workflow_id] when String then partition_values else nil end diff --git a/spec/sourced/ccc/durable_workflow_spec.rb b/spec/sourced/ccc/durable_workflow_spec.rb index 00d97a3b..22524b97 100644 --- a/spec/sourced/ccc/durable_workflow_spec.rb +++ b/spec/sourced/ccc/durable_workflow_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' require 'sourced/ccc' require 'sourced/ccc/store' +require 'sourced/ccc/testing/rspec' require 'sequel' module CCCDurableTests @@ -101,6 +102,8 @@ def execute end RSpec.describe Sourced::CCC::DurableWorkflow do + include Sourced::CCC::Testing::RSpec + let(:workflow_id) { 'durable-test-1' } let(:name) { 'Joe' } @@ -162,25 +165,19 @@ def make_started(klass, args: [], workflow_id: 'durable-test-1') get_ip_key = Sourced::CCC::DurableWorkflow.step_key(:get_ip, []) geolocate_key = Sourced::CCC::DurableWorkflow.step_key(:geolocate, ['11.111.111']) - history = [ - CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id:, args: [name] }), - CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, args: [] }), - CCCDurableTests::Task::StepComplete.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, output: '11.111.111' }), - CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: geolocate_key, step_name: :geolocate, args: ['11.111.111'] }) - ] - expect(CCCDurableTests::IPResolver).not_to receive(:resolve) expect(CCCDurableTests::Geolocator).to receive(:locate).with('11.111.111').and_return('Santiago, Chile') - until history.last.is_a?(CCCDurableTests::Task::WorkflowComplete) - next_action = CCCDurableTests::Task.handle(history.last, history:) - expect(next_action).to be_a Sourced::CCC::Actions::Append - history += next_action.messages - end - - task = CCCDurableTests::Task.from(history) - expect(task.status).to eq(:complete) - expect(task.output).to eq('Hello Joe, your IP is 11.111.111 and its location is Santiago, Chile') + with_reactor(CCCDurableTests::Task, workflow_id: workflow_id) + .given(CCCDurableTests::Task::WorkflowStarted, workflow_id:, args: [name]) + .given(CCCDurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) + .given(CCCDurableTests::Task::StepComplete, workflow_id:, key: get_ip_key, step_name: :get_ip, output: '11.111.111') + .when(CCCDurableTests::Task::StepStarted, workflow_id:, key: geolocate_key, step_name: :geolocate, args: ['11.111.111']) + .then( + CCCDurableTests::Task::StepComplete.new(payload: { + workflow_id:, key: geolocate_key, step_name: :geolocate, output: 'Santiago, Chile' + }) + ) end end @@ -188,27 +185,21 @@ def make_started(klass, args: [], workflow_id: 'durable-test-1') it 'does not try again' do get_ip_key = Sourced::CCC::DurableWorkflow.step_key(:get_ip, []) - history = [ - CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id:, args: [name] }), - CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, args: [] }), - CCCDurableTests::Task::StepFailed.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, error_class: 'NetworkError', error_message: 'foo', backtrace: [] }), - CCCDurableTests::Task::WorkflowFailed.new(payload: { workflow_id: }) - ] - - step_started = CCCDurableTests::Task::StepStarted.new(payload: { workflow_id:, key: get_ip_key, step_name: :get_ip, args: [] }) - next_action = CCCDurableTests::Task.handle(step_started, history:) - expect(next_action).to eq(Sourced::CCC::Actions::OK) + with_reactor(CCCDurableTests::Task, workflow_id: workflow_id) + .given(CCCDurableTests::Task::WorkflowStarted, workflow_id:, args: [name]) + .given(CCCDurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) + .given(CCCDurableTests::Task::StepFailed, workflow_id:, key: get_ip_key, step_name: :get_ip, error_class: 'NetworkError', error_message: 'foo', backtrace: []) + .given(CCCDurableTests::Task::WorkflowFailed, workflow_id:) + .when(CCCDurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) + .then(Sourced::CCC::Testing::RSpec::NONE) end end context 'with a different workflow handling irrelevant messages' do it 'blows up' do - started = make_started(CCCDurableTests::AnotherTask, args: [name]) - history = [started] - - expect { - CCCDurableTests::Task.handle(history.last, history:) - }.to raise_error(Sourced::CCC::DurableWorkflow::UnknownMessageError) + with_reactor(CCCDurableTests::Task, workflow_id: workflow_id) + .when(CCCDurableTests::AnotherTask::WorkflowStarted, workflow_id:, args: [name]) + .then(Sourced::CCC::DurableWorkflow::UnknownMessageError) end end From d0e0540f324d4201eb4a9972c17359eecdb47827 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 12:42:45 +0100 Subject: [PATCH 103/115] Promote CCC to top-level Sourced namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the old stream+seq implementation with the stream-less, partition-based design from Sourced::CCC. All former CCC code now lives under the top-level Sourced namespace; the old Actor / Unit / ActiveRecord backend / Rails / Falcon / stream-based DurableWorkflow are deleted outright. - Rename internal handler prefixes ccc_decide / ccc_reaction / ccc_evolution to sourced_decide / sourced_reaction / sourced_evolution - Merge Sourced::Topology so the new stream-less build lives alongside the Prism source analyzer - Keep Injector, Types, ErrorStrategy, WorkQueue, CatchUpPoller, InlineNotifier, Async/Thread executors — all used by the promoted code - Replace root README with the promoted reactor docs, stripped of CCC terminology - Move examples/ccc_app to examples/app; delete examples that depended on removed APIs - Drop Sourced::InvalidMessageError, Types::TrailingModuleName, Types::ModuleToMethodName (no remaining callers) - All 472 previously-CCC specs pass against the new top-level API Co-Authored-By: Claude Opus 4.6 (1M context) --- Gemfile | 2 - Gemfile.lock | 4 - README.md | 1850 ++++++----------- examples/app/Gemfile | 9 + examples/app/Gemfile.lock | 286 +++ examples/app/README.md | 245 +++ examples/app/app.rb | 93 + examples/app/config.ru | 19 + examples/app/domain.rb | 142 ++ examples/app/falcon.rb | 14 + examples/app/storage/.gitkeep | 0 examples/cart.rb | 232 --- examples/lite_cart.rb | 155 -- examples/pub.rb | 10 - examples/socket.rb | 18 - examples/workers.rb | 5 - lib/sourced.rb | 358 ++-- lib/sourced/actions.rb | 212 +- lib/sourced/actor.rb | 386 ---- lib/sourced/backends/pg_backend.rb | 21 - lib/sourced/backends/sequel_backend.rb | 980 --------- .../backends/sequel_backend/group_updater.rb | 45 - .../backends/sequel_backend/installer.rb | 88 - .../001_create_sourced_tables.rb.erb | 83 - .../001_create_sourced_tables_sqlite.rb.erb | 79 - .../backends/sequel_backend/pg_notifier.rb | 96 - lib/sourced/backends/sqlite_backend.rb | 454 ---- lib/sourced/backends/test_backend.rb | 277 --- lib/sourced/backends/test_backend/group.rb | 180 -- lib/sourced/backends/test_backend/state.rb | 86 - lib/sourced/ccc.rb | 274 --- lib/sourced/ccc/README.md | 967 --------- lib/sourced/ccc/actions.rb | 145 -- lib/sourced/ccc/command_context.rb | 116 -- lib/sourced/ccc/configuration.rb | 77 - lib/sourced/ccc/consumer.rb | 166 -- lib/sourced/ccc/decider.rb | 134 -- lib/sourced/ccc/dispatcher.rb | 223 -- lib/sourced/ccc/durable_workflow.rb | 400 ---- lib/sourced/ccc/evolve.rb | 63 - lib/sourced/ccc/falcon.rb | 5 - lib/sourced/ccc/falcon/environment.rb | 34 - lib/sourced/ccc/falcon/service.rb | 38 - lib/sourced/ccc/installer.rb | 80 - lib/sourced/ccc/message.rb | 265 --- lib/sourced/ccc/projector.rb | 87 - lib/sourced/ccc/react.rb | 177 -- lib/sourced/ccc/router.rb | 180 -- lib/sourced/ccc/scheduled_message_poller.rb | 38 - lib/sourced/ccc/stale_claim_reaper.rb | 77 - lib/sourced/ccc/store.rb | 1305 ------------ lib/sourced/ccc/supervisor.rb | 101 - lib/sourced/ccc/sync.rb | 91 - lib/sourced/ccc/testing/rspec.rb | 284 --- lib/sourced/ccc/topology.rb | 437 ---- lib/sourced/ccc/worker.rb | 94 - lib/sourced/command_context.rb | 145 +- lib/sourced/command_methods.rb | 151 -- lib/sourced/configuration.rb | 197 +- lib/sourced/consumer.rb | 218 +- lib/sourced/decider.rb | 132 ++ lib/sourced/dispatcher.rb | 120 +- lib/sourced/durable_workflow.rb | 330 +-- lib/sourced/evolve.rb | 146 +- lib/sourced/falcon/environment.rb | 41 +- lib/sourced/falcon/service.rb | 35 +- lib/sourced/handler.rb | 84 - lib/sourced/house_keeper.rb | 76 - lib/sourced/installer.rb | 78 + lib/sourced/message.rb | 273 +-- .../001_create_sourced_tables.rb.erb} | 0 lib/sourced/projector.rb | 261 +-- lib/sourced/pubsub/pg.rb | 253 --- lib/sourced/pubsub/test.rb | 86 - lib/sourced/rails/install_generator.rb | 57 - lib/sourced/rails/railtie.rb | 16 - lib/sourced/rails/templates/bin_sors | 8 - .../rails/templates/create_sors_tables.rb.erb | 55 - lib/sourced/react.rb | 216 +- lib/sourced/router.rb | 291 ++- lib/sourced/scheduled_message_poller.rb | 36 + lib/sourced/stale_claim_reaper.rb | 75 + lib/sourced/store.rb | 1303 ++++++++++++ lib/sourced/supervisor.rb | 74 +- lib/sourced/sync.rb | 109 +- lib/sourced/testing/rspec.rb | 448 ++-- lib/sourced/topology.rb | 97 +- lib/sourced/types.rb | 12 +- lib/sourced/unit.rb | 336 --- lib/sourced/worker.rb | 67 +- spec/actions_spec.rb | 88 - spec/actor_spec.rb | 267 --- spec/async_executor_spec.rb | 10 - spec/backends/concurrent_projectors_spec.rb | 115 - spec/backends/pg_backend_spec.rb | 130 -- spec/backends/pg_notifier_spec.rb | 134 -- spec/backends/sqlite_backend_spec.rb | 39 - spec/backends/test_backend_spec.rb | 18 - spec/catchup_poller_spec.rb | 48 - spec/command_context_spec.rb | 235 ++- spec/command_methods_spec.rb | 103 - spec/configuration_spec.rb | 315 ++- spec/consumer_spec.rb | 74 - spec/{sourced/ccc => }/decider_spec.rb | 150 +- spec/dispatcher_spec.rb | 369 +++- spec/durable_workflow_spec.rb | 205 +- spec/error_strategy_spec.rb | 86 - spec/evolve_spec.rb | 155 +- spec/{sourced/ccc => }/handle_spec.rb | 106 +- spec/handler_spec.rb | 55 - spec/injector_spec.rb | 125 -- spec/load_actor_spec.rb | 104 - spec/{sourced/ccc => }/load_spec.rb | 76 +- spec/message_spec.rb | 525 ++++- spec/notifier_spec.rb | 137 -- spec/projector_spec.rb | 505 +++-- spec/pubsub/pg_spec.rb | 42 - spec/pubsub/test_spec.rb | 140 -- spec/react_dsl_spec.rb | 36 - spec/react_spec.rb | 284 ++- spec/router_spec.rb | 757 ++++--- spec/shared_examples/backend_examples.rb | 1338 ------------ spec/shared_examples/executor_examples.rb | 41 - spec/shared_examples/unit_examples.rb | 375 ---- spec/sourced/ccc/command_context_spec.rb | 242 --- spec/sourced/ccc/configuration_spec.rb | 276 --- spec/sourced/ccc/dispatcher_spec.rb | 375 ---- spec/sourced/ccc/durable_workflow_spec.rb | 352 ---- spec/sourced/ccc/evolve_spec.rb | 102 - spec/sourced/ccc/message_spec.rb | 481 ----- spec/sourced/ccc/projector_spec.rb | 439 ---- spec/sourced/ccc/react_spec.rb | 218 -- spec/sourced/ccc/router_spec.rb | 557 ----- spec/sourced/ccc/supervisor_spec.rb | 166 -- spec/sourced/ccc/testing/rspec_spec.rb | 290 --- spec/sourced/ccc/topology_spec.rb | 432 ---- spec/sourced_spec.rb | 80 - spec/spec_helper.rb | 18 - .../ccc => }/stale_claim_reaper_spec.rb | 14 +- spec/{sourced/ccc => }/store_spec.rb | 506 ++--- spec/supervisor_spec.rb | 139 +- spec/support/unit_test_fixtures.rb | 240 --- spec/sync_spec.rb | 59 - spec/testing/rspec_spec.rb | 497 +++-- spec/thread_executor_spec.rb | 10 - spec/topology_spec.rb | 377 ++-- spec/unit_sequel_postgres_spec.rb | 22 - spec/unit_spec.rb | 13 - spec/work_queue_spec.rb | 81 - spec/worker_spec.rb | 138 -- 150 files changed, 7896 insertions(+), 23528 deletions(-) create mode 100644 examples/app/Gemfile create mode 100644 examples/app/Gemfile.lock create mode 100644 examples/app/README.md create mode 100644 examples/app/app.rb create mode 100644 examples/app/config.ru create mode 100644 examples/app/domain.rb create mode 100644 examples/app/falcon.rb create mode 100644 examples/app/storage/.gitkeep delete mode 100644 examples/cart.rb delete mode 100644 examples/lite_cart.rb delete mode 100644 examples/pub.rb delete mode 100644 examples/socket.rb delete mode 100644 examples/workers.rb delete mode 100644 lib/sourced/actor.rb delete mode 100644 lib/sourced/backends/pg_backend.rb delete mode 100644 lib/sourced/backends/sequel_backend.rb delete mode 100644 lib/sourced/backends/sequel_backend/group_updater.rb delete mode 100644 lib/sourced/backends/sequel_backend/installer.rb delete mode 100644 lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables.rb.erb delete mode 100644 lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables_sqlite.rb.erb delete mode 100644 lib/sourced/backends/sequel_backend/pg_notifier.rb delete mode 100644 lib/sourced/backends/sqlite_backend.rb delete mode 100644 lib/sourced/backends/test_backend.rb delete mode 100644 lib/sourced/backends/test_backend/group.rb delete mode 100644 lib/sourced/backends/test_backend/state.rb delete mode 100644 lib/sourced/ccc.rb delete mode 100644 lib/sourced/ccc/README.md delete mode 100644 lib/sourced/ccc/actions.rb delete mode 100644 lib/sourced/ccc/command_context.rb delete mode 100644 lib/sourced/ccc/configuration.rb delete mode 100644 lib/sourced/ccc/consumer.rb delete mode 100644 lib/sourced/ccc/decider.rb delete mode 100644 lib/sourced/ccc/dispatcher.rb delete mode 100644 lib/sourced/ccc/durable_workflow.rb delete mode 100644 lib/sourced/ccc/evolve.rb delete mode 100644 lib/sourced/ccc/falcon.rb delete mode 100644 lib/sourced/ccc/falcon/environment.rb delete mode 100644 lib/sourced/ccc/falcon/service.rb delete mode 100644 lib/sourced/ccc/installer.rb delete mode 100644 lib/sourced/ccc/message.rb delete mode 100644 lib/sourced/ccc/projector.rb delete mode 100644 lib/sourced/ccc/react.rb delete mode 100644 lib/sourced/ccc/router.rb delete mode 100644 lib/sourced/ccc/scheduled_message_poller.rb delete mode 100644 lib/sourced/ccc/stale_claim_reaper.rb delete mode 100644 lib/sourced/ccc/store.rb delete mode 100644 lib/sourced/ccc/supervisor.rb delete mode 100644 lib/sourced/ccc/sync.rb delete mode 100644 lib/sourced/ccc/testing/rspec.rb delete mode 100644 lib/sourced/ccc/topology.rb delete mode 100644 lib/sourced/ccc/worker.rb delete mode 100644 lib/sourced/command_methods.rb create mode 100644 lib/sourced/decider.rb delete mode 100644 lib/sourced/handler.rb delete mode 100644 lib/sourced/house_keeper.rb create mode 100644 lib/sourced/installer.rb rename lib/sourced/{ccc/migrations/001_create_ccc_tables.rb.erb => migrations/001_create_sourced_tables.rb.erb} (100%) delete mode 100644 lib/sourced/pubsub/pg.rb delete mode 100644 lib/sourced/pubsub/test.rb delete mode 100644 lib/sourced/rails/install_generator.rb delete mode 100644 lib/sourced/rails/railtie.rb delete mode 100644 lib/sourced/rails/templates/bin_sors delete mode 100644 lib/sourced/rails/templates/create_sors_tables.rb.erb create mode 100644 lib/sourced/scheduled_message_poller.rb create mode 100644 lib/sourced/stale_claim_reaper.rb create mode 100644 lib/sourced/store.rb delete mode 100644 lib/sourced/unit.rb delete mode 100644 spec/actions_spec.rb delete mode 100644 spec/actor_spec.rb delete mode 100644 spec/async_executor_spec.rb delete mode 100644 spec/backends/concurrent_projectors_spec.rb delete mode 100644 spec/backends/pg_backend_spec.rb delete mode 100644 spec/backends/pg_notifier_spec.rb delete mode 100644 spec/backends/sqlite_backend_spec.rb delete mode 100644 spec/backends/test_backend_spec.rb delete mode 100644 spec/catchup_poller_spec.rb delete mode 100644 spec/command_methods_spec.rb delete mode 100644 spec/consumer_spec.rb rename spec/{sourced/ccc => }/decider_spec.rb (52%) delete mode 100644 spec/error_strategy_spec.rb rename spec/{sourced/ccc => }/handle_spec.rb (59%) delete mode 100644 spec/handler_spec.rb delete mode 100644 spec/injector_spec.rb delete mode 100644 spec/load_actor_spec.rb rename spec/{sourced/ccc => }/load_spec.rb (73%) delete mode 100644 spec/notifier_spec.rb delete mode 100644 spec/pubsub/pg_spec.rb delete mode 100644 spec/pubsub/test_spec.rb delete mode 100644 spec/react_dsl_spec.rb delete mode 100644 spec/shared_examples/backend_examples.rb delete mode 100644 spec/shared_examples/executor_examples.rb delete mode 100644 spec/shared_examples/unit_examples.rb delete mode 100644 spec/sourced/ccc/command_context_spec.rb delete mode 100644 spec/sourced/ccc/configuration_spec.rb delete mode 100644 spec/sourced/ccc/dispatcher_spec.rb delete mode 100644 spec/sourced/ccc/durable_workflow_spec.rb delete mode 100644 spec/sourced/ccc/evolve_spec.rb delete mode 100644 spec/sourced/ccc/message_spec.rb delete mode 100644 spec/sourced/ccc/projector_spec.rb delete mode 100644 spec/sourced/ccc/react_spec.rb delete mode 100644 spec/sourced/ccc/router_spec.rb delete mode 100644 spec/sourced/ccc/supervisor_spec.rb delete mode 100644 spec/sourced/ccc/testing/rspec_spec.rb delete mode 100644 spec/sourced/ccc/topology_spec.rb delete mode 100644 spec/sourced_spec.rb rename spec/{sourced/ccc => }/stale_claim_reaper_spec.rb (93%) rename spec/{sourced/ccc => }/store_spec.rb (76%) delete mode 100644 spec/support/unit_test_fixtures.rb delete mode 100644 spec/sync_spec.rb delete mode 100644 spec/thread_executor_spec.rb delete mode 100644 spec/unit_sequel_postgres_spec.rb delete mode 100644 spec/unit_spec.rb delete mode 100644 spec/work_queue_spec.rb delete mode 100644 spec/worker_spec.rb diff --git a/Gemfile b/Gemfile index fb62b510..4a9f830d 100644 --- a/Gemfile +++ b/Gemfile @@ -14,8 +14,6 @@ group :development do end group :test do - gem 'dotenv' - gem 'pg' gem 'rspec', '~> 3.0' gem 'sequel' gem 'sqlite3' diff --git a/Gemfile.lock b/Gemfile.lock index bb1843f8..b7a50169 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,7 +33,6 @@ GEM irb (~> 1.10) reline (>= 0.3.8) diff-lcs (1.5.1) - dotenv (3.1.4) fiber-annotation (0.2.0) fiber-local (1.1.0) fiber-storage @@ -57,7 +56,6 @@ GEM parser (3.3.7.1) ast (~> 2.4.1) racc - pg (1.5.8) plumb (0.0.17) bigdecimal concurrent-ruby @@ -119,9 +117,7 @@ PLATFORMS DEPENDENCIES debug docco! - dotenv logger - pg rake (~> 13.0) rspec (~> 3.0) rubocop diff --git a/README.md b/README.md index 9a1ac9e5..aac13856 100644 --- a/README.md +++ b/README.md @@ -1,1619 +1,967 @@ -# sourced +# Sourced — Stream-less Event Sourcing -**WORK IN PROGRESS** +Sourced is a Ruby library for aggregateless, stream-less event sourcing. Events go into a flat, globally-ordered log. Consistency context is assembled dynamically by querying relevant facts via key-value pairs extracted from event payloads, rather than being pre-assigned to fixed streams. -Event Sourcing / CQRS library for Ruby. -There's many ES gems available already. The objectives here are: -* Cohesive and toy-like DX. -* Eventual consistency by default. Actor-like execution model. -* Low-level APIs for durable messaging. -* Supports the [Decide, Evolve, React pattern](https://ismaelcelis.com/posts/decide-evolve-react-pattern-in-ruby/) -* Control concurrency by modeling. -* Simple to operate: it should be as simple to run as most Ruby queuing systems. -* Explore ES as a programming model for Ruby apps. +## Core Concepts -A small demo app [here](https://github.com/ismasan/sourced_todo). +- **No streams or aggregates** — all messages share a single append-only log with auto-increment positions. +- **Partitioning by attributes** — reactors declare which payload attributes define their consistency boundary (e.g. `partition_by :course_id`). The store uses these to build query conditions and claim work. +- **Decide → Evolve → React** — reactors handle commands, evolve state from history, and react to events. +- **Optimistic concurrency** — reads return a `ConsistencyGuard` that detects conflicting writes at append time. -## The programming model +## Messages -If you're unfamiliar with Event Sourcing, you can read this first: [Event Sourcing from the ground up, with Ruby examples](https://ismaelcelis.com/posts/event-sourcing-ruby-examples) -For a high-level overview of the mental model, [read this](https://ismaelcelis.com/posts/2025-04-give-it-time/). Or the video version, [here](https://www.youtube.com/watch?v=EgUwnzUJHMA). +Messages have no `stream_id` or `seq` — they get a global `position` when stored. -The entire behaviour of an event-sourced app is described via **commands**, **events** and **reactions**. +```ruby +# Define base classes for your domain (optional but recommended) +class MyEvent < Sourced::Event; end +class MyCommand < Sourced::Command; end -sourced-arch-diagram +# Define typed messages with payload attributes +CreateCourse = MyCommand.define('courses.create') do + attribute :course_id, String + attribute :course_name, String +end +CourseCreated = MyEvent.define('courses.created') do + attribute :course_id, String + attribute :course_name, String +end +``` -* **Commands** are _intents_ to effect some change in the state of the system. Ex. `Add cart item`, `Place order`, `Update email`, etc. -* **Events** are produced after handling a command and they describe _facts_ or state changes in the system. Ex. `Item added to cart`, `order placed`, `email updated`. Events are stored and you can use them to build views ("projections"), caches and reports to support UIs, or other artifacts. -* **Reactions** are blocks of code that run _after_ an event has been processed and can dispatch new commands in a workflow or automation. -* **State** is whatever object you need to hold the current state of a part of the system. It's usually derived from past events, and it's just enough to interrogate the state of the system and make the next decision. +### Message features -## Actors +- **Auto-generated UUIDs** for `id`, `causation_id`, and `correlation_id` +- **Causal tracing** via `#correlate(other_message)` — sets `causation_id` and `correlation_id` +- **Auto-indexed keys** — `#extracted_keys` returns `[["course_id", "c1"], ["course_name", "Algebra"]]` from payload attributes, used by the store to index messages for querying +- **Registry** — `Sourced::Message.registry` indexes defined types. Use `.from(type: "courses.created", payload: {...})` to instantiate from a type string. +- **Typed payloads** — Plumb/Types DSL for attribute coercion and validation -### Overview +## Store -Actors are classes that encapsulate the full life-cycle of a concept in your domain, backed by an event stream. This includes loading state from past events and handling commands for a part of your system. They can also define reactions to their own events, or events emitted by other actors. This is a simple shopping cart actor. +`Sourced::Store` is an SQLite-backed store (via Sequel) providing the flat message log, key-pair indexing, and consumer group management. ```ruby -class Cart < Sourced::Actor - # Define what cart state looks like. - # This is the initial state which will be updated by applying events. - # The state holds whatever data is relevant to decide how to handle a command. - # It can be any object you need. A custom class instance, a Hash, an Array, etc. - CartState = Struct.new(:id, :status, :items) do - def total = items.sum { |it| it.price * it.quantity } - end - - CartItem = Struct.new(:product_id, :price, :quantity) - - # This factory is called to initialise a blank cart. - state do |id| - CartState.new(id:, status: 'open', items: []) - end - - # Define a command and its handling logic. - # The command handler will be passed the current state of the cart, - # and the command instance itself. - # Its main job is to validate business rules and decide whether new events - # can be emitted to update the state - command :add_item, product_id: String, price: Integer, quantity: Integer do |cart, cmd| - # Validate that this command can run - raise "cart is not open!" unless cart.status == 'open' - # Produce a new event with the same attributes as the command - event :item_added, cmd.payload - end - - # Define an event handler that will "evolve" the state of the cart by adding an item to it. - # These handlers are also used to "hydrate" the initial state from Sourced's storage backend - # when first handling a command - event :item_added, product_id: String, price: Integer, quantity: Integer do |cart, event| - cart.items << CartItem.new(**event.payload.to_h) - end - - # Optionally, define how this actor reacts to the event above. - # .reaction blocks can dispatch new commands that will be routed to their handlers. - # This allows you to build workflows. - reaction :item_added do |event| - # Evaluate whether we should dispatch the next command. - # Here we could fetch some external data or query that might be needed - # to populate the new commands. - # Here we dispatch a command to the same stream_id present in the event - dispatch(:send_admin_email, product_id: event.payload.product_id) - end - - # Handle the :send_admin_email dispatched by the reaction above - command :send_admin_email, product_id: String do |cart, cmd| - # maybe produce new events - end -end -``` - -Using the `Cart` actor in an IRB console. This will use Sourced's in-memory backend by default. +require 'sequel' -```ruby -cart = Cart.new(id: 'test-cart') -cart.state.total # => 0 -# Instantiate a command and handle it -cmd = Cart::AddItem.build('test-cart', product_id: 'p123', price: 1000, quantity: 2) -events = cart.decide(cmd) -# => [Cart::ItemAdded.new(...)] -cmd.valid? # true -# Inspect state -cart.state.total # 2000 -cart.items.items.size # 1 -# Inspect that events were stored -cart.seq # 2 the sequence number or "version" in storage. Ie. how many commands / events exist for this cart -# Append new messages to the backend -Sourced.config.backend.append_to_stream('test-cart', events) -# Load events for cart -events = Sourced.history_for(cart) -# => an array with instances of [Cart::AddItem, Cart::ItemAdded] -events.map(&:type) # ['cart.add_item', 'cart.item_added'] +db = Sequel.sqlite('my_app.db') +store = Sourced::Store.new(db) +store.install! # creates tables (idempotent) ``` -Try loading a new cart instance from recorded events +### Appending messages ```ruby -cart2, events = Sourced.load(Cart, 'test-cart') -cart2.seq # 2 -cart2.state.total # 2000 -cart2.state.items.size # 1 +cmd = CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) +position = store.append(cmd) # returns the assigned position ``` -### Registering actors - -Invoking commands directly on an actor instance works in an IRB console or a synchronous-only web handler, but for actors to be available to background workers, and to react to other actor's events, you need to register them. +### Reading with query conditions ```ruby -Sourced.register(Cart) -``` - -This achieves two things: - -1. Messages can be routed to this actor by background processes, using `Sourced.dispatch(message)`. -2. The actor can _react_ to other events in the system (more on event choreography later), via its low-level `.handle(event)` [Reactor Interface](#the-reactor-interface). - -These two properties are what enables asynchronous, eventually-consistent systems in Sourced. +# Build conditions for a specific course +conditions = CourseCreated.to_conditions(course_id: 'c1') +# => [QueryCondition('courses.created', attrs: { course_id: 'c1' })] -### Expanded message syntax - -Commands and event structs can also be defined separately as `Sourced::Command` and `Sourced::Event` sub-classes. - -These definitions include a message _type_ (for storage) and payload attributes schema, if any. - -```ruby -module Carts - # A command to add an item to the cart - # Commands may come from HTML forms, so we use Types::Lax to coerce attributes - AddItem = Sourced::Command.define('carts.add_item') do - attribute :product_id, Types::Lax::Integer - attribute :quantity, Types::Lax::Integer.default(1) - attribute :price, Types::Lax::Integer.default(0) - end - - # An event to track items added to the cart - # Events are only produced by valid commands, so we don't - # need validations or coercions - ItemAdded = Sourced::Event.define('carts.item_added') do - attribute :product_id, Integer - attribute :quantity, Integer - attribute :price, Integer - end - - ## Now define command and event handlers in a Actor - class Cart < Sourced::Actor - # Initial state, etc... - - command AddItem do |cart, cmd| - # logic here - event ItemAdded, cmd.payload - end - - event ItemAdded do |cart, event| - cart.items << CartItem.new(**event.payload.to_h) - end - end -end +# Read matching messages +result = store.read(conditions) +result.messages # => [PositionedMessage, ...] +result.guard # => ConsistencyGuard (for optimistic concurrency) ``` -### `.command` block - -The class-level `.command` block defines a _command handler_. Its job is to take a command (from a user, an automation, etc), validate it, and apply state changes by publishing new events. - -sourced-command-handler - +### Optimistic concurrency ```ruby -command AddItem do |cart, cmd| - # logic here... - # apply and publish one or more new events - # using instance-level #event(event_type, **payload) - event ItemAdded, product_id: cmd.payload.product_id -end -``` - - - -### `.event` block +result = store.read(conditions) -The class-level `.event` block registers an _event handler_ used to _evolve_ the actor's internal state. - -These blocks are used both to load the initial state when handling a command, and to apply new events to the state in command handlers. - -sourced-evolve-handler - - -```ruby -event ItemAdded do |cart, event| - cart.items << CartItem.new(**event.payload.to_h) -end +# ... later, append with conflict detection +store.append(new_events, guard: result.guard) +# raises Sourced::ConcurrentAppendError if conflicting messages +# were appended after the read ``` -These handlers are pure: given the same state and event, they should always update the state in the same exact way. They should never reach out to the outside (API calls, current time, etc), and they should never run validations. They work on events already committed to history, which by definition are assumed to be valid. +### Partition reads -### `.before_evolve` block - -The class-level `.before_evolve` block registers a callback that runs **before** each registered event handler during state evolution. This is useful for common logic that should run before all event handlers, such as updating timestamps or recording metadata. +`read_partition` uses AND semantics — a message is included only when every partition attribute it declares matches the given value. ```ruby -class CartListings < Sourced::Projector::StateStored - state do |id| - { id:, items: [], updated_at: nil, seq: 0 } - end - - # This block runs before any .event handler - before_evolve do |state, event| - state[:updated_at] = event.created_at - state[:seq] = event.seq - end - - event Cart::ItemAdded do |state, event| - state[:items] << event.payload.to_h - end - - event Cart::Placed do |state, event| - state[:status] = :placed - end -end +result = store.read_partition( + { course_id: 'c1' }, + handled_types: ['courses.created', 'courses.enrolled'] +) ``` -The `before_evolve` callback only runs for events that have a registered handler via the `.event` macro. If an event is not handled by this class, the callback is skipped for that event. - -### `.reaction` block - -The class-level `.reaction` block registers an event handler that _reacts_ to events already published by this or other Actors. - -`.reaction` blocks can dispatch the next command in a workflow with the instance-level `#dispatch` helper. - -sourced-react-handler +### Browsing the global log +`read_all` paginates the entire message log, without requiring query conditions or partition attributes. It returns a `ReadAllResult` with `messages` and `last_position` (the current max position in the store), so clients know whether more pages exist. ```ruby -reaction ItemAdded do |cart, event| - # dispatch the next command to the event's stream_id - dispatch( - CheckInventory, - product_id: event.payload.product_id, - quantity: event.payload.quantity - ) -end -``` - -You can also dispatch commanda to _other_ streams. For example for starting concurrent workflows. +# First page (default limit: 50, ascending order) +result = store.read_all(limit: 20) +result.messages # => [PositionedMessage, ...] +result.last_position # => 100 (max position in the store) -```ruby -# dispatch a command to a new custom-made stream_id -dispatch(CheckInventory, event.payload).to("cart-#{Time.now.to_i}") +# Next page — pass the last message's position as cursor +result = store.read_all(from_position: result.messages.last.position, limit: 20) -# Or use Sourced.new_stream_id -dispatch(CheckInventory, event.payload).to(Sourced.new_stream_id) +# Check if there are more pages +has_more = result.messages.any? && result.messages.last.position < result.last_position -# Or start a new stream and dispatch commands to another actor -dispatch(:notify, message: 'hello!').to(NotifierActor) +# Destructuring is also supported +messages, last_position = store.read_all(limit: 20) ``` -#### `.reaction` block with actor state - - `.reaction` blocks receive the actor state, which is derived by applying past events to it (same as when handling commands). +Use `order: :desc` for reverse-chronological browsing (newest first). Pagination works the same way — `from_position` fetches messages *before* the given position. ```ruby -# Define an event handler to evolve state -event ItemAdded do |state, event| - state[:item_count] += 1 -end +result = store.read_all(order: :desc, limit: 20) -# Now react to it and check state -reaction ItemAdded do |state, event| - if state[:item_count] > 30 - dispatch NotifyBigCart - end -end +# Next page of older messages +result = store.read_all(from_position: result.messages.last.position, order: :desc, limit: 20) ``` -#### `.reaction` with state for all events - -If the event name or class is omitted, the `.reaction` macro registers reaction handlers for all events already registered for the actor with the `.event` macro, minus events that have specific reaction handlers defined. - -```ruby -# wildcard reaction for all evolved events -reaction do |state, event| - if state[:item_count] > 30 - dispatch NotifyBigCart - end -end -``` +#### Iterating all messages with `to_enum` -#### `.reaction` for multiple events +`ReadAllResult#to_enum` returns a lazy `Enumerator` that transparently fetches subsequent pages as you iterate, using the same `order` and `limit` from the original query. ```ruby -reaction ItemAdded, InventoryChecked do |state, event| - # etc +# Iterate all messages in pages of 50 +store.read_all(limit: 50).to_enum.each do |msg| + puts "#{msg.position}: #{msg.type}" end -``` -It also works with symbols, for messages that have been defined as symbols (ex `event :item_added`) +# Works with Enumerable methods +store.read_all(order: :desc, limit: 100).to_enum.map(&:type) -```ruby -reaction :item_added, InventoryChecked do |state, event| - # etc -end +# Supports lazy enumeration — stops fetching pages once satisfied +store.read_all(limit: 20).to_enum.lazy.select { |m| + m.type == 'courses.created' +}.first(5) ``` -## Causation and correlation - -When a command produces events, or when an event makes a reactor dispatch a new command, the cause-and-effect relationship between these messages is tracked by Sourced in the form of `correlation_id` and `causation_id` properties in each message's metadata. - -sourced-causation-correlation - - -This helps the system keep a full audit trail of the cause-and-effect behaviour of the entire system. - -CleanShot 2025-11-11 at 23 59 40 +### Database setup -## Background vs. foreground execution +`Store#install!` creates all required tables directly (useful for scripts, tests, and quick prototyping). For production apps using Sequel migrations, the store can export a migration file instead. -By default Sourced processes commands and events **asynchronously** through background workers. Each reactor picks up messages at its own pace — the system is eventually consistent. - -Sometimes you need **synchronous, all-or-nothing** execution: a web request handler that must know the full outcome before responding, or a test that needs deterministic behaviour. `Sourced::Unit` provides this. - -### `Sourced::Unit` - -A Unit wires one or more reactors together and runs the full command → event → reaction → command chain inside a **single backend transaction**, using breadth-first traversal of the message graph. If any step raises, the entire transaction rolls back. +#### Quick setup (e.g. scripts, tests) ```ruby -unit = Sourced::Unit.new( - OrderActor, - PaymentActor, - InventoryProjector, - backend: Sourced.config.backend -) - -cmd = PlaceOrder.new(stream_id: 'order-1', payload: { amount: 100 }) -results = unit.handle(cmd) +db = Sequel.sqlite('my_app.db') +store = Sourced::Store.new(db) +store.install! ``` -Messages produced by one reactor are immediately routed to any other reactor in the unit that handles them — no background workers needed. - -#### Extracting results +#### Exporting a Sequel migration -`Unit#handle` returns a `Results` object you can query per reactor class. +Use `Store#copy_migration_to` to generate a migration file compatible with `Sequel::Migrator`: ```ruby -results = unit.handle(cmd) +db = Sequel.sqlite('my_app.db') +store = Sourced::Store.new(db) -# Hash of { instance => [events] } for a given reactor -results[OrderActor].each do |instance, events| - puts "#{instance.id}: #{events.map(&:type)}" -end +# Option 1: pass a directory (uses a default filename) +store.copy_migration_to('db/migrations') -# Flat list of events -results.events_for(OrderActor) -# => [OrderPlaced, ...] +# Option 2: pass a block for full control over the path +store.copy_migration_to do + "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_sourced_tables.rb" +end ``` -#### Skipping command persistence +Then run your migrations as usual: -By default every message (commands and events) is written to the store. Pass `persist_commands: false` to write only events. - -```ruby -unit = Sourced::Unit.new(OrderActor, backend: backend, persist_commands: false) -unit.handle(cmd) # only events are persisted; commands still flow through the chain +```bash +sequel -m db/migrations sqlite://my_app.db ``` -This is useful when commands are transient intents that don't need an audit trail. - -#### Loop detection +#### Custom table prefix -The BFS traversal is capped at 100 iterations by default. If a reaction dispatches a command whose event triggers the same reaction, the unit raises `Sourced::Unit::InfiniteLoopError` before running away. +By default, tables are prefixed with `sourced_` (e.g. `sourced_messages`, `sourced_consumer_groups`). Pass a `prefix:` to `Store.new` to customise this — for example when running multiple Sourced stores in the same database: ```ruby -unit = Sourced::Unit.new(LoopyActor, backend: backend, max_iterations: 10) -unit.handle(cmd) -# => raises Sourced::Unit::InfiniteLoopError after 10 steps +store = Sourced::Store.new(db, prefix: 'billing') +store.install! +# Creates: billing_messages, billing_key_pairs, billing_consumer_groups, ... ``` -#### ACK tracking - -After the BFS completes, the unit ACKs every handled message for each reactor's consumer group. This means background workers won't re-process messages that the unit already handled. - -#### When to use Unit vs. background workers - -| | `Sourced::Unit` | Background workers | -|---|---|---| -| Consistency | Immediate (single transaction) | Eventual | -| Failure mode | All-or-nothing rollback | Per-message retry / stop | -| Concurrency | Single-threaded BFS | Concurrent across streams | -| Use case | Web request handlers, tests, scripts | Long-running workflows, side-effects | - -You can combine both: use a Unit for the synchronous core of a request, while other reactors pick up the same events asynchronously in the background. - -### Actions - -Actions are the return values of reactor `.handle` methods. They tell the runtime (Unit or background worker) what side-effects to perform. Each persistable action class implements an `#execute(backend, source_message)` method that correlates messages and persists them via the backend. +The prefix is carried through to exported migrations automatically. -| Action | Description | `#execute` behaviour | -|---|---|---| -| `Actions::AppendAfter` | Append to a stream with optimistic locking (sequence check) | Correlate → `backend.append_to_stream` | -| `Actions::AppendNext` | Append to stream(s), auto-incrementing sequence | Correlate → `backend.append_next_to_stream` per stream | -| `Actions::Schedule` | Schedule messages for future delivery | Correlate → `backend.schedule_messages` | -| `Actions::Sync` | Run a synchronous side-effect (cache write, API call) | Call the work block, return nil | -| `Actions::OK` | Acknowledge the message (no persistence) | — | -| `Actions::RETRY` | Tell the runtime to retry this message later | — | -| `Actions::Ack` | ACK an arbitrary message by ID | — | +#### Using the Installer directly -`OK`, `RETRY`, and `Ack` are caller-specific signals — they don't implement `#execute`. +The installer is also available as a standalone object, which is useful for Rake tasks or setup scripts: ```ruby -# Inside a reactor's .handle method: -def self.handle(message) - started = Order::Started.build(message.stream_id) - [Sourced::Actions::AppendNext.new([started])] -end +installer = Sourced::Installer.new(db, logger: Logger.new($stdout), prefix: 'sourced') +installer.install # create tables +installer.installed? # check if tables exist +installer.uninstall # drop tables (test env only) +installer.copy_migration_to('db/migrations') ``` -## Projectors - -Projectors react to events published by actors and update views, search indices, caches, or other representations of current state useful to the app. They can both react to events as they happen in the system, and also "catch up" to past events. Sourced keeps track of where in the global event stream each projector is. - -From the outside-in, projectors are classes that implement the _Reactor interface_. - -Sourced ships with two ready-to-use projectors, but you can also build your own. - -### State-stored projector +## Deciders -A state-stored projector fetches initial state from storage somewhere (DB, files, API), and then after reacting to events and updating state, it can save it back to the same or different storage. +Deciders handle commands, enforce invariants, and produce events. They rebuild state from event history before each decision. ```ruby -class CartListings < Sourced::Projector::StateStored - # Fetch listing record from DB, or new one. - state do |id| - CartListing.find_or_initialize(id) - end - - # Evolve listing record from events - event Carts::ItemAdded do |listing, event| - listing.total += event.payload.price - end +class CourseDecider < Sourced::Decider + # Defines the consistency boundary + partition_by :course_name - # Sync listing record back to DB - sync do |state:, events:, replaying:| - state.save! + # Initial state factory (receives partition values hash) + state do |_partition_values| + { name_taken: false } end -end -``` - -### Event-sourced projector -An event-sourced projector fetches initial state from past events in the event store, and then after reacting to events and updating state, it can save it to a DB table, a file, etc. - -```ruby -class CartListings < Sourced::Projector::EventSourced - # Initial in-memory state - state do |id| - { id:, total: 0 } + # Evolve state from events (rebuilds history) + evolve CourseCreated do |state, _event| + state[:name_taken] = true end - # Evolve listing record from events - event Carts::ItemAdded do |listing, event| - listing[:total] += event.payload.price - end + # Command handler — enforce invariants, then produce events + command CreateCourse do |state, cmd| + raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] - # Sync listing record to a file - sync do |state:, events:, replaying:| - File.write("/listings/#{state[:id]}.json", JSON.dump(state)) + event CourseCreated, + course_id: cmd.payload.course_id, + course_name: cmd.payload.course_name end end ``` -### Registering projectors - -Like any other _reactor_, projectors need to be registered for background workers to route events to them. - -```ruby -# In your app's configuration -Sourced.register(CartListings) -``` - -### Reacting to events and scheduling the next command from projectors - -Sourced projectors can define `.reaction` handlers that will be called after evolving state via their `.event` handlers, in the same transaction. - -This can be useful to implement TODO List patterns where a projector persists projected data, and then reacts to the data update using the data to schedule the next command in a workflow. - -![CleanShot 2025-05-30 at 18 43 01](https://github.com/user-attachments/assets/ef8a61b7-6b99-49a1-9767-af94b9c2c4e2) +### Synchronous command handling +`Sourced.handle!` loads history, runs the decider, appends the command + events, and advances consumer group offsets — all in one call. Designed for web controllers. ```ruby -class ReadyOrders < Sourced::Projector::StateStored - # Fetch listing record from DB, or new one. - state do |id| - OrderListing.find_or_initialize(id) - end +cmd = CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) +cmd, decider, events = Sourced.handle!(CourseDecider, cmd) - event Orders::ItemAdded do |listing, event| - listing.line_items << event.payload - end - - # Evolve listing record from events - event Orders::PaymentConfirmed do |listing, event| - listing.payment_confirmed = true - end - - event Orders::BuildConfirmed do |listing, event| - listing.build_confirmed = true - end - - # Sync listing record back to DB - sync do |state:, events:, replaying:| - state.save! - end - - # If a listing has both the build and payment confirmed, - # automate dispatching the next command in the workflow - reaction do |listing, event| - if listing.payment_confirmed? && listing.build_confirmed? - dispatch Orders::Release, **listing.attributes - end - end +if cmd.valid? + # Success — events were appended +else + # Validation failure — cmd.errors has details end ``` -Projectors can also define `.reaction event_class do |state, event|` to react to specific events, or `reaction event1, event2` to react to more than one event with the same block. - -### Skipping projector reactions when replaying events - -When a projector's offsets are reset (so that it starts re-processing events and re- building projections), Sourced skips invoking a projector's `.reaction` handlers. This is because building projections should be deterministic, and rebuilding them should not trigger side-effects such as automations (we don't want to call 3rd party APIs, send emails, or just dispatch the same commands over and over when rebuilding projections). - -To do this, Sourced keeps track of each consumer groups' highest acknowledged event sequence. When a consumer group is reset and starts re-processing past events, this sequence number is compared with each event's sequence, which tells us whether the event has been processed before. - -## Concurrency model - -Concurrency in Sourced is achieved by explicitely _modeling it in_. +Raises `Sourced::ConcurrentAppendError` on conflicts, or `RuntimeError` on domain invariant violations (e.g. "Course already exists"). -Sourced workers process messages by acquiring locks on `[reactor group ID][stream ID]`. For example `"CartActor:cart-123"` +### CommandContext -This means that all events for a given reactor/stream are processed in order, but events for different streams can be processed concurrently. You can define workflows where some work is done concurrently by modeling them as a collaboration of streams. - -### Single-stream sequential execution - -In the following (simplified!) example, a Holiday Booking workflow is modelled as a single stream ("Actor"). The infrastructure makes sure these steps are run sequentially. - -sourced-concurrency-single-lane - -The Actor glues its steps together by reacting to events emitted by the previous step, and dispatching the next command. +`Sourced::CommandContext` is a factory for building Sourced commands from raw Hash attributes (e.g. HTTP params), injecting defaults like `metadata`. It mirrors `Sourced::CommandContext` but without `stream_id`, since Sourced messages are stream-less. ```ruby -class HolidayBooking < Sourced::Actor - # State and details omitted... - - command :start_booking do |state, cmd| - event :booking_started - end - - reaction :booking_started do |event| - dispatch :book_flight - end - - command :book_flight do |state, cmd| - event :flght_booked - end - - reaction :flight_booked do |event| - dispatch :book_hotel - end - - command :book_hotel do |state, cmd| - event :hotel_booked - end - - # Define event handlers if you haven't... - event :booking_started, # ..etc - event :flight_booked, # ..etc -end -``` - -### Multi-stream concurrent execution - -In this other example, the same workflow is split into separate streams/actors, so that Flight and Hotel bookings can run concurrently from each other. When completed, they each notify the parent Holiday actor, so the whole process coalesces into a sequential operation again. +# In a web controller, build a context with shared metadata +ctx = Sourced::CommandContext.new( + metadata: { user_id: session[:user_id] } +) -sourced-concurrency-multi-lane +# Build from a type string + payload hash (e.g. from JSON params) +cmd = ctx.build(type: 'courses.create', payload: { course_id: 'c1', course_name: 'Algebra' }) +cmd.metadata[:user_id] # => session[:user_id] -```ruby -# An actor dispatches a message to different stream -# messages for different streams are processed concurrently -reaction BookingStarted do |state, event| - dispatch(BookHotel).to("#{event.stream_id}-hotel") -end +# Or pass an explicit command class +cmd = ctx.build(CreateCourse, payload: { course_id: 'c1', course_name: 'Algebra' }) ``` -### Units of work - -CleanShot 2025-11-15 at 14 38 05 +String keys are automatically symbolized, so `ctx.build('type' => '...', 'payload' => { ... })` works too. -The diagram shows the units of work in an example Sourced workflow. The operations within each of the red boxes are protected by a combination of transactions and locking strategies on the consumer group + stream ID, so they are isolated from other concurrent processing. They can be said to be **immediately consistent**. -The data-flow _between_ these boxes is propagated asynchronously by Sourced's infrastructure so, relative to each other, the entire system is **eventually consistent**. +#### Callback hooks (`on` and `any`) -These transactional boundaries are guarded by the same locks that enforce the concurrency model, so that for example the same message can't be processed twice by the same Reactor (workflow, projector, etc). +Subclass `CommandContext` and register class-level hooks to enrich or transform commands at build time — e.g. injecting session data or adding metadata from the request scope. -## Durable workflows +- **`on(MessageClass, ...)`** — runs for one or more command types. Multiple `on` calls for the same class accumulate (all blocks run in registration order). +- **`any`** — runs for all commands (multiple blocks allowed, executed in order) -There's a `Sourced::DurableWorkflow` class that can be subclassed to define Reactors with a synchronous-looking API. This is *work in progress*. +Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. Blocks are evaluated in the context of the `CommandContext` instance, so they can call private helper methods defined on the subclass. ```ruby -class BookHoliday < Sourced::DurableWorkflow - # This method can be called like a regular method - # The methods inside also have blocking semantics - # but they're in fact event-sourced, and will be - # retried on failure until the booking completes. - # Methods that were succesful will be idempotent on retry - def execute(flight_info, hotel_info) - flight = book_flight(flight_info) - hotel = book_hotel(hotel_info) - confirm_booking(flight, hotel) +class AppCommandContext < Sourced::CommandContext + # Enrich a specific command with data from the app scope + on CreateCourse do |app, cmd| + cmd.with_payload(created_by: app.current_user.id) end - - # The .durable macro turns a regular method - # into an event-sourced workflow - durable def book_flight(info) - FlightsAPI.book(info) + + # Same block for multiple command types + on EnrolStudent, DropStudent do |app, cmd| + cmd.with_metadata(campus: app.current_campus) end - - durable def book_hotel(info) - HotelsAPI.book(info) + + # Additional block for EnrolStudent — both blocks run in order + on EnrolStudent do |app, cmd| + cmd.with_metadata(enrolment_source: 'web') end - - durable def confirm_booking(flight, hotel) - # etc, + + # Add metadata to every command + any do |app, cmd| + cmd.with_metadata( + request_id: app.request_id, + session_id: app.session_id + ) end end ``` -These executions will be handed off to the runtime to be run by one or more workers, while preserving ordering. You can optionally wait for a result. +Pass the request-scoped `app` object at construction time: ```ruby -result = BookHoliday.execute(flight_info, hotel_info).wait.output -# Confirmed booking, or whatever error result your code returns -``` - -Events for the full execution are recorded to the backend. -CleanShot 2025-11-13 at 13 48 27@2x - -Durable workflows must be registered with the runtime, like any other Reactor. +# In a web controller +ctx = AppCommandContext.new( + metadata: { user_id: session[:user_id] }, + app: self # e.g. Sinatra app instance, Rack env wrapper, etc. +) -```ruby -Sourced.register BookHoliday +cmd = ctx.build(type: 'courses.create', payload: { course_id: 'c1', course_name: 'Algebra' }) +cmd.metadata[:request_id] # => set by the `any` hook ``` -## Handler DSL +`app` defaults to `nil`, so existing callers without hooks are unaffected. Hooks are inherited by subclasses. -The `Sourced::Handler` mixin provides a lighter-weight DSL for simple reactors. +Since blocks run in instance context, you can extract shared logic into private methods: ```ruby -class OrderTelemetry - include Sourced::Handler - - # Handle these Order events - # and log them - on Order::Started do |event| - Logger.info ['order started', event.stream_id] - [] - end - - on Order::Placed do |event| - Logger.info ['order placed', event.stream_id] - [] +class AppCommandContext < Sourced::CommandContext + on CreateCourse do |app, cmd| + cmd.with_metadata(user_id: build_user_id(app)) end -end - -# Register it -Sourced.register OrderTelemetry -``` -Handlers can optionally define the `:history` argument. The runtime will provide the full message history for the stream ID being handled. + private -```ruby -on Order::Placed do |event, history:| - total = history - .filter { |e| Order::ProductAdded === e } - .reduce(0) { |n, e| n + e.payload.price } - - if total > 10000 - return [Order::AddDiscount.build(event.stream_id, amount: 100)] + def build_user_id(app) + "user-#{app.session_id}" end - - [] -end -``` - -It also supports multiple event types, for generic handling. - -```ruby -on Order::Placed, Order::Complete do |event| - Logger.info "received event #{event.inspect}" - [] end ``` -## Command methods for Actors +#### Scoping to a command subset -The optional `Sourced::CommandMethods` mixin allows invoking an Actor's commands as regular methods. - -`CommandMethods` automatically generates instance methods from command definitions, -allowing you to invoke commands in two ways: - -1. **In-memory version** (e.g., `actor.start(name: 'Joe')`) - - Validates the command and executes the decision handler - - Returns a tuple of [cmd, new_events] - - Does NOT persist events to backend -2. **Durable version** (e.g., `actor.start!(name: 'Joe')`) - - Same as in-memory, but also appends events to backend - - Raises `FailedToAppendMessagesError` if backend fails - -Include the module in an Actor and define commands normally: +By default, `CommandContext` looks up types in `Sourced::Command.registry`. Pass a `scope:` to restrict lookups to a specific command subclass — attempts to build commands outside the scope raise `Sourced::UnknownMessageError`. ```ruby -class MyActor < Sourced::Actor - include Sourced::CommandMethods +class PublicCommand < Sourced::Command; end - command :create_item, name: String do |state, cmd| - event :item_created, cmd.payload - end +CreateCourse = PublicCommand.define('courses.create') do + attribute :course_id, String + attribute :course_name, String end -actor = MyActor.new(id: 'actor-123') -cmd, events = actor.create_item(name: 'Widget') # In-memory -cmd, events = actor.create_item!(name: 'Widget') # Persists to backend -``` - - - -## Orchestration and choreography - -### Orchestration - -Orchestration is when the flow control of a multi-collaborator workflow is centralised into a single entity. This can be achieved by having one Actor coordinate the communication by reacting to events and sending commands to other actors. - -```ruby -class HolidayBooking < Sourced::Actor - state do |id| - BookingState.new(id) - end - - command StartBooking do |booking, cmd| - # validations, etc - event BookingStarted, cmd.payload - end - - event BookingStarted - - # React to BookingStarted and start sub-workflows - reaction BookingStarted do |booking, event| - dispatch(HotelBooking::Start) - end - - # React to events emitted by sub-workflows - reaction HotelBooking::Started do |booking, event| - dispatch(ConfirmHotelBooking, event.payload) - end - - command ConfirmHotelBooking do |booking, cmd| - unless booking.hotel.booked? - event HotelBookingConfirmed, cmd.payload - end - end - - event HotelBookingConfirmed do |booking, event| - # update booking state - booking.confirm_hotel(event.payload) - end -end +# Only PublicCommand subclasses are allowed +ctx = Sourced::CommandContext.new(scope: PublicCommand) +ctx.build(type: 'courses.create', payload: { ... }) # OK +ctx.build(type: 'admin.delete_all', payload: {}) # raises UnknownMessageError ``` -This is a verbose step-by-step choreography, but it can be made more succint by ommiting the mirroring of commands/events, if needed (or by using the [Reactor Interface](#the-reactor-interface) directly). - -*TODO*: a way for Actors to initialise their internal state with event attributes other than the `stream_id`. For example, events may carry a `booking_id` for the overall workflow. - -### Choreography - -Choreography is when each component reacts to other components' events without centralised control. The overall workflow "emerges" from this collaboration. +### Loading a decider's state ```ruby -class HotelBooking < Sourced::Actor - # The HotelBooking defines its own - # reactions to booking events - reaction HolidayBooking::StartBooking do |state, event| - # dispatch a command to itself to start its own life-cycle - dispatch Start, event.payload - end - - command Start do |state, cmd| - # validations, etc - # other Actors in the choreography - # can choose to react to events emitted here - event Started, cmd.payload - end - - event Started do |state, event| - # update state, etc - end -end +decider, read_result = Sourced.load(CourseDecider, course_name: 'Algebra') +decider.state # => { name_taken: true } ``` -## Appending and reading messages - -### Appending messages without optimistic locking - -Use `Backend#append_next_to_stream` to append messages to a stream, with no questions asked. - -```ruby -message = ProductAdded.build('order-123', product_id: 123, price: 100) -Sourced.config.backend.append_next_to_stream('order-123', [message]) +## Projectors -# Shortcut: -Sourced.dispatch(message) -``` +Projectors consume events to build read models. Two flavours: -### Appending messages with optimistic locking +### EventSourced projector -Using `Backend#append_to_stream`, the backend expects the new messages `seq` property (sequence number) to be greater than the last message in storage for the same stream. This is to catch concurrent writes where a different client or thread may append to the stream while your code was preparing for it. +Rebuilds state from full history on every batch (like deciders). ```ruby -# Your code must make sure to increment sequence numbers -past_events = Sourced.config.backend.read_stream('order-123') -last_known_seq = past_events.last&.seq # ex. 10 -# Instantiate new messages and make sure to increment their sequences -message = ProductAdded.new( - stream_id: 'order-123', - seq: last_known_seq + 1, # <== incremented sequence - payload: { product_id: 123, price: 100 } -) - -# This will raise an exception if there's already a message -# for this stream with this sequence number in storage. -Sourced.backend.append_to_stream('order-123', [message]) -``` - -`Sourced::Actor` classes do this incrementing automatically when they produce new messages. +class CourseCatalogProjector < Sourced::Projector::EventSourced + partition_by :course_id -### Scheduling messages in the future + state do |_partition_values| + { course_id: nil, course_name: nil, students: [] } + end -You can append messages to a separate log, with a schedule time. Sourced workers will periodically poll this log and move these messages into the main log at the right time. + evolve CourseCreated do |state, event| + state[:course_id] = event.payload.course_id + state[:course_name] = event.payload.course_name + end -```ruby -message = ProductAdded.build('order-123', product_id: 123, price: 100) -Sourced.config.backend.schedule_messages([message], at: Time.now + 20) -``` + evolve StudentEnrolled do |state, event| + state[:students] << event.payload.student_id + end -Actor reactions can use the `#dispatch` and `#at` helpers to schedule commands to run at a future time. + # Sync block runs within the store transaction after evolving + sync do |state:, messages:, **| + next unless state[:course_id] + # Write projection to disk, database, cache, etc. + File.write("projections/#{state[:course_id]}.json", state.to_json) + end -```ruby -reaction ProductAdded do |order, event| - dispatch(NotifyNewProduct).at(Time.now + 20) + # After-sync block runs after the transaction commits. + # Use for side effects that should only happen on successful commit + # (e.g. sending emails, HTTP calls, pushing to external queues). + after_sync do |state:, messages:, **| + NotificationService.notify("Course #{state[:course_name]} updated") + end end ``` -## Replaying messages +### StateStored projector -You can use the backend API to reset offsets for a specific consumer group, which will cause workers to start replaying messages for that group. +Loads persisted state via the `state` block, evolves only new (unprocessed) messages. ```ruby -Sourced.config.backend.reset_consumer_group(ReadyOrder) -``` +class MyProjector < Sourced::Projector::StateStored + partition_by :course_id -See [below](#stopping-and-starting-consumer-groups) for other consumer lifecycle methods. - -## The Reactor Interface - -All built-in Reactors (Actors, Projections) build on the low-level Reactor Interface. + state do |partition_values| + # Load existing state from your storage + existing = MyDB.find(partition_values[:course_id]) + existing || { course_id: nil, students: [] } + end -The runtime invokes `.handle_batch` on each reactor, passing a batch of `[message, replaying]` pairs from the same stream. The `Sourced::Consumer` module provides a default `handle_batch` that delegates to a per-message `.handle` method, so simple reactors only need to implement `.handle`. + evolve StudentEnrolled do |state, event| + state[:students] << event.payload.student_id + end -```ruby -class MyReactor - extend Sourced::Consumer - - # The runtime will poll and hand over messages of this type - def self.handled_messages = [Order::Started, Order::Placed] - - # The default handle_batch (from Consumer) calls this per message. - # Return an Array of one or more Actions. - def self.handle(new_message) - actions = [] - - # Just acknowledge new_message - actions << Sourced::Actions::OK - - # Append these new messages to the event store - # Sourced will automatically increment the stream's sequence number - # (ie. no optimistic locking) - started = Order::Started.build(new_message.stream_id) - actions << Sourced::Actions::AppendNext.new([started]) - - # Append these new messages to the event store. - # The messages are expected to have a :seq incremented after new_message.seq - # Messages will fail to append if other messages have been appended - # with overlapping sequence numbers (optimistic locking) - started = Order::Started.new(stream_id: new_message.stream_id, seq: new_message.seq + 1) - actions << Sourced::Actions::AppendAfter.new(new_message.stream_id, [started]) - - # Tell the runtime to retry this message - actions << Sourced::Actions::RETRY - - actions + sync do |state:, messages:, **| + MyDB.upsert(state) end end ``` -You can implement your own low-level reactors following the interface above. Then register them as normal. - -```ruby -Sourced.register MyReactor -``` - -### Batch processing +## Reactions -Workers fetch up to `worker_batch_size` messages per lock cycle (default: 50). Built-in reactors optimize batch processing automatically: - -- **Projector::StateStored**: loads state once, evolves all batch messages, syncs once (instead of N state loads + N syncs). -- **Projector::EventSourced**: evolves history once, syncs once (instead of N x O(H) evolves + N syncs). -- **Actor**: replaying messages return OK immediately; live messages are handled individually. - -For custom reactors, you can override `handle_batch` directly for full control: +Both deciders and projectors can react to events to produce new commands or events, enabling workflow orchestration. ```ruby -class MyBatchReactor - extend Sourced::Consumer +class EnrolmentDecider < Sourced::Decider + partition_by :course_id - def self.handled_messages = [Order::Started, Order::Placed] + # ... evolve and command handlers ... - # Override handle_batch for custom batch processing. - # Must return Array of [actions, source_message] pairs. - def self.handle_batch(batch) - batch.map do |message, replaying| - actions = process(message) - [actions, message] - end + # React to an event by producing new messages + reaction StudentEnrolled do |state, event| + NotifyStudent.new(payload: { student_id: event.payload.student_id }) end end ``` -Individual reactors can override the global `worker_batch_size` via the `consumer` DSL: - -```ruby -class OrderProjector < Sourced::Projector::StateStored - consumer do |c| - c.batch_size = 100 - end -end -``` +Reactions are skipped during replay (when `replaying: true`), so side effects don't re-fire. -When set, the reactor's `batch_size` takes precedence over the global `worker_batch_size`. When not set (default), the global value is used. +## Sync and After-Sync Blocks -#### Partial ACK on failure +Both deciders and projectors support `sync` and `after_sync` blocks for running side effects during message processing. -When a message raises mid-batch, Sourced ACKs up to the last successfully processed message instead of retrying the entire batch. The failed message and any subsequent messages are retried in the next batch. This is handled automatically by `each_with_partial_ack` in the Consumer module, which all built-in reactor types use. +- **`sync`** blocks run **inside** the store transaction, alongside event persistence and offset acknowledgement. Use them for writes that must be atomic with the event append (e.g. updating a database projection). +- **`after_sync`** blocks run **after** the transaction commits. Use them for side effects that should only happen if the commit succeeds (e.g. sending emails, HTTP calls, pushing to external queues). -**Actors and plain Consumer reactors** process each message independently (a new instance per message), so partial ACK is straightforward and safe with any batch size. +Both receive the same keyword arguments as the reactor's action-building step: -**Projectors** use an evolve-all-sync-once optimization: all batch messages are evolved into a single instance's state, then synced once. Reactions are processed one by one — if a reaction fails mid-batch, all previously successful messages (including their reactions and a correct partial sync) are ACKed, and only the failed message onward is retried. On partial failure, the projector automatically rebuilds a fresh instance with only the successfully processed messages to produce a correct sync (via the `on_partial_sync` block in `sync_and_react`). +| Reactor type | Keyword arguments | +|--------------|---------------------------------------| +| Decider | `state:`, `messages:`, `events:` | +| Projector | `state:`, `messages:`, `replaying:` | ```ruby -class PaymentProcessor < Sourced::Projector::StateStored - consumer do |c| - c.batch_size = 10 +class OrderDecider < Sourced::Decider + partition_by :order_id + + # ... evolve / command handlers ... + + sync do |state:, messages:, events:| + # Runs inside the transaction + OrderCache.update(state[:order_id], state) end - reaction PaymentStarted do |state, evt| - # If this HTTP call succeeds for messages 1-3 but fails on message 4, - # messages 1-3 are fully ACKed (reactions + sync). Message 4 onward is retried. - response = PaymentGateway.post(:process_payment, state[:payment_info]) - if response.ok? - dispatch ConfirmPayment, payment_id: response.body[:payment_id] - else - dispatch RejectPayment, errors: response.body[:errors] - end + after_sync do |state:, messages:, events:| + # Runs after successful commit + Mailer.send_confirmation(state[:order_id]) if events.any? { |e| e.is_a?(OrderPlaced) } end end ``` -### Reactors that require message history - -Reactors that declare the `:history` keyword in their `.handle_batch` (or `.handle`) signature will be provided the full message history for the stream being handled. +Multiple `sync` and `after_sync` blocks can be registered; they execute in registration order. Blocks are inherited by subclasses. -This is how event-sourced Actors are implemented. +## Configuration ```ruby -def self.handle(new_message, history:) - # evolve state from history, - # handle command, return new events, etc - [] -end -``` - -### `:replaying` flag +require 'sourced' -Your `.handle` method can also declare a `:replaying` boolean, which tells the reactor whether the stream is replaying events, or handling new messages. Reactors use this to run or omit side-effects (for example, replaying Projectors don't run `reaction` blocks). +Sourced.configure do |c| + # Pass a Sequel SQLite connection or a Sourced::Store instance + c.store = Sequel.sqlite('my_app.db') -```ruby -def self.handle(new_message, history:, replaying:) - if replaying - # Omit side-effects - else - # Trigger side-effects - end + # Optional settings + c.worker_count = 4 # background worker fibers (default: 2) + c.batch_size = 50 # messages per claim (default: 50) + c.catchup_interval = 5 # seconds between catch-up polls (default: 5) + c.max_drain_rounds = 10 # max drain iterations per pickup (default: 10) + c.claim_ttl_seconds = 120 # stale claim threshold (default: 120) + c.housekeeping_interval = 30 # heartbeat/reap cycle (default: 30) end ``` -## Testing +## Failure handling and retries -There's a couple of experimental RSpec helpers that allow testing Sourced reactors in GIVEN, WHEN, THEN style. +Sourced already supports consumer-group retries on failure. -*GIVEN* existing events A, B, C -WHEN new command D is sent -THEN I expect new events E and F +- On reactor errors, `Router#handle_next_for` calls the reactor's `on_exception` hook. +- By default, that hook uses `Sourced.config.error_strategy`. +- The default `Sourced::ErrorStrategy` marks the consumer group as failed immediately. +- If you configure a retrying error strategy, Sourced stores the next retry time in the consumer group's `retry_at` column and skips claiming work for that group until that time has passed. -### Single reactor +So retries are built in already, but they are opt-in via the error strategy configuration. -Use `with_reactor` to unit-test the life-cycle of a single reactor. +### Example: exponential backoff retries ```ruby -require 'sourced/testing/rspec' +require 'sourced' -RSpec.describe Order do - include Sourced::Testing::RSpec +Sourced.configure do |c| + c.store = Sequel.sqlite('my_app.db') - it 'adds product to order' do - with_reactor(Order, 'order-123') - .when(Order::AddProduct, product_id: 1, price: 100) - .then(Order::ProductAdded.build('order-123', product_id: 1, price: 100)) - end + c.error_strategy = Sourced::ErrorStrategy.new do |s| + s.retry( + times: 5, + after: 2, + backoff: ->(retry_after, retry_count) { retry_after * (2**(retry_count - 1)) } + ) - it 'is a noop if product already in order' do - with_reactor(Order, 'order-123') - .given(Order::ProductAdded, product_id: 1, price: 100) - .when(Order::AddProduct, product_id: 1, price: 100) - .then([]) + s.on_retry do |retry_count, exception, message, later| + LOGGER.warn( + "Sourced retry ##{retry_count} for #{message.type} (#{message.id}) " \ + "at #{later}: #{exception.class}: #{exception.message}" + ) + end + + s.on_fail do |exception, message| + LOGGER.error( + "Sourced failing consumer group after retries for #{message.type} (#{message.id}): " \ + "#{exception.class}: #{exception.message}" + ) + end end end ``` -`#then` can also take a block, which will be given the low level `Sourced::Actions` objects returned by your `.handle()` interface. +With the configuration above, failures retry after: -You can use this block to test reactors that trigger side effects. +- retry 1: 2 seconds +- retry 2: 4 seconds +- retry 3: 8 seconds +- retry 4: 16 seconds +- retry 5: 32 seconds -```ruby -with_reactor(Webhooks, 'webhook-1') - .when(Webooks::Dispatch, name: 'Joe') - .then do |actions| - expect(api_request).to have_been_requested - end -``` +After the configured retries are exhausted, the consumer group is marked as failed. -You can mix argument and block assertions with `.then()` +## Registering reactors ```ruby -with_reactor(Webhooks, 'webhook-1') - .when(Webooks::Dispatch, name: 'Joe') - .then do |_| - expect(api_request).to have_been_requested - end - .then(Webhooks::Dispatched, reference: 'webhook-abc') +Sourced.register(CourseDecider) +Sourced.register(EnrolmentDecider) +Sourced.register(CourseCatalogProjector) ``` -For reactors that have `sync` blocks for side-effects (ex. Projectors), use `#then!` to trigger those side-effects and assert their results. +This registers the reactor's consumer group with the store and adds it to the global router. -```ruby -with_reactor(PlacedOrders, 'order-123') - .given(Order::Started) - .given(Order::ProductAdded, product_id: 1, price: 100, units: 2) - .given(Order::Placed) - .then! do |_| - expect(OrderRecord.find('order-123').total).to eq(200) - end -``` +## Background processing -### Testing exceptions +### Falcon (recommended) -`#then` also accepts an exception class or instance, to assert that a command handler raises a specific error. +`Sourced::Falcon` provides a ready-made Falcon service that runs both the web server and Sourced background workers as sibling fibers. No separate worker process needed. ```ruby -# Match on exception class -with_reactor(Order, 'order-123') - .given(Order::Placed) - .when(Order::Place) - .then(Order::AlreadyPlacedError) - -# Match on exception class and message -with_reactor(Order, 'order-123') - .given(Order::Placed) - .when(Order::Place) - .then(Order::AlreadyPlacedError.new('order already placed')) -``` - -When given an exception class, the test passes if any error of that type is raised. When given an exception instance, it also checks that the error message matches. - -### Multiple reactors (A.K.A "Sagas") +# falcon.rb +#!/usr/bin/env falcon-host +require_relative 'domain' +require_relative 'app' +require 'sourced/falcon' -Use `with_reactors` to test the collaboration of multiple reactors sending and picking up eachother's messages. +service "my-app" do + include Sourced::Falcon::Environment + include Falcon::Environment::Rackup -```ruby -it 'tests collaboration of reactors' do - order_stream = 'actor-1' - payment_stream = 'actor-1-payment' - telemetry_stream = Testing::Telemetry::STREAM_ID - - # With these reactors - with_reactors(Order, Payment, Telemetry) - # GIVEN that these events exist in history - .given(Order::Started.build(order_stream, name: 'foo')) - # WHEN I dispatch this new command - .when(Order::StartPayment.build(order_stream)) - # Then I expect - .then do |stage| - # The different reactors collaborated and - # left this message trail behind - # Backend#messages is only available in the TestBackend - expect(stage.backend.messages).to match_sourced_messages([ - Order::Started.build(order_stream, name: 'foo'), - Order::StartPayment.build(order_stream), - Order::PaymentStarted.build(order_stream), - Telemetry::Logged.build(telemetry_stream, source_stream: order_stream), - Payment::Process.build(payment_stream), - Payment::Processed.build(payment_stream), - Telemetry::Logged.build(telemetry_stream, source_stream: payment_stream), - ]) - end + url "http://localhost:9292" + count 1 end ``` -`with_reactors` sets up its own in-memory backend, so you can test multi-reactor workflows in terms of what messages they produce without database or network requests, and there's no need for database setup or tear-down. Just test the behaviour! +Start with: -The `.then` block can take an optional second argument, which will be passed as only the _new_ messages produced by the reactors, appended after any messages setup with `given`. - -```ruby -.then do |stage, new_messages| - expect(new_messages).to match_sourced_messages([...]) -end +```bash +bundle exec falcon host ``` +The service automatically calls `Sourced.setup!` in each forked process, which replays the `Sourced.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe. +#### How it works -## Setup +- `Sourced::Falcon::Environment` — mixin that sets the `service_class` to `Sourced::Falcon::Service`. Include it in your Falcon service definition alongside `Falcon::Environment::Rackup`. +- `Sourced::Falcon::Service` — extends `Falcon::Service::Server`. On `run`, it calls `Sourced.setup!`, starts the web server, and spawns a `Sourced::Dispatcher` with all settings from `Sourced.config`. On `stop`, it shuts down the dispatcher before the server. +- No separate HouseKeeper fibers are needed — the `StaleClaimReaper` is embedded in the Sourced Dispatcher. -Sourced uses the Sequel gem for database access. It supports **PostgreSQL** (recommended for production) and **SQLite** (useful for development, scripts, and single-process apps). +### Supervisor (standalone) -### PostgreSQL - -You'll need the `pg` and `sequel` gems. - -```ruby -gem 'sourced', github: 'ismasan/sourced' -gem 'pg' -gem 'sequel' -``` - -Create a Postgres database and configure the backend. +For running workers without a web server, the supervisor starts workers that claim partitions, process messages, and ack offsets. ```ruby -Sourced.configure do |config| - config.backend = Sequel.connect(ENV.fetch('DATABASE_URL')) - - # Worker and housekeeping options (shown with defaults) - config.worker_count = 2 # Number of worker fibers - config.worker_batch_size = 50 # Messages fetched per lock cycle (batch processing) - config.housekeeping_count = 1 # Number of housekeeper fibers - config.housekeeping_interval = 3 # Seconds between scheduling cycles - config.housekeeping_heartbeat_interval = 5 # Seconds between worker heartbeats - config.housekeeping_claim_ttl_seconds = 120 # Seconds before stale claims are reaped -end +# Start blocking (handles INT/TERM signals for graceful shutdown) +Sourced::Supervisor.start -Sourced.config.backend.install unless Sourced.config.backend.installed? +# Or create and start manually +supervisor = Sourced::Supervisor.new( + router: Sourced.router, + count: 4 +) +supervisor.start ``` -Passing a `Sequel::Postgres::Database` connection auto-selects `PGBackend`, which supports PG-specific features: `LISTEN/NOTIFY` for real-time worker dispatch, advisory locks, and `FOR UPDATE SKIP LOCKED` for concurrent stream processing. - -These options are used by both `Sourced::Supervisor` and the Falcon integration. When running workers alongside a web server (Falcon, or any other Async-compatible server), these control how many worker and housekeeper fibers are spawned per OS process. The `worker_batch_size` controls how many messages from the same stream are fetched and processed in a single lock cycle (see [Batch processing](#batch-processing)). - -### SQLite +### How it works -You'll need the `sqlite3` and `sequel` gems. +1. **Store** appends messages and notifies listeners of new message types +2. **Dispatcher** routes notifications to a `WorkQueue`, mapping message types to interested reactors +3. **Workers** pop reactors from the queue, claim a partition via `Router#handle_next_for`, process messages, and ack +4. **CatchUpPoller** periodically pushes all reactors as a safety net (handles missed notifications) +5. **ScheduledMessagePoller** promotes due delayed messages into the main Sourced log +6. **StaleClaimReaper** releases claims held by dead workers -```ruby -gem 'sourced', github: 'ismasan/sourced' -gem 'sqlite3' -gem 'sequel' -``` +### Router (direct usage) -Configure with a Sequel SQLite connection. +The router can also be used directly for testing or scripting: ```ruby -Sourced.configure do |config| - # File-based database - config.backend = Sequel.sqlite('myapp.db') +router = Sourced.router - # Or in-memory (useful for scripts and tests) - # config.backend = Sequel.sqlite -end +# Process one batch for a specific reactor +router.handle_next_for(CourseDecider) -Sourced.config.backend.install unless Sourced.config.backend.installed? +# Drain all pending work across all reactors +router.drain ``` -Passing a `Sequel::SQLite::Database` connection auto-selects `SQLiteBackend`. The SQLite backend sets up WAL mode and busy timeouts automatically. +## Consumer groups -**Differences from PostgreSQL:** +Each reactor class is a consumer group. The store tracks per-partition offsets so multiple reactors process the same events independently. -- **In-process pub/sub**: Uses an in-memory, thread/fiber-safe pub/sub (`PubSub::Test`) instead of PG `LISTEN/NOTIFY`. Worker dispatch is synchronous (the `InlineNotifier` pushes work to the `WorkQueue` inline when messages are appended), with the `CatchUpPoller` as a safety net. -- No advisory locks or `SKIP LOCKED` — concurrency is handled via SQLite's transaction-level write locks. -- Best suited for single-process deployments, development, scripts, and tests. - -See `examples/lite_cart.rb` for a complete working example. - -### Generating Sequel migrations - -If your app already uses Sequel's migrator, you can copy Sourced's migration into your migrations directory instead of using `backend.install`. +The lifecycle methods (`stop_consumer_group`, `start_consumer_group`, `reset_consumer_group`, `consumer_group_active?`) accept either a String group ID or any object responding to `#group_id` (e.g. a reactor class). ```ruby -backend = Sourced.config.backend -backend.copy_migration_to("db/migrations") -# => writes db/migrations/001_create_sourced_tables.rb -``` +store = Sourced.store -Or use a block to control the file name (e.g. timestamped migrations): +# Pass reactor classes directly +store.stop_consumer_group(CourseDecider) +store.start_consumer_group(CourseDecider) +store.reset_consumer_group(CourseDecider) # reprocess from beginning +store.consumer_group_active?(CourseDecider) # => true/false -```ruby -backend.copy_migration_to do - "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_sourced_tables.rb" -end +# Or use plain strings +store.stop_consumer_group('CourseApp::CourseDecider') ``` -The generated file is a standard `Sequel.migration { change { ... } }` that works with `Sequel::Migrator`. It respects the `prefix` and `schema` options passed when configuring the backend: - -```ruby -Sourced.configure do |config| - db = Sequel.connect(ENV.fetch('DATABASE_URL')) - config.backend = Sourced::Backends::SequelBackend.new(db, prefix: 'myapp', schema: 'events') -end +When retries are configured via `Sourced.config.error_strategy`, failed consumer groups remain active but paused until their `retry_at` time. Once that time passes, they become claimable again automatically. -# Migration will create tables like events.myapp_messages, events.myapp_streams, etc. -Sourced.config.backend.copy_migration_to("db/migrations") -``` +### Lifecycle hooks via Router -Register your Actors and Reactors. +The Router provides lifecycle methods that wrap the Store operations and invoke optional callbacks on the reactor class. This lets reactors run cleanup or setup logic when their consumer group is stopped, reset, or started. ```ruby -Sourced.register(Leads::Actor) -Sourced.register(Leads::Listings) -Sourced.register(Webooks::Dispatcher) -``` +# Accept a reactor class or a string group_id +Sourced.stop_consumer_group(CourseDecider, 'maintenance window') +Sourced.reset_consumer_group(CourseDecider) +Sourced.start_consumer_group(CourseDecider) -### Running workers as a separate process - -When using a web server that doesn't share Sourced's Async event loop (e.g. Puma), or in non-web applications, run workers as a standalone process using `Sourced::Supervisor`: - -```ruby -# worker.rb -require_relative 'config/environment' -# start workers with 10 worker fibers or threads per OS process -# depending on Sourced.config.executor (:async, :thread, or custom) -Sourced::Supervisor.start(count: 10) +# String group_id works too — the router resolves it to the registered class +Sourced.stop_consumer_group('CourseApp::CourseDecider') ``` -This requires managing two processes in deployment: one for your web server, one for workers. +These delegate to `Router#stop_consumer_group`, `Router#reset_consumer_group`, and `Router#start_consumer_group`, which: -### Running workers with Falcon +1. Resolve the argument to a registered reactor class (raising `ArgumentError` if the string doesn't match any registered reactor) +2. Call the corresponding `Store` method +3. Invoke the reactor's callback (`on_stop`, `on_reset`, `on_start`) -If you use [Falcon](https://github.com/socketry/falcon) as your web server, you can run Sourced workers in the same process. Both Falcon and Sourced use the [Async](https://github.com/socketry/async) gem, so workers run as lightweight fibers alongside web requests — no separate worker process needed. +#### Defining callbacks -This requires `Sourced.config.executor = :async` (the default). Do not change it to `:thread` when using Falcon, as workers must run as fibers to share Falcon's event loop. - -Add a `./falcon.rb` file to the root of your app, which requieres `sourced/falcon` (no hard dependency on Falcon in sourced.gemspec): +Override the no-op class methods on your reactor to hook into lifecycle events: ```ruby -# falcon.rb -#!/usr/bin/env falcon-host -require 'bundler/setup' -require 'sourced/falcon' -require_relative 'config/environment' # <= YOUR app setup, Sourced.configure, register reactors, etc. +class CourseDecider < Sourced::Decider + partition_by :course_name -service "my-app" do - include Sourced::Falcon::Environment - include Falcon::Environment::Rackup # loads config.ru - - # -- Falcon / Async options -- - url "http://[::]:9292" # Server bind URL (default: "http://[::]:9292") - count 2 # Number of OS processes to fork (default: Etc.nprocessors) - timeout 30 # Connection timeout in seconds (default: nil) - verbose false # Enable verbose logging (default: false) - cache true # Enable HTTP response caching (default: false) - - # Sourced worker options default to Sourced.config values. - # Override per-service if needed: - # sourced_worker_count 4 - # sourced_worker_batch_size 50 - # sourced_housekeeping_count 1 - # sourced_housekeeping_interval 3 - # sourced_housekeeping_heartbeat_interval 5 - # sourced_housekeeping_claim_ttl_seconds 120 -end -``` + # Called when the consumer group is stopped. + # `message` is the optional reason string passed to stop_consumer_group. + def self.on_stop(message = nil) + Rails.logger.info "CourseDecider stopped: #{message}" + end -Run with: + # Called when the consumer group is reset (offsets cleared). + def self.on_reset + Rails.cache.delete_matched('course_projections/*') + end -``` -falcon host + # Called when the consumer group is started. + def self.on_start + Rails.logger.info 'CourseDecider started' + end +end ``` -Total Sourced workers = `count * sourced_worker_count`. For example, `count 2` and `sourced_worker_count 4` gives 8 worker fibers across 2 OS processes, all competing for events via database locks (same as running multiple Supervisors). +Reactors without custom callbacks work fine — the defaults are no-ops. -Set `config.worker_count = 0` to run Falcon as a web-only process with no Sourced workers. This is useful if you want to run workers separately via `Sourced::Supervisor` while still using Falcon for HTTP, or if you explicitely don't want workers adding unnecessary pressure on the database. +## Monitoring -On shutdown (`Ctrl-C` / `SIGTERM`), Falcon signals workers to stop. Their poll loops exit gracefully with no stale claims. +`Store#stats` returns system-wide diagnostics for monitoring and debugging Sourced deployments. -### How worker dispatch works +```ruby +stats = store.stats +stats.max_position # => 42 (latest position in the message log) +stats.groups # => array of per-consumer-group hashes +``` -Instead of each worker polling the database independently, Sourced uses a signal-driven dispatch model. Workers block on a shared `WorkQueue` waiting for signals, and two sources feed that queue: +Each group hash contains: -1. **Backend notifier** (real-time): The backend exposes a generic pub/sub notifier (`PGNotifier` for PostgreSQL, `InlineNotifier` for others). When messages are appended, the backend publishes a `messages_appended` event with the message types. When a stopped reactor is resumed, it publishes a `reactor_resumed` event with the consumer group ID. For PostgreSQL, these are delivered over PG `LISTEN/NOTIFY`; for other backends, they fire synchronously. +| Key | Description | +|----------------------|----------------------------------------------------------------| +| `group_id` | Consumer group identifier (e.g. `"CourseDecider"`) | +| `status` | `"active"`, `"stopped"`, or `"failed"` | +| `retry_at` | `Time` of next retry, or `nil` | +| `error_context` | Hash with error details (`{}` when healthy, see below) | +| `oldest_processed` | `MIN(last_position)` across partitions where processing started | +| `newest_processed` | `MAX(last_position)` across partitions | +| `partition_count` | Number of offset rows (partitions) for this group | -2. **CatchUpPoller** (safety net): A single fiber pushes all registered reactors into the WorkQueue every few seconds (default 5). This covers startup catch-up, missed notifications, offset resets, and PG reconnections. +### `error_context` -The `Dispatcher` subscribes to the backend notifier and maps these events to reactor classes, pushing them onto the `WorkQueue`. For `messages_appended`, it resolves message types to interested reactors via an eager lookup table. For `reactor_resumed`, it resolves the group ID directly to the reactor class. +The `error_context` hash is empty (`{}`) for healthy groups. When a group is stopped or has failed, it may contain: -When a worker pops a reactor from the queue, it enters a **bounded drain loop**: it processes up to `max_drain_rounds` batches (default 10) for that reactor, then re-enqueues the reactor if it hit the cap. This ensures no single reactor monopolizes workers, and multiple workers can drain the same reactor concurrently on different streams (via `SKIP LOCKED`). +| Key | Present when | Description | +|----------------------|--------------|--------------------------------------| +| `:message` | Stopped | Operator-supplied reason for stopping | +| `:exception_class` | Failed | Exception class name (e.g. `"RuntimeError"`) | +| `:exception_message` | Failed | Exception message string | -The `WorkQueue` caps pending entries per reactor (equal to the worker count), so notification bursts are coalesced without queue bloat. +When retries are configured, `error_context` also accumulates retry state set by `GroupUpdater#retry_later`. +```ruby +stats = store.stats +stats.groups.each do |g| + puts "#{g[:group_id]}: #{g[:status]} (#{g[:partition_count]} partitions, up to position #{g[:newest_processed]})" + if g[:status] == 'failed' + puts " error: #{g[:error_context][:exception_class]}: #{g[:error_context][:exception_message]}" + end +end ``` -Backend notifier ────┐ - (PG LISTEN or ├──▶ WorkQueue (capped/reactor) ──▶ Worker fibers - inline pub/sub) │ │ │ -CatchUpPoller (5s) ──┘ │◀── re-enqueue ────────────┘ - (if max_drain_rounds hit) -``` - -This design preserves natural back-pressure (workers only fetch when ready), eliminates polling-interval lag for new messages, and handles both real-time and catch-up work in a single operating mode. -## Custom attribute types and coercions. +### `Store#read_offsets` — inspecting partition offsets -Define a module to hold your attribute types using [Plumb](https://github.com/ismasan/plumb) +`read_offsets` lists individual consumer group offsets with optional filtering and cursor-based pagination. Useful for inspecting the progress and claim status of each partition. ```ruby -module Types - include Plumb::Types - - # Your own types here. - CorporateEmail = Email[/@apple\.com^/] -end +result = store.read_offsets +result.offsets # => array of offset hashes +result.total_count # => total number of matching offsets (ignoring pagination) ``` -Then you can use any [built-in Plumb types](https://github.com/ismasan/plumb?tab=readme-ov-file#built-in-types), as well as your own, when defining command or event structs (or any other data structures for your app). +#### Parameters -```ruby -UpdateEmail = Sourced::Command.define('accounts.update_email') do - attribute :email, Types::CorporateEmail -end -``` +| Parameter | Type | Default | Description | +|-------------|----------------|---------|----------------------------------------------------------| +| `group_id:` | `String`, `nil` | `nil` | Filter by consumer group. `nil` returns all groups. | +| `limit:` | `Integer` | `50` | Max offsets per page. | +| `from_id:` | `Integer`, `nil`| `nil` | Cursor — return offsets with `id >= from_id` (inclusive). | -## Error handling +#### Offset hash fields -Sourced workflows are eventually-consistent by default. This means that commands and events are handled in background processes, and any exceptions raised can't be immediatly surfaced back to the user (and, there might not be a user anyway!). +Each offset in the result is a Hash with: -Most "domain errors" in command handlers should be handled by the developer and recorded as domain events, so that the domain can react and/or compensate for them. +| Key | Type | Description | +|------------------|---------------|------------------------------------------------------| +| `:id` | `Integer` | Offset primary key (used as pagination cursor) | +| `:group_name` | `String` | Consumer group identifier | +| `:group_status` | `String` | `"active"`, `"stopped"`, or `"failed"` | +| `:partition_key` | `String` | Partition identifier (e.g. `"device_id:dev-1"`) | +| `:last_position` | `Integer` | Highest acked position for this partition | +| `:claimed` | `Boolean` | Whether a worker currently holds this partition | +| `:claimed_at` | `String`, `nil`| ISO8601 timestamp of the claim | +| `:claimed_by` | `String`, `nil`| Worker ID holding the claim | -To handle true _exceptions_ (code or data bugs, network or IO exceptions) Sourced provides a default error strategy that will mark the affected consumer group as failed (the Postgres backend will log the exception in the `consumer_groups` table). +#### Filtering by group + +```ruby +result = store.read_offsets(group_id: 'CourseDecider') +result.offsets.each do |o| + puts "#{o[:partition_key]}: position #{o[:last_position]}, claimed=#{o[:claimed]}" +end +``` -You can configure the error strategy with retries and exponential backoff, as well as `on_retry` and `on_fail` callbacks. +#### Pagination ```ruby -Sourced.configure do |config| - # config.backend = Sequel.connect(ENV.fetch('DATABASE_URL')) - config.error_strategy do |s| - s.retry( - # Retry up to 3 times - times: 3, - # Wait 5 seconds before retrying - after: 5, - # Custom backoff: given after=5, retries in 5, 10 and 15 seconds before failing - backoff: ->(retry_after, retry_count) { retry_after * retry_count } - ) - - # Trigger this callback on each retry - s.on_retry do |n, exception, message, later| - LOGGER.info("Retrying #{n} times") - end +# First page +page1 = store.read_offsets(limit: 20) - # Finally, trigger this callback - # after all retries have failed and the consumer group is failed. - s.on_fail do |exception, message| - Sentry.capture_exception(exception) - end - end -end +# Next page using cursor +page2 = store.read_offsets(limit: 20, from_id: page1.offsets.last[:id] + 1) ``` -### Custom error strategy +#### Auto-pagination with `to_enum` -You can also configure your own error strategy. It must respond to `#call(exception, message, group)` +`OffsetsResult#to_enum` returns a lazy `Enumerator` that fetches subsequent pages automatically. ```ruby -CUSTOM_STRATEGY = proc do |exception, message, group| - case exception - when Faraday::Error - group.retry(Time.now + 10) - else - group.stop(exception) - end +# Iterate all offsets in pages of 20 +store.read_offsets(limit: 20).to_enum.each do |offset| + puts "#{offset[:group_name]} / #{offset[:partition_key]}: #{offset[:last_position]}" end -Sourced.configure do |config| - # Configure backend, etc - config.error_strategy = CUSTOM_STRATEGY -end +# Works with Enumerable methods +behind = store.read_offsets(limit: 50).to_enum.lazy.select { |o| + o[:last_position] < store.latest_position - 100 +}.to_a ``` -## Stopping and starting consumer groups. - -`Sourced.config.backend` provides an API for stopping and starting consumer groups. For example to resume groups that were stopped by raised exceptions, after the error has been corrected. +#### Array destructuring ```ruby -Sourced.config.backend.stop_consumer_group('Carts::Listings') -Sourced.config.backend.start_consumer_group('Carts::Listings') +offsets, total_count = store.read_offsets(group_id: 'CourseDecider') ``` -## Topology +## Testing -`Sourced.topology` returns a flat array of node structs describing the message flow graph of all registered reactors. This is useful for building visualizations, documentation, or tooling that needs to understand how commands, events, automations and read models connect. +Sourced ships with RSpec helpers for Given-When-Then testing of deciders and projectors. The helpers call `handle_batch` directly — no store, router, or consumer group setup needed. ```ruby -Sourced.register(Cart) -Sourced.register(CartListings) +require 'sourced/testing/rspec' -nodes = Sourced.topology -# => [CommandNode, EventNode, AutomationNode, ReadModelNode, ...] +RSpec.configure do |config| + config.include Sourced::Testing::RSpec +end ``` -The result is memoized. Call `Sourced.reset_topology` to clear the cache after registering new reactors. - -### Node types +### Testing deciders -#### CommandNode - -Represents a command handled by an actor. `produces` lists the event type strings that the command handler can emit (extracted via static analysis). +`with_reactor` takes a decider class and partition attributes, then chains `.given` (history), `.when` (command), and `.then` (expected outcomes). ```ruby -# Fields: type, id, name, group_id, produces, schema -{ type: "command", id: "carts.add_item", name: "Carts::AddItem", - group_id: "Carts::Cart", produces: ["carts.item_added"], - schema: { "type" => "object", "properties" => { ... } } } -``` +RSpec.describe CourseDecider do + include Sourced::Testing::RSpec -#### EventNode + it 'creates a course' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') + .then(CourseCreated, course_id: 'c1', course_name: 'Algebra') + end -Represents an event type. Deduplicated across reactors — the first reactor to reference an event owns its `group_id`. + it 'rejects duplicate course names' do + with_reactor(CourseDecider, course_name: 'Algebra') + .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') + .when(CreateCourse, course_id: 'c2', course_name: 'Algebra') + .then(RuntimeError, "Course 'Algebra' already exists") + end -```ruby -# Fields: type, id, name, group_id, produces, schema -{ type: "event", id: "carts.item_added", name: "Carts::ItemAdded", - group_id: "Carts::Cart", produces: [], - schema: { "type" => "object", "properties" => { ... } } } + it 'produces no events for a no-op command' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(SomeNoopCommand, course_name: 'Algebra') + .then([]) + end +end ``` -#### AutomationNode +#### Multiple expected messages -Represents a `.reaction` block. `consumes` lists what triggers the reaction (event types for actors, readmodel IDs for projectors). `produces` lists the command type strings dispatched by the reaction (extracted via static analysis). +When a decider produces events and reactions, pass all expected messages as instances: ```ruby -# Fields: type, id, name, group_id, consumes, produces -# Actor reaction — consumes an event directly: -{ type: "automation", id: "carts.item_added-Carts::Cart-aut", - name: "reaction(Carts::ItemAdded)", group_id: "Carts::Cart", - consumes: ["carts.item_added"], produces: ["carts.check_inventory"] } - -# Projector reaction — consumes the readmodel: -{ type: "automation", id: "carts.item_added-Carts::CartListings-aut", - name: "reaction(Carts::ItemAdded)", group_id: "Carts::CartListings", - consumes: ["carts.cart_listings-rm"], produces: ["carts.notify_admin"] } +it 'produces event and reaction' do + with_reactor(EnrolmentDecider, course_id: 'c1') + .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') + .when(EnrolStudent, course_id: 'c1', student_id: 's1') + .then( + StudentEnrolled.new(payload: { course_id: 'c1', student_id: 's1' }), + NotifyStudent.new(payload: { student_id: 's1' }) + ) +end ``` -#### ReadModelNode +#### Block form -Represents a projector as a consumer of events. `consumes` lists the event types the projector evolves. `produces` lists the IDs of any automation nodes derived from the projector's reactions. +Pass a block to `.then` to receive the raw action pairs for custom assertions: ```ruby -# Fields: type, id, name, group_id, consumes, produces, schema -{ type: "readmodel", id: "carts.cart_listings-rm", - name: "Carts::CartListings", group_id: "Carts::CartListings", - consumes: ["carts.item_added", "carts.placed"], - produces: ["carts.item_added-Carts::CartListings-aut"], - schema: {} } +it 'inspects action pairs' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') + .then { |pairs| + actions, source_msg = pairs.first + append = Array(actions).find { |a| a.is_a?(Sourced::Actions::Append) } + expect(append.messages.first).to be_a(CourseCreated) + } +end ``` -### How nodes link together +#### `.then!` — run sync and after_sync actions -The `produces` and `consumes` fields reference other node IDs, forming a directed graph: +Use `.then!` instead of `.then` to execute both `sync` and `after_sync` actions before assertions: -``` -CommandNode ──produces──▶ EventNode -EventNode ──consumes──▶ AutomationNode (actor reactions) -EventNode ──consumes──▶ ReadModelNode ──produces──▶ AutomationNode (projector reactions) -AutomationNode ──produces──▶ CommandNode +```ruby +it 'runs sync block' do + with_reactor(CourseDecider, course_name: 'Algebra') + .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') + .then! { |pairs| ... } +end ``` -### Catch-all reactions +### Testing projectors -When a reactor uses a catch-all `reaction do ... end` (no event argument), the topology collapses all covered events into a single automation node named after the reactor, instead of one automation per event. +Projectors use `.given` (events to evolve) and `.then` with a block that receives the projected state. `.when` is not supported — projectors don't handle commands. + +#### StateStored ```ruby -class ReadyOrders < Sourced::Projector::StateStored - event Orders::PaymentConfirmed do |state, event| - # ... +RSpec.describe ItemProjector do + include Sourced::Testing::RSpec + + it 'builds state from events' do + with_reactor(ItemProjector, list_id: 'L1') + .given(ItemAdded, list_id: 'L1', name: 'Apple') + .given(ItemAdded, list_id: 'L1', name: 'Banana') + .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } end - event Orders::BuildConfirmed do |state, event| - # ... + it 'handles removal' do + with_reactor(ItemProjector, list_id: 'L1') + .given(ItemAdded, list_id: 'L1', name: 'Apple') + .and(ItemArchived, list_id: 'L1', name: 'Apple') + .then { |state| expect(state[:items]).to eq([]) } end - # Catch-all: reacts to all evolved events - reaction do |state, event| - if state[:ready] - dispatch Orders::Release - end + it 'runs sync actions with then!' do + with_reactor(ItemProjector, list_id: 'L1') + .given(ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |state| expect(state[:synced]).to be true } end end ``` -This produces a single automation node: - -```ruby -{ type: "automation", id: "ready_orders-aut", - name: "reaction(ReadyOrders)", group_id: "ReadyOrders", - consumes: ["ready_orders-rm"], produces: ["orders.release"] } -``` - -Rather than separate automation nodes for `PaymentConfirmed` and `BuildConfirmed`. - -## Rails integration - -Soon. - -## Sourced vs. ActiveJob - -ActiveJob is a great way to handle background jobs in Rails. It's simple and easy to use. However, it's not designed for event sourcing. -ActiveJob backends (and other job queues) are optimised for parallel processing of jobs, this means that multiple jobs for the same business entity may be processed in parallel without any ordering guarantees. +#### EventSourced -sourced-job-queue-diagram - -Sourced's concurrency model is designed to process events for the same entity in order, while allowing for parallel processing of events for different entities. - -sourced-ordered-streams-diagram - -## Gotchas - -By default `Sourced` processes commands and events asynchronously through -background workers. This can be confusing if you expect reactions to run -automatically when you issue commands. - -For synchronous, all-or-nothing execution use [`Sourced::Unit`](#sourcedunit), -which runs the full command → event → reaction chain inside a single transaction. +Same API — the helper creates an instance, evolves from all given messages, and yields state: ```ruby -# Synchronous execution with Unit -unit = Sourced::Unit.new(Chat, backend: Sourced.config.backend) -results = unit.handle(SendMessage.new(stream_id: 'chat-123', payload: { content: query })) -results.events_for(Chat) # => [MessageSent, ...] -``` - -If you're using the `Sourced::CommandMethods` mixin directly (without a Unit), -note that it persists events but does not trigger reactions. You'd need to -explicitly call `#react` after issuing commands. +RSpec.describe CatalogProjector do + include Sourced::Testing::RSpec -```ruby -chat = Sourced.load(Chat, 'chat-123') -# Persists but does not call reactions -_cmd, events = chat.send_message!(content: query) -# Have to react manually -commands = chat.react(events) + it 'rebuilds state from full history' do + with_reactor(CatalogProjector, course_id: 'c1') + .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') + .given(StudentEnrolled, course_id: 'c1', student_id: 's1') + .then { |state| expect(state[:students]).to eq(['s1']) } + end +end ``` +### Message matching -## Installation - -Install the gem and add to the application's Gemfile by executing: - - $ bundle add sourced - -**Note**: this gem is under active development, so you probably want to install from Github: -In your Gemfile: - - $ gem 'sourced', github: 'ismasan/sourced' - -## Development - -After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. - -To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). +`.then` compares messages by **class** and **payload** only. Fields like `id`, `created_at`, `causation_id`, `correlation_id`, and `metadata` are ignored, so tests don't need to match auto-generated values. -## Contributing +## Full example -Bug reports and pull requests are welcome on GitHub at https://github.com/ismasan/sourced. +See `examples/app/` for a complete Sinatra application with: +- Two deciders (course creation with name uniqueness, student enrolment with capacity limits) +- An event-sourced projector writing JSON files +- Synchronous command handling via `Sourced.handle!` in HTTP endpoints +- Background worker processing via Falcon diff --git a/examples/app/Gemfile b/examples/app/Gemfile new file mode 100644 index 00000000..4eff7441 --- /dev/null +++ b/examples/app/Gemfile @@ -0,0 +1,9 @@ +source 'https://rubygems.org' + +gem 'sourced', path: '../..' +gem 'sourced-ui', path: '/Users/ismasan/code/personal/gems/sourced-ui' +gem 'sequel' +gem 'sqlite3' +gem 'sinatra' +gem 'falcon' +gem 'irb' diff --git a/examples/app/Gemfile.lock b/examples/app/Gemfile.lock new file mode 100644 index 00000000..f3eab322 --- /dev/null +++ b/examples/app/Gemfile.lock @@ -0,0 +1,286 @@ +PATH + remote: ../.. + specs: + sourced (0.0.1) + async + plumb (>= 0.0.17) + +PATH + remote: /Users/ismasan/code/personal/gems/sourced-ui + specs: + sourced-ui (0.1.0) + datastar (= 1.0.2) + logger + phlex + rack + sourced + +GEM + remote: https://rubygems.org/ + specs: + async (2.36.0) + console (~> 1.29) + fiber-annotation + io-event (~> 1.11) + metrics (~> 0.12) + traces (~> 0.18) + async-container (0.34.2) + async (~> 2.22) + async-container-supervisor (0.10.0) + async-service + io-endpoint + memory (~> 0.7) + memory-leak (~> 0.10) + process-metrics + async-http (0.94.2) + async (>= 2.10.2) + async-pool (~> 0.11) + io-endpoint (~> 0.14) + io-stream (~> 0.6) + metrics (~> 0.12) + protocol-http (~> 0.58) + protocol-http1 (~> 0.36) + protocol-http2 (~> 0.22) + protocol-url (~> 0.2) + traces (~> 0.10) + async-http-cache (0.4.6) + async-http (~> 0.56) + async-pool (0.11.1) + async (>= 2.0) + async-service (0.20.1) + async + async-container (~> 0.34) + string-format (~> 0.2) + bake (0.24.1) + bigdecimal + samovar (~> 2.1) + base64 (0.3.0) + bigdecimal (4.0.1) + concurrent-ruby (1.3.6) + console (1.34.3) + fiber-annotation + fiber-local (~> 1.1) + json + datastar (1.0.2) + json + logger + rack (>= 3.2) + date (3.5.1) + erb (6.0.1) + falcon (0.54.2) + async + async-container (~> 0.20) + async-container-supervisor (~> 0.6) + async-http (~> 0.75) + async-http-cache (~> 0.4) + async-service (~> 0.19) + bundler + localhost (~> 1.1) + openssl (>= 3.0) + protocol-http (~> 0.31) + protocol-rack (~> 0.7) + samovar (~> 2.3) + fiber-annotation (0.2.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (1.0.1) + io-console (0.8.2) + io-endpoint (0.17.2) + io-event (1.14.2) + io-stream (0.11.1) + irb (1.17.0) + pp (>= 0.6.0) + prism (>= 1.3.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.18.1) + localhost (1.7.0) + logger (1.7.0) + mapping (1.1.3) + memory (0.12.0) + bake (~> 0.15) + console + msgpack + memory-leak (0.10.2) + process-metrics (>= 0.10.1) + metrics (0.15.0) + msgpack (1.8.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + openssl (4.0.1) + phlex (2.4.1) + refract (~> 1.0) + zeitwerk (~> 2.7) + plumb (0.0.17) + bigdecimal + concurrent-ruby + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.9.0) + process-metrics (0.10.2) + console (~> 1.8) + json (~> 2) + samovar (~> 2.1) + protocol-hpack (1.5.1) + protocol-http (0.59.0) + protocol-http1 (0.37.0) + protocol-http (~> 0.58) + protocol-http2 (0.24.0) + protocol-hpack (~> 1.4) + protocol-http (~> 0.47) + protocol-rack (0.21.1) + io-stream (>= 0.10) + protocol-http (~> 0.58) + rack (>= 1.0) + protocol-url (0.4.0) + psych (5.3.1) + date + stringio + rack (3.2.5) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rdoc (7.2.0) + erb + psych (>= 4.0.0) + tsort + refract (1.1.0) + prism + zeitwerk + reline (0.6.3) + io-console (~> 0.5) + ruby2_keywords (0.0.5) + samovar (2.4.1) + console (~> 1.0) + mapping (~> 1.0) + sequel (5.101.0) + bigdecimal + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + sqlite3 (2.9.0-aarch64-linux-gnu) + sqlite3 (2.9.0-aarch64-linux-musl) + sqlite3 (2.9.0-arm-linux-gnu) + sqlite3 (2.9.0-arm-linux-musl) + sqlite3 (2.9.0-arm64-darwin) + sqlite3 (2.9.0-x86-linux-gnu) + sqlite3 (2.9.0-x86-linux-musl) + sqlite3 (2.9.0-x86_64-darwin) + sqlite3 (2.9.0-x86_64-linux-gnu) + sqlite3 (2.9.0-x86_64-linux-musl) + string-format (0.2.0) + stringio (3.2.0) + tilt (2.7.0) + traces (0.18.2) + tsort (0.2.0) + zeitwerk (2.7.5) + +PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + falcon + irb + sequel + sinatra + sourced! + sourced-ui! + sqlite3 + +CHECKSUMS + async (2.36.0) sha256=090623f4c65706664355c9efa6c7bfb86771a513e65cd681c51cb27747530550 + async-container (0.34.2) sha256=83ac767e74a832c42e156110c14e10b1fb3fd7583fa1beb3dfe45269e3554746 + async-container-supervisor (0.10.0) sha256=96a2b6048312e679993c116f0b29b70fc25696b0a6e8c68d8c4f9b8e666a8614 + async-http (0.94.2) sha256=c5ca94b337976578904a373833abe5b8dfb466a2946af75c4ae38c409c5c78b2 + async-http-cache (0.4.6) sha256=2038d1f093182f16b50b4db271c25085e3938da10bfcfc2904cadb0530fddfd6 + async-pool (0.11.1) sha256=98e1583e199a75f7dc70f8e65fc8d0d3b28636c3f256595d43e206642ad8fbda + async-service (0.20.1) sha256=d77ca8912e31c729f6e62c783ae3f364385ccc91a34c97998b761d7bda72b01b + bake (0.24.1) sha256=8bfac7e61514b17720e3b13cf6a5e122243f43123c6802707b150904bec5f4c7 + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7 + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + console (1.34.3) sha256=869fbd74697efc4c606f102d2812b0b008e4e7fd738a91c591e8577140ec0dcc + datastar (1.0.2) sha256=3fd1430a279f0668142e9d5ec5f7f81087b1ea3defc75f55ea8aa2ac036f651c + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + falcon (0.54.2) sha256=ab35a702466f318d6b3189e638e4aee53290261a5404c9f681115f4a8aca153e + fiber-annotation (0.2.0) sha256=7abfadf1d119f508867d4103bf231c0354d019cc39a5738945dec2edadaf6c03 + fiber-local (1.1.0) sha256=c885f94f210fb9b05737de65d511136ea602e00c5105953748aa0f8793489f06 + fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1 + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-endpoint (0.17.2) sha256=3feaf766c116b35839c11fac68b6aaadc47887bb488902a57bf8e1d288fb3338 + io-event (1.14.2) sha256=b0a069190eafe86005c22f7464f744971b5bd82f153740d34e6ab49548d4f613 + io-stream (0.11.1) sha256=fa5f551fcff99581c1757b9d1cee2c37b124f07d2ca4f40b756a05ab9bd21b87 + irb (1.17.0) sha256=168c4ddb93d8a361a045c41d92b2952c7a118fa73f23fe14e55609eb7a863aae + json (2.18.1) sha256=fe112755501b8d0466b5ada6cf50c8c3f41e897fa128ac5d263ec09eedc9f986 + localhost (1.7.0) sha256=09b32819537f914ccdf0a7c595fab162517401b6ef644a2afd3708d943c4547f + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mapping (1.1.3) sha256=2274931d20ecd46eaafdd1e00c58cc7472133b213bcac335cc7733d3c75f4da2 + memory (0.12.0) sha256=786a14d84cec8e5667a491da02ebbf492b9ec3d19d35161131ac9d47abb684b4 + memory-leak (0.10.2) sha256=c486973e3dd2339d837f050bbb0ffc5a72584f8553982c60f59d486a6ad04a5a + metrics (0.15.0) sha256=61ded5bac95118e995b1bc9ed4a5f19bc9814928a312a85b200abbdac9039072 + msgpack (1.8.0) sha256=e64ce0212000d016809f5048b48eb3a65ffb169db22238fb4b72472fecb2d732 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + openssl (4.0.1) sha256=e27974136b7b02894a1bce46c5397ee889afafe704a839446b54dc81cb9c5f7d + phlex (2.4.1) sha256=e596717fbfe38b5271840266758779ebe75092e02629f0c170287e6290a70b12 + plumb (0.0.17) sha256=434138323bde29cefbec136bdd2f23f3e37d5ea9eca72fafb5bce6801d64a56d + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + process-metrics (0.10.2) sha256=c0593c7e6695d0e75a5e10fb9c13728a0a23c1ad3514b747dee2e7b19eed00d3 + protocol-hpack (1.5.1) sha256=6feca238b8078da1cd295677d6f306c6001af92d75fe0643d33e6956cbc3ad91 + protocol-http (0.59.0) sha256=90e20ad817cb3ffe947d4fd6194fe0651f385625dcce055386d1c356ee32547b + protocol-http1 (0.37.0) sha256=5bdd739e28792b341134596f6f5ab21a9d4b395f67bae69e153743eb0e69d123 + protocol-http2 (0.24.0) sha256=65327a019b7e36d2774e94050bf57a43bb60212775d2fcf02ae1d2ed4f01ef28 + protocol-rack (0.21.1) sha256=366ff16efbf4c2f8d2e3fad4e992effa2357610f70effbccfa2767d26fedc577 + protocol-url (0.4.0) sha256=64d4c03b6b51ad815ac6fdaf77a1d91e5baf9220d26becb846c5459dacdea9e1 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + rack (3.2.5) sha256=4cbd0974c0b79f7a139b4812004a62e4c60b145cba76422e288ee670601ed6d3 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192 + refract (1.1.0) sha256=ee3b9627e39f7692831101e2fedd73e0d09a592ff5d5c05f171d14211fc7a9c7 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + samovar (2.4.1) sha256=c3b91dd0580771e3bc600621c1111f29542529dcffafaac3b6bf068b3f309e80 + sequel (5.101.0) sha256=d2ae3fd997a7c4572e8357918e777869faf90dc19310fcd6332747122aed2b29 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + sourced (0.0.1) + sourced-ui (0.1.0) + sqlite3 (2.9.0-aarch64-linux-gnu) sha256=cfe1e0216f46d7483839719bf827129151e6c680317b99d7b8fc1597a3e13473 + sqlite3 (2.9.0-aarch64-linux-musl) sha256=56a35cb2d70779afc2ac191baf2c2148242285ecfed72f9b021218c5c4917913 + sqlite3 (2.9.0-arm-linux-gnu) sha256=a19a21504b0d7c8c825fbbf37b358ae316b6bd0d0134c619874060b2eef05435 + sqlite3 (2.9.0-arm-linux-musl) sha256=fca5b26197c70e3363115d3faaea34d7b2ad9c7f5fa8d8312e31b64e7556ee07 + sqlite3 (2.9.0-arm64-darwin) sha256=a917bd9b84285766ff3300b7d79cd583f5a067594c8c1263e6441618c04a6ed3 + sqlite3 (2.9.0-x86-linux-gnu) sha256=47317ba230f6c2c361981aa5fc1bf9de1b99727317171393ba90abab092c5b5f + sqlite3 (2.9.0-x86-linux-musl) sha256=b627f3a2ca59aaaa5e10b8666cdbd7122469b49afa4bd895133cecb7b5c1368d + sqlite3 (2.9.0-x86_64-darwin) sha256=59fe51baa3cb33c36d27ce78b4ed9360cd33ccca09498c2ae63850c97c0a6026 + sqlite3 (2.9.0-x86_64-linux-gnu) sha256=72fff9bd750070ba3af695511ba5f0e0a2d8a9206f84869640b3e99dfaf3d5a5 + sqlite3 (2.9.0-x86_64-linux-musl) sha256=ef716ba7a66d7deb1ccc402ac3a6d7343da17fac862793b7f0be3d2917253c90 + string-format (0.2.0) sha256=bc981c14116b061f12134549f32fa2d61a17b5a35dd6fd36596c21722a789af6 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.7.0) sha256=0d5b9ba69f6a36490c64b0eee9f6e9aad517e20dcc848800a06eb116f08c6ab3 + traces (0.18.2) sha256=80f1649cb4daace1d7174b81f3b3b7427af0b93047759ba349960cb8f315e214 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + zeitwerk (2.7.5) sha256=d8da92128c09ea6ec62c949011b00ed4a20242b255293dd66bf41545398f73dd + +BUNDLED WITH + 4.0.3 diff --git a/examples/app/README.md b/examples/app/README.md new file mode 100644 index 00000000..9948f0d3 --- /dev/null +++ b/examples/app/README.md @@ -0,0 +1,245 @@ +# CCC Demo: Student Enrolment + +A demo app exercising CCC's core features: sync deciders with optimistic concurrency, multi-entity context validation, uniqueness checks, and an async projector for read models. + +## Setup + +```bash +cd examples/ccc_app +bundle install +``` + +## Running + +```bash +bundle exec falcon host +``` + +The app starts at `http://localhost:9292`. + +## Domain + +- **CourseDecider** -- enforces course name uniqueness (partitioned by `course_name`) +- **EnrolmentDecider** -- validates course exists, no duplicate students, max 20 per course (partitioned by `course_id`) +- **CourseCatalogProjector** -- async read model updated by background workers (partitioned by `course_id`) + +Deciders run synchronously in the request (via the resource endpoints) or asynchronously via the generic `/commands` endpoint. The projector runs in the background via CCC's Dispatcher, so reads are eventually consistent. + +## IRB + +```bash +irb -r ./domain +``` + +```ruby +CourseApp.setup! +# Now you can use CCC.load, CCC.store.append, etc. +``` + +## Endpoints + +### Generic command endpoint + +``` +POST /commands +``` + +Accepts any registered CCC message type. The command is appended to the store and processed asynchronously by background workers. Returns **202** on success. + +| Field | Type | Required | +|-------|------|----------| +| `type` | string | yes | +| `payload` | object | yes | + +```bash +curl -X POST http://localhost:9292/commands \ + -H 'Content-Type: application/json' \ + -d '{"type": "courses.create", "payload": {"course_id": "abc-123", "course_name": "Algebra"}}' +``` + +```json +{"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "type": "courses.create"} +``` + +Invalid payload returns **422** with field-level errors: + +```bash +curl -X POST http://localhost:9292/commands \ + -H 'Content-Type: application/json' \ + -d '{"type": "courses.enrol", "payload": {"course_id": "", "student_id": ""}}' +``` + +```json +{"payload": {"course_id": "must be present", "student_id": "must be present"}} +``` + +Unknown message type returns **422**: + +```json +{"error": "Unknown message type: foo.bar"} +``` + +### List courses + +``` +GET / +``` + +Returns the course catalog (populated by the async projector). + +```bash +curl http://localhost:9292/ +``` + +```json +[ + {"course_id": "abc-123", "course_name": "Algebra", "student_count": 2} +] +``` + +### Create a course + +``` +POST /courses +``` + +| Field | Type | Required | +|-------|------|----------| +| `course_name` | string | yes | + +```bash +curl -X POST http://localhost:9292/courses \ + -H 'Content-Type: application/json' \ + -d '{"course_name": "Algebra"}' +``` + +```json +{"course_id": "abc-123", "course_name": "Algebra"} +``` + +Duplicate name returns **422**: + +```bash +curl -X POST http://localhost:9292/courses \ + -H 'Content-Type: application/json' \ + -d '{"course_name": "Algebra"}' +``` + +```json +{"error": "Course 'Algebra' already exists"} +``` + +### Course detail + +``` +GET /courses/:id +``` + +```bash +curl http://localhost:9292/courses/abc-123 +``` + +```json +{ + "course_id": "abc-123", + "course_name": "Algebra", + "students": ["stu-1", "stu-2"], + "student_count": 1 +} +``` + +Returns **404** if the course hasn't been projected yet or doesn't exist. + +### Enrol a student + +``` +POST /courses/:id/enrolments +``` + +| Field | Type | Required | +|-------|------|----------| +| `student_id` | string | yes | + +```bash +curl -X POST http://localhost:9292/courses/abc-123/enrolments \ + -H 'Content-Type: application/json' \ + -d '{"student_id": "stu-1"}' +``` + +```json +{"course_id": "abc-123", "student_id": "stu-1"} +``` + +Errors return **422**: + +- Non-existent course: `{"error": "Course 'abc-123' does not exist"}` +- Duplicate student: `{"error": "Student 'stu-1' is already enrolled"}` +- Course full: `{"error": "Course is full (max 20 students)"}` + +Concurrent modification returns **409**: + +```json +{"error": "Concurrent modification — please retry"} +``` + +## Full walkthrough (resource endpoints) + +```bash +# Create a course +curl -s -X POST http://localhost:9292/courses \ + -H 'Content-Type: application/json' \ + -d '{"course_name": "Algebra"}' | jq . +# Note the course_id from the response + +# List courses (may take a moment for the projector to catch up) +curl -s http://localhost:9292/ | jq . + +# Enrol students (replace with the actual ID) +curl -s -X POST http://localhost:9292/courses//enrolments \ + -H 'Content-Type: application/json' \ + -d '{"student_id": "student-1"}' | jq . + +curl -s -X POST http://localhost:9292/courses//enrolments \ + -H 'Content-Type: application/json' \ + -d '{"student_id": "student-2"}' | jq . + +# View course detail +curl -s http://localhost:9292/courses/ | jq . + +# Try duplicate name (422) +curl -s -X POST http://localhost:9292/courses \ + -H 'Content-Type: application/json' \ + -d '{"course_name": "Algebra"}' | jq . + +# Try duplicate enrolment (422) +curl -s -X POST http://localhost:9292/courses//enrolments \ + -H 'Content-Type: application/json' \ + -d '{"student_id": "student-1"}' | jq . +``` + +## Full walkthrough (generic /commands endpoint) + +```bash +# Create a course (async — processed by background workers) +curl -s -X POST http://localhost:9292/commands \ + -H 'Content-Type: application/json' \ + -d '{"type": "courses.create", "payload": {"course_id": "abc-123", "course_name": "Algebra"}}' | jq . + +# Enrol a student +curl -s -X POST http://localhost:9292/commands \ + -H 'Content-Type: application/json' \ + -d '{"type": "courses.enrol", "payload": {"course_id": "abc-123", "student_id": "student-1"}}' | jq . + +# List courses (may take a moment for workers to process) +curl -s http://localhost:9292/ | jq . + +# Invalid payload (422) +curl -s -X POST http://localhost:9292/commands \ + -H 'Content-Type: application/json' \ + -d '{"type": "courses.create", "payload": {"course_id": "", "course_name": ""}}' | jq . + +# Unknown type (422) +curl -s -X POST http://localhost:9292/commands \ + -H 'Content-Type: application/json' \ + -d '{"type": "nope", "payload": {}}' | jq . +``` diff --git a/examples/app/app.rb b/examples/app/app.rb new file mode 100644 index 00000000..b4be4fc4 --- /dev/null +++ b/examples/app/app.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require_relative 'domain' +require 'sinatra/base' +require 'json' +require 'securerandom' + +class App < Sinatra::Base + set :default_content_type, 'application/json' + + helpers do + def json_body + JSON.parse(request.body.read, symbolize_names: true) + rescue JSON::ParserError + halt 400, { error: 'Invalid JSON' }.to_json + end + end + + # Generic command endpoint + post '/commands' do + data = json_body + message = CourseApp::Command.from(data.merge(metadata: { source: 'http' })) + + unless message.valid? + halt 422, message.errors.to_json + end + + Sourced.store.append([message]) + + status 202 + { id: message.id, type: message.type }.to_json + + rescue Sourced::UnknownMessageError => e + halt 422, { error: e.message }.to_json + end + + # List all courses + get '/' do + courses = CourseApp::CourseCatalogProjector.all_courses.map do |c| + { course_id: c[:course_id], course_name: c[:course_name], student_count: c[:student_count] } + end + courses.to_json + end + + # Create a course + post '/courses' do + body = json_body + course_id = SecureRandom.uuid + + cmd = CourseApp::CreateCourse.new(payload: { course_id: course_id, course_name: body[:course_name] }) + cmd, _decider, _events = Sourced.handle!(CourseApp::CourseDecider, cmd) + + halt 422, cmd.errors.to_json unless cmd.valid? + + status 201 + { course_id: course_id, course_name: cmd.payload.course_name }.to_json + + rescue RuntimeError => e + halt 422, { error: e.message }.to_json + rescue Sourced::ConcurrentAppendError + halt 409, { error: 'Concurrent modification — please retry' }.to_json + end + + # Course detail + get '/courses/:id' do + course = CourseApp::CourseCatalogProjector.read_course(params[:id]) + + halt 404, { error: 'Course not found' }.to_json unless course + + course.to_json + end + + # Enrol a student + post '/courses/:id/enrolments' do + body = json_body + + cmd = CourseApp::EnrolStudent.new(payload: { + course_id: params[:id], + student_id: body[:student_id] + }) + cmd, _decider, _events = Sourced.handle!(CourseApp::EnrolmentDecider, cmd) + + halt 422, cmd.errors.to_json unless cmd.valid? + + status 201 + { course_id: cmd.payload.course_id, student_id: cmd.payload.student_id }.to_json + + rescue RuntimeError => e + halt 422, { error: e.message }.to_json + rescue Sourced::ConcurrentAppendError + halt 409, { error: 'Concurrent modification — please retry' }.to_json + end +end diff --git a/examples/app/config.ru b/examples/app/config.ru new file mode 100644 index 00000000..425e4671 --- /dev/null +++ b/examples/app/config.ru @@ -0,0 +1,19 @@ +require_relative 'app' + +require 'sourced/ui/dashboard' +require 'datastar/async_executor' +Datastar.config.executor = Datastar::AsyncExecutor.new + +Sourced::UI::Dashboard.configure do |config| + config.header_links([ + { label: 'back to app', href: '/', url: false } + ]) +end + +map '/sourced' do + run Sourced::UI::Dashboard +end + +map '/' do + run App +end diff --git a/examples/app/domain.rb b/examples/app/domain.rb new file mode 100644 index 00000000..3e6cb31c --- /dev/null +++ b/examples/app/domain.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +require 'bundler/setup' +require 'sourced' +require 'sourced' +require 'sequel' +require 'fileutils' +require 'json' + + +module CourseApp + # --- Messages --- + + class Event < Sourced::Message; end + class Command < Sourced::Message; end + + CreateCourse = Command.define('courses.create') { attribute :course_id, String; attribute :course_name, String } + CourseCreated = Event.define('courses.created') { attribute :course_id, String; attribute :course_name, String } + EnrolStudent = Command.define('courses.enrol') { attribute :course_id, String; attribute :student_id, String } + StudentEnrolled = Event.define('courses.enrolled') { attribute :course_id, String; attribute :student_id, String } + + # --- Deciders --- + + # Enforces course name uniqueness. + # Partition by :course_name so Sourced.load reads all CourseCreated with that name. + class CourseDecider < Sourced::Decider + partition_by :course_name + + state do |_partition_values| + { name_taken: false } + end + + evolve CourseCreated do |state, _event| + state[:name_taken] = true + end + + command CreateCourse do |state, cmd| + raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] + + event CourseCreated, course_id: cmd.payload.course_id, course_name: cmd.payload.course_name + end + end + + # Enforces enrolment rules: course must exist, no duplicates, max 20 students. + # Partition by :course_id so Sourced.load reads CourseCreated + StudentEnrolled for that course. + class EnrolmentDecider < Sourced::Decider + partition_by :course_id + + state do |_partition_values| + { course_exists: false, student_ids: [], student_count: 0 } + end + + evolve CourseCreated do |state, _event| + state[:course_exists] = true + end + + evolve StudentEnrolled do |state, event| + state[:student_ids] << event.payload.student_id + state[:student_count] += 1 + end + + command EnrolStudent do |state, cmd| + raise "Course '#{cmd.payload.course_id}' does not exist" unless state[:course_exists] + raise "Student '#{cmd.payload.student_id}' is already enrolled" if state[:student_ids].include?(cmd.payload.student_id) + raise "Course is full (max 20 students). Has #{state[:student_count]}" if state[:student_count] >= 20 + + event StudentEnrolled, + course_id: cmd.payload.course_id, + student_id: cmd.payload.student_id + end + end + + # --- Projector (async read model) --- + + # Builds a file-backed course catalog from events. + # Each course is written to a JSON file in storage/projections/. + # Registered with Sourced for background processing by workers. + class CourseCatalogProjector < Sourced::Projector::EventSourced + partition_by :course_id + + PROJECTIONS_DIR = File.join(__dir__, 'storage', 'projections') + + class << self + def projection_path(course_id) + File.join(PROJECTIONS_DIR, "#{course_id}.json") + end + + def read_course(course_id) + path = projection_path(course_id) + return nil unless File.exist?(path) + + JSON.parse(File.read(path), symbolize_names: true) + end + + def all_courses + Dir.glob(File.join(PROJECTIONS_DIR, '*.json')).filter_map do |path| + JSON.parse(File.read(path), symbolize_names: true) + rescue JSON::ParserError + nil + end + end + end + + state do |partition_values| + { course_id: nil, course_name: nil, students: [] } + end + + evolve CourseCreated do |state, event| + state[:course_id] = event.payload.course_id + state[:course_name] = event.payload.course_name + end + + evolve StudentEnrolled do |state, event| + state[:students] << event.payload.student_id + end + + sync do |state:, messages:, **| + next unless state[:course_id] + + FileUtils.mkdir_p(PROJECTIONS_DIR) + data = { + course_id: state[:course_id], + course_name: state[:course_name], + students: state[:students], + student_count: state[:students].size + } + File.write(self.class.projection_path(state[:course_id]), JSON.pretty_generate(data)) + end + end + + # --- Configuration --- + + DB_PATH = File.join(__dir__, 'storage', 'ccc_app.db') + + Sourced.configure do |c| + c.store = Sequel.sqlite(DB_PATH) + end + + Sourced.register(CourseDecider) + Sourced.register(EnrolmentDecider) + Sourced.register(CourseCatalogProjector) +end diff --git a/examples/app/falcon.rb b/examples/app/falcon.rb new file mode 100644 index 00000000..6084363f --- /dev/null +++ b/examples/app/falcon.rb @@ -0,0 +1,14 @@ +#!/usr/bin/env falcon-host +# frozen_string_literal: true + +require_relative 'domain' +require_relative 'app' +require 'sourced/falcon' + +service "ccc-app" do + include Sourced::Falcon::Environment + include Falcon::Environment::Rackup + + # url "http://localhost:9292" + count 1 +end diff --git a/examples/app/storage/.gitkeep b/examples/app/storage/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/cart.rb b/examples/cart.rb deleted file mode 100644 index baa51d3f..00000000 --- a/examples/cart.rb +++ /dev/null @@ -1,232 +0,0 @@ -# frozen_string_literal: true - -require 'bundler' -Bundler.setup(:test) - -require 'sourced' -require 'sequel' - -# ActiveRecord::Base.establish_connection(adapter: 'postgresql', database: 'decider') -unless ENV['backend_configured'] - puts 'aggregate config' - Sourced.configure do |config| - config.backend = Sequel.postgres('sourced_development') - end - Sourced.config.backend.install - ENV['backend_configured'] = 'true' -end - -# A cart Actor/Aggregate -# Example: -# cart = Cart.new('cart-1') -# cart.add_item(name: 'item1', price: 100) -# cart.place -# cart.events -# -# The above sends a Cart::Place command -# which produces a Cart::Placed event -class Cart < Sourced::Actor - State = Struct.new(:status, :notified, :items, :mailer_id) do - def total = items.sum(&:price) - end - - state do |id| - State.new(:open, false, [], nil) - end - - ItemAdded = Sourced::Message.define('cart.item_added') do - attribute :name, String - attribute :price, Integer - end - - Placed = Sourced::Message.define('cart.placed') - Notified = Sourced::Message.define('cart.notified') do - attribute :mailer_id, String - end - - # Defines a Cart::AddItem command struct - command :add_item, name: String, price: Integer do |cart, cmd| - event(ItemAdded, cmd.payload.to_h) - end - - # Defines a Cart::Place command struct - command :place do |_, cmd| - event(Placed) - end - - # Defines a Cart::Notify command struct - command :notify, mailer_id: String do |_, cmd| - puts "#{self.class.name} #{cmd.stream_id} NOTIFY" - event(Notified, mailer_id: cmd.payload.mailer_id) - end - - def self.on_exception(exception, _message, group) - if group.error_context[:retry_count] < 3 - later = 5 + 5 * group.error_context[:retry_count] - group.retry(later) - else - group.stop(exception) - end - end - - event ItemAdded do |cart, event| - cart.items << event.payload - end - - event Placed do |cart, _event| - cart.status = :placed - end - - event Notified do |cart, event| - cart.notified = true - cart.mailer_id = event.payload.mailer_id - end - - # This block will run - # in the same transaction as appending - # new events to the store. - # So if either fails, eveything is rolled back. - # ergo, strong consistency. - sync do |command:, events:, state:| - puts "#{self.class.name} #{events.last.seq} SYNC" - end - - # Or register a Reactor interface to react to events - # synchronously - # sync CartListings -end - -class Mailer < Sourced::Actor - EmailSent = Sourced::Message.define('mailer.email_sent') do - attribute :cart_id, String - end - - state do |id| - [] - end - - command :send_email, cart_id: String do |_, cmd| - # Send email here, emit EmailSent if successful - event(EmailSent, cart_id: cmd.payload.cart_id) - end - - event EmailSent do |list, event| - list << event - end -end - -# A Saga that orchestrates the flow between Cart and Mailer -class CartEmailsSaga < Sourced::Actor - # Listen for Cart::Placed events and - # send command to Mailer - reaction Cart::Placed do |event| - dispatch(Mailer::SendEmail, cart_id: event.stream_id).to("mailer-#{event.stream_id}") - end - - # Listen for Mailer::EmailSent events and - # send command to Cart - reaction Mailer::EmailSent do |event| - dispatch(Cart::Notify, mailer_id: event.stream_id).to(event.payload.cart_id) - end -end - -# A projector -# "reacts" to events registered with .evolve -class CartListings < Sourced::Actor - class << self - def handled_events = self.handled_events_for_evolve - - # The Reactor interface - # @param events [Array] - def handle_events(events) - # For this type of event sourced projections - # that load current state from events - # then apply "new" events - # TODO: the current state already includes - # the new events, so we need to load upto events.first.seq - instance = load(events.first.stream_id, upto: events.first.seq - 1) - instance.handle_events(events) - end - end - - def handle_events(events) - evolve(state, events) - save - [] # no commands - end - - def initialize(id, **_args) - super - FileUtils.mkdir_p('examples/carts') - @path = "./examples/carts/#{id}.json" - end - - private def save - backend.transaction do - run_sync_blocks(state, nil, []) - end - end - - def init_state(id) - { id:, items: [], status: :open, seq: 0, seqs: [] } - end - - sync do |cart, _command, _events| - File.write(@path, JSON.pretty_generate(cart)) - end - - # Register all events from Cart - # So that before_evolve runs before all cart events - evolve_all Cart.handled_commands - evolve_all Cart - - before_evolve do |cart, event| - cart[:seq] = event.seq - cart[:seqs] << event.seq - end - - event Cart::Placed do |cart, event| - cart[:status] = :placed - end - - event Cart::ItemAdded do |cart, event| - cart[:items] << event.payload.to_h - end -end - -class LoggingReactor - extend Sourced::Consumer - - class << self - # Register as a Reactor that cares about these events - # The workers will use this to fetch the right events - # and ACK offsets after processing - # - # @return [Array] - def handled_events = [Cart::Placed, Cart::ItemAdded] - - # Workers pass available events to this method - # in order, with exactly-once semantics - # If a list of commands is returned, - # workers will send them to the router - # to be dispatched to the appropriate command handlers. - # - # @param events [Array] - # @option replaying [Boolean] whether this is a replay of events - # @return [Array [reaction] Order placed! #{state.items.size} items, total: #{state.total}" - end -end - -# ============================================================================= -# Demo script -# ============================================================================= -puts "=== LiteCart Demo (SQLite backend) ===\n\n" - -# Create a cart instance with a specific stream ID. -cart, evts = Sourced.load(LiteCart, 'cart-1') -# cart = LiteCart.new(id: 'cart-1') - -# Use bang methods (add_item!) to persist events to the SQLite backend. -puts "--- Adding items ---" -_cmd, events = cart.add_item!(name: 'Notebook', price: 1200) -puts " Added Notebook (1200) -> #{events.map(&:type)}" - -_cmd, events = cart.add_item!(name: 'Pen', price: 300) -puts " Added Pen (300) -> #{events.map(&:type)}" - -_cmd, events = cart.add_item!(name: 'Eraser', price: 50) -puts " Added Eraser (50) -> #{events.map(&:type)}" - -puts "\n--- Removing an item ---" -_cmd, events = cart.remove_item!(name: 'Eraser') -puts " Removed Eraser -> #{events.map(&:type)}" - -puts "\n--- Placing order ---" -_cmd, events = cart.place_order! -puts " Order placed -> #{events.map(&:type)}" - -puts "\n--- Current cart state (in-memory) ---" -puts " status: #{cart.state.status}" -puts " items: #{cart.state.items}" -puts " total: #{cart.state.total}" - -# Read the full event stream back from the database. -puts "\n--- Event stream for cart-1 ---" -events = Sourced.config.backend.read_stream('cart-1') -events.each do |e| - puts " seq:#{e.seq} #{e.type.ljust(30)} #{e.payload.to_h}" -end - -# Reload a fresh cart from persisted events to show state is rebuilt. -puts "\n--- Reload cart from events ---" -reloaded, _events = Sourced.load(LiteCart, 'cart-1') -puts " status: #{reloaded.state.status}" -puts " items: #{reloaded.state.items}" -puts " total: #{reloaded.state.total}" - -puts "\nDone!" diff --git a/examples/pub.rb b/examples/pub.rb deleted file mode 100644 index 4e1bd917..00000000 --- a/examples/pub.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'sequel' -require 'sqlite3' -require 'sourced/pubsub/sqlite' - -PUB = Sourced::PubSub::SQLite.new( - db: Sequel.sqlite('./sourced_pubsub.db'), - serializer: Sourced::PubSub::SQLite::JSONSerializer.new, -) - -CH = PUB.subscribe('foo') diff --git a/examples/socket.rb b/examples/socket.rb deleted file mode 100644 index f1a27ce9..00000000 --- a/examples/socket.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'sourced' -require 'sourced/pubsub/socket' - -Msg = Sourced::Event.define('test') do - attribute :name, String -end - -# Setup -PUB = Sourced::PubSub::Socket.new(socket_dir: '/tmp/sourced_sockets') - -# Publishing -# pubsub.publish('events', some_event) - -# Subscribing -CH = PUB.subscribe('events') -# channel.start do |event, ch| -# # handle event -# end diff --git a/examples/workers.rb b/examples/workers.rb deleted file mode 100644 index 43b4458a..00000000 --- a/examples/workers.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -require_relative './cart' - -Sourced::Supervisor.start diff --git a/lib/sourced.rb b/lib/sourced.rb index 79e94d7c..b3cc79d3 100644 --- a/lib/sourced.rb +++ b/lib/sourced.rb @@ -1,41 +1,20 @@ # frozen_string_literal: true require_relative 'sourced/version' +require 'sourced/types' +require 'sourced/injector' -require 'securerandom' -require 'sourced/message' - -# Sourced is an Event Sourcing / CQRS library for Ruby built around the "Decide, Evolve, React" pattern. -# It provides eventual consistency by default with an actor-like execution model for building -# event-sourced applications. -# -# @example Basic setup with Sequel backend -# Sourced.configure do |config| -# config.backend = Sequel.connect('postgres://localhost/mydb') -# end -# -# @example Register actors and projectors -# Sourced.register(MyActor) -# Sourced.register(MyProjector) -# -# @example Start background workers -# Sourced::Supervisor.start(count: 10) -# -# @see https://github.com/ismasan/sourced +# Sourced is an event-sourcing library for Ruby built around stream-less, +# partition-based consistency. Events go into a flat, globally-ordered log and +# consistency context is assembled dynamically by querying relevant facts via +# key-value pairs extracted from event payloads. module Sourced # Base error class for all Sourced-specific exceptions class Error < StandardError; end - - # Raised when concurrent writes to the same stream are detected + ConcurrentAppendError = Class.new(Error) - - # Raised when concurrent acknowledgments of the same event are detected - ConcurrentAckError = Class.new(Error) - - # Raised when an invalid reactor is registered - InvalidReactorError = Class.new(Error) - - BackendError = Class.new(Error) + UnknownMessageError = Class.new(ArgumentError) + PastMessageDateError = Class.new(ArgumentError) # Raised when a batch is partially processed before a message raises. # Carries the action_pairs for successfully processed messages, @@ -51,190 +30,69 @@ def initialize(action_pairs, failed_message, cause) end end - class InvalidMessageError < Error - attr_reader :message - - def initialize(message) - @message = message - super <<~ERR - Invalid message #{message.class} ('#{message.type}') - Errors: #{message.errors.inspect} - ERR - end + # @return [Configuration] the global Sourced configuration instance + def self.config + @config ||= Configuration.new end - # Generate a new unique stream identifier, optionally with a prefix. - # Stream IDs define concurrency boundaries - events for the same stream ID - # are processed sequentially, while different stream IDs can be processed concurrently. - # - # @param prefix [String, nil] Optional prefix for the stream ID - # @return [String] A new UUID-based stream ID - # @example Generate a simple stream ID - # Sourced.new_stream_id #=> "123e4567-e89b-12d3-a456-426614174000" - # @example Generate a prefixed stream ID - # Sourced.new_stream_id("cart") #=> "cart-123e4567-e89b-12d3-a456-426614174000" - def self.new_stream_id(prefix = nil) - uuid = SecureRandom.uuid - prefix ? "#{prefix}-#{uuid}" : uuid + # Configure the Sourced module. Stores the block for re-running after fork + # (see {.setup!}), then runs it immediately. + # @yieldparam config [Configuration] + def self.configure(&block) + @configure_block = block + setup! end - # Access the global Sourced configuration instance. - # - # @return [Configuration] The current configuration instance - def self.config - @config ||= Configuration.new + # Run (or re-run) the configure block on a fresh Configuration. + # Safe to call after a process fork to re-establish database connections. + def self.setup! + @config = Configuration.new + @configure_block&.call(@config) + @config.setup! + @config.freeze end - # Configure Sourced with backend, error handling, and other settings. - # The configuration is frozen after the block executes to prevent - # accidental modification during runtime. - # - # @yield [config] Yields the configuration object for setup - # @yieldparam config [Configuration] The configuration instance to configure - # @return [Configuration] The frozen configuration - # @example Basic configuration with Sequel - # Sourced.configure do |config| - # config.backend = Sequel.connect(ENV['DATABASE_URL']) - # config.logger = Logger.new(STDOUT) - # end - # @example Configuration with error handling - # Sourced.configure do |config| - # config.backend = Sequel.connect(ENV['DATABASE_URL']) - # config.error_strategy do |s| - # s.retry(times: 3, after: 5) - # s.on_fail { |e, msg| Sentry.capture_exception(e) } - # end - # end - def self.configure(&) - yield config if block_given? + # Register a reactor class with the global router. + def self.register(reactor) config.setup! - config.freeze - config + config.router.register(reactor) end - # Register an Actor or Projector class to make it available for background processing. - # Registered reactors can handle commands and react to events asynchronously. - # - # @param reactor [Class] Actor or Projector class that implements the reactor interface - # @return [void] - # @raise [InvalidReactorError] if the reactor doesn't implement required interface methods - # @example Register an actor - # Sourced.register(CartActor) - # @example Register a projector - # Sourced.register(CartListingsProjector) - # @see Actor - # @see Projector - def self.register(reactor) - Router.register(reactor) + # @return [Sourced::Store] + def self.store + config.setup! + config.store end - # @return [Boolean] - def self.registered?(reactor) - Router.registered?(reactor) + # @return [Sourced::Router] + def self.router + config.setup! + config.router end - # Append messages (probably commands) to a stream - # auto-incrementing the sequence number - # Raises if the message is invalid - # This is meant for an app's front-end to dispatch commands into the system - # so it's defensive and raises on error - # helpers upstream of this can first validate the message - # and surface errors back to the UI - # TODO: in future we might want to restrict what messages - # can be publicly dispatched. Whether that lives here, or - # somewhere else, I'm not sure. - # - # @param message [Message] the mesagge to append - # @raise [InvalidMessageError] if !message.valid? - # @return [Message] - # @example - # command = CreateCart.new(stream_id: 'cart-123', payload: {}), - # Sourced.dispatch(command) - def self.dispatch(message) - raise InvalidMessageError.new(message) unless message.valid? - - appended = config.backend.append_next_to_stream(message.stream_id, [message]) - raise BackendError, "Backend #{config.backend}#append_next_to_stream failed with message #{message.inspect}" unless appended - - message + def self.stop_consumer_group(reactor_or_id, message = nil) + config.router.stop_consumer_group(reactor_or_id, message) end - class Loader - def initialize(backend: Sourced.config.backend) - @backend = backend - end - - def load(actor, after: nil, upto: nil) - after ||= actor.seq - events = @backend.read_stream(actor.id, after:, upto:) - actor.evolve(events) - [actor, events] - end + def self.reset_consumer_group(reactor_or_id) + config.router.reset_consumer_group(reactor_or_id) end - # Load or catch up an Actor from its event history - # @example - # actor = MyActor.new(id: '123') - # Sourced.load(actor) - # actor.seq # Integer - # - # Actor must implement: - # #id() => String - # #seq() => Integer - # #evolve(events) - # - # It also supports passing a Reactor class (Actor, Evolver) - # and a stream_id - # @example - # actor, events = Sourced.load(MyActor, 'order-123') - # actor, events = Sourced.load(MyActor, 'order-123', after: 20) - def self.load(*args) - reactor, options = case args - in [ReactorInterface => r, String => stream_id, Hash => opts] - [r.new(id: stream_id), opts] - in [ReactorInterface => r, String => stream_id] - [r.new(id: stream_id), {}] - in [Evolve => r, Hash => opts] - [r, opts] - in [Evolve => r] - [r, {}] - else - raise ArgumentError, "expected a Reactor class and stream_id, or a Reactor instance, but got #{args.inspect}" - end - - backend = options.delete(:backend) || config.backend - Loader.new(backend:).load(reactor, **options) + def self.start_consumer_group(reactor_or_id) + config.router.start_consumer_group(reactor_or_id) end - # Load history for a reactor, or a stream id string - # @example - # history = Sourced.history_for('order-123') - # history = Sourced.history_for('order-123', upto: 20) - # history = Sourced.history_for(order_actor) - # - # @param stream_id [String, #id, #stream_id] - # @option after [nil, Integer] load messages after this sequence number - # @option upto [nil, Integer] load messsages upto this sequence number - # @return [Enumerable] - def self.history_for(stream_id, after: nil, upto: nil) - stream_id = if stream_id.respond_to?(:stream_id) - stream_id.stream_id - elsif stream_id.respond_to?(:id) - stream_id.id - else - stream_id - end - - config.backend.read_stream(stream_id, after:, upto:) + # Reset the global configuration. For test teardown. + def self.reset! + @config = nil + @configure_block = nil + @topology = nil end - # Build the topology graph from all registered async reactors. - # Returns a flat array of CommandNode, EventNode, and AutomationNode structs. - # The result is memoized; call Sourced.reset_topology to clear. - # - # @return [Array] + # Build and cache the topology graph from all reactors registered with + # the global {.router}. def self.topology - @topology ||= Topology.build(Router.instance.async_reactors) + @topology ||= Topology.build(router.reactors) end def self.reset_topology @@ -242,56 +100,96 @@ def self.reset_topology end # Generate a standardized method name for message handlers. - # Used internally to create consistent handler method names. - # - # @param prefix [String] The handler type prefix (e.g., 'command', 'event') - # @param name [String] The message class name - # @return [String] The generated method name # @api private def self.message_method_name(prefix, name) "__handle_#{prefix}_#{name.split('::').map(&:downcase).join('_')}" end - # @!group Type Interfaces - - # Interface that command handlers (Deciders) must implement. - # @!attribute [r] handled_commands - # @return [Array] Command classes this decider handles - # @!attribute [r] handle_command - # @return [Method] Method to handle incoming commands - # @!attribute [r] on_exception - # @return [Method] Method to handle exceptions during command processing - DeciderInterface = Types::Interface[:handled_commands, :handle_command, :on_exception] - - # Interface that event handlers (Reactors) must implement. - # @!attribute [r] consumer_info - # @return [Sourced::Consumer::ConsumerInfo] Consumer group information for this reactor - # @!attribute [r] handled_messages - # @return [Array] Message classes this reactor handles - # @!attribute [r] handle - # @return [Method] Method to handle incoming events - # @!attribute [r] on_exception - # @return [Method] Method to handle exceptions during event processing - ReactorInterface = Types::Interface[:handle_batch, :consumer_info, :handled_messages, :on_exception] + # Returned by {.handle!} with command, reactor instance, and new events. + HandleResult = Data.define(:command, :reactor, :events) do + def to_ary = [command, reactor, events] + end + + # Handle a command synchronously: validate, load history, decide, append, ACK. + def self.handle!(reactor_class, command, store: nil) + store ||= self.store + + partition_attrs = extract_partition_attrs(command, reactor_class) + values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } + instance = reactor_class.new(values) + + unless command.valid? + return HandleResult.new(command: command, reactor: instance, events: []) + end + + needs_history = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) + if needs_history + instance, read_result = load(reactor_class, store: store, **partition_attrs) + end + + raw_events = instance.decide(command) + correlated_events = raw_events.map { |e| command.correlate(e) } + + guard = read_result&.guard + to_append = [command] + correlated_events + last_position = store.append(to_append, guard: guard) + + advance_registered_offsets(store, reactor_class, partition_attrs, last_position) + + HandleResult.new(command: command, reactor: instance, events: correlated_events) + end + + # Load a reactor instance from its event history using AND-filtered partition reads. + def self.load(reactor_class, store: nil, **values) + store ||= self.store + partition_attrs = reactor_class.partition_keys.to_h { |k| [k, values[k]] } + handled_types = reactor_class.handled_messages_for_evolve.map(&:type).uniq + read_result = store.read_partition(partition_attrs, handled_types:) + instance = reactor_class.new(values) + + instance.evolve(read_result.messages) + + [instance, read_result] + end + + private_class_method def self.extract_partition_attrs(command, reactor_class) + reactor_class.partition_keys.each_with_object({}) do |key, h| + value = command.payload&.respond_to?(key) ? command.payload.send(key) : nil + h[key] = value if value + end + end + + private_class_method def self.advance_registered_offsets(store, reactor_class, partition_attrs, position) + return unless config.router + + partition = partition_attrs.transform_keys(&:to_s) + + config.router.reactors.each do |registered_reactor| + next unless registered_reactor == reactor_class + + store.advance_offset( + registered_reactor.group_id, + partition: partition, + position: position + ) + end + end end -require 'sourced/consumer' +require 'sourced/configuration' +require 'sourced/message' require 'sourced/actions' +require 'sourced/consumer' require 'sourced/evolve' require 'sourced/react' require 'sourced/sync' -require 'sourced/configuration' -require 'sourced/router' -require 'sourced/message' -require 'sourced/actor' -require 'sourced/handler' +require 'sourced/decider' require 'sourced/projector' -require 'sourced/work_queue' -require 'sourced/inline_notifier' -require 'sourced/catchup_poller' +require 'sourced/router' +require 'sourced/worker' +require 'sourced/stale_claim_reaper' require 'sourced/dispatcher' -require 'sourced/supervisor' require 'sourced/command_context' -require 'sourced/unit' require 'sourced/topology' -# require 'sourced/rails/railtie' if defined?(Rails::Railtie) +require 'sourced/supervisor' +require 'sourced/durable_workflow' diff --git a/lib/sourced/actions.rb b/lib/sourced/actions.rb index 6e87bc65..48eae73e 100644 --- a/lib/sourced/actions.rb +++ b/lib/sourced/actions.rb @@ -1,154 +1,140 @@ # frozen_string_literal: true module Sourced - # Actions represent the side effects produced by command/event handlers. - # Each persistable action class implements {#execute}, which correlates messages - # against a source message and persists them via the backend. - # This provides a single place for the correlate-then-persist logic, - # used by both backend reactors and {Sourced::Unit} synchronous processing. + # Action builders and executable action types for reactors. module Actions - # Split a list of messages into - # {AppendNext} or {Schedule} actions - # based on their +created_at+ relative to now. + OK = :ok + RETRY = :retry + + # Split produced messages into immediate append actions and delayed schedule actions. # - # @param messages [Array] - # @return [Array] - def self.build_for(messages) + # @param messages [Sourced::Message, Array] messages produced by a reactor + # @param guard [ConsistencyGuard, nil] optional concurrency guard for immediate appends + # @param source [Sourced::Message, nil] source message used for correlation when executing + # @param correlated [Boolean] whether +messages+ are already correlated + # @return [Array] executable actions in append/schedule groups + def self.build_for(messages, guard: nil, source: nil, correlated: false) actions = [] + messages = Array(messages) return actions if messages.empty? - # TODO: I really need a uniform Clock object now = Time.now - to_schedule, to_append = messages.partition { |e| e.created_at > now } - actions << AppendNext.new(to_append) if to_append.any? - to_schedule.group_by(&:created_at).each do |at, msgs| - actions << Schedule.new(msgs, at:) + to_schedule, to_append = messages.partition { |message| message.created_at > now } + + actions << Append.new(to_append, guard:, source:, correlated:) if to_append.any? + to_schedule.group_by(&:created_at).each do |at, scheduled_messages| + actions << Schedule.new(scheduled_messages, at:, source:, correlated:) end actions end - RETRY = :retry - OK = :ok - - # ACK an arbitrary message ID - Ack = Data.define(:message_id) - - # Append messages to the event store using Backend#append_next_to_stream, - # which auto-increments stream sequence numbers. - # Messages may target different streams and will be grouped by stream_id. - class AppendNext - include Enumerable - - # @return [Array] - attr_reader :messages - - # @param messages [Array] - def initialize(messages) - @messages = messages - freeze - end - - def ==(other) - other.is_a?(self.class) && messages == other.messages - end - - # @yield [stream_id, message] - # @yieldparam stream_id [String] - # @yieldparam message [Sourced::Message] - # @return [Enumerator] if no block given - def each(&block) - return enum_for(:each) unless block_given? - - messages.each do |message| - block.call(message.stream_id, message) - end + # Append messages to the store with optional consistency guard. + # Auto-correlates messages with the source message at execution time. + # + # When +source:+ is provided, it overrides the runtime's source_message + # for correlation (e.g. reactions correlated with the event, not the command). + # + # When +correlated: true+, messages are assumed to be already correlated + # and are appended as-is without re-correlation. + class Append + attr_reader :messages, :guard, :source + + # @param messages [Sourced::Message, Array] messages to append + # @param guard [ConsistencyGuard, nil] optional optimistic concurrency guard + # @param source [Sourced::Message, nil] explicit correlation source + # @param correlated [Boolean] whether +messages+ are already correlated + def initialize(messages, guard: nil, source: nil, correlated: false) + @messages = Array(messages) + @guard = guard + @source = source + @correlated = correlated end - # Correlate messages against the source, then persist via backend. - # - # @param backend [#append_next_to_stream] the storage backend - # @param source_message [Sourced::Message] message that caused this action - # @return [Array] correlated messages that were persisted - def execute(backend, source_message) - correlated = messages.map { |m| source_message.correlate(m) } - correlated.group_by(&:stream_id).each do |stream_id, stream_messages| - backend.append_next_to_stream(stream_id, stream_messages) + # @return [Boolean] whether messages should be appended without re-correlation + def correlated? = @correlated + + # @param store [Sourced::Store] + # @param source_message [Sourced::Message] default message to correlate from + # @return [Array] correlated messages that were appended + def execute(store, source_message) + to_append = if @correlated + messages + else + correlate_from = @source || source_message + messages.map { |m| correlate_from.correlate(m) } end - correlated + store.append(to_append, guard: guard) + to_append end end - # Append messages to a specific stream in the event store, - # expecting messages to be in order and with correct sequence numbers. - # The backend will raise {Sourced::ConcurrentAppendError} if messages - # with the same sequence already exist (optimistic concurrency control). - class AppendAfter - # @return [String] - attr_reader :stream_id - # @return [Array] - attr_reader :messages - - # @param stream_id [String] - # @param messages [Array] - def initialize(stream_id, messages) - @stream_id = stream_id - @messages = messages + # Schedule messages for future promotion into the main log. + class Schedule + attr_reader :messages, :at, :source + + # @param messages [Sourced::Message, Array] messages to schedule + # @param at [Time] when the messages should become available for promotion + # @param source [Sourced::Message, nil] explicit correlation source + # @param correlated [Boolean] whether +messages+ are already correlated + def initialize(messages, at:, source: nil, correlated: false) + @messages = Array(messages) + @at = at + @source = source + @correlated = correlated end - # Correlate messages against the source, then persist via backend. - # - # @param backend [#append_to_stream] the storage backend - # @param source_message [Sourced::Message] message that caused this action - # @return [Array] correlated messages that were persisted - def execute(backend, source_message) - correlated = messages.map { |m| source_message.correlate(m) } - backend.append_to_stream(stream_id, correlated) - correlated + # @return [Boolean] whether messages should be scheduled without re-correlation + def correlated? = @correlated + + # @param store [Sourced::Store] + # @param source_message [Sourced::Message] default message to correlate from + # @return [Array] correlated messages that were scheduled + def execute(store, source_message) + to_schedule = if @correlated + messages + else + correlate_from = @source || source_message + messages.map { |message| correlate_from.correlate(message) } + end + store.schedule_messages(to_schedule, at: at) + to_schedule end end - # Schedule messages for future delivery at a specific time. - class Schedule - # @return [Array] - attr_reader :messages - # @return [Time] - attr_reader :at - - # @param messages [Array] - # @param at [Time] when the messages should become available - def initialize(messages, at:) - @messages, @at = messages, at + # Execute a synchronous side effect within the current transaction. + class Sync + # @param work [#call] callable to execute + def initialize(work) + @work = work end - # Correlate messages against the source, then schedule via backend. - # - # @param backend [#schedule_messages] the storage backend - # @param source_message [Sourced::Message] message that caused this action - # @return [Array] correlated messages that were scheduled - def execute(backend, source_message) - correlated = messages.map { |m| source_message.correlate(m) } - backend.schedule_messages(correlated, at: at) - correlated + # @return [Object] the callable's return value + def call = @work.call + + # @param _store [Object] unused + # @param _source_message [Object] unused + # @return [nil] + def execute(_store, _source_message) + call + nil end end - # Execute a synchronous side effect (e.g. cache write, HTTP call) - # within the current transaction. Does not persist messages. - class Sync + # Execute a side effect after the transaction commits. + class AfterSync # @param work [#call] callable to execute def initialize(work) @work = work end - def call(...) = @work.call(...) + # @return [Object] the callable's return value + def call = @work.call - # Execute the work block. Backend and source_message are ignored. - # - # @param _backend [Object] unused + # @param _store [Object] unused # @param _source_message [Object] unused # @return [nil] - def execute(_backend, _source_message) + def execute(_store, _source_message) call nil end diff --git a/lib/sourced/actor.rb b/lib/sourced/actor.rb deleted file mode 100644 index c7d445ed..00000000 --- a/lib/sourced/actor.rb +++ /dev/null @@ -1,386 +0,0 @@ -# frozen_string_literal: true - -module Sourced - class Actor - include Evolve - include React - include Sync - extend Consumer - - PREFIX = 'decide' - - UndefinedMessageError = Class.new(KeyError) - Error = Class.new(StandardError) - - class DualMessageRegistrationError < Error - def initialize(msg_type, handled_by) - msg = if handled_by == :reaction - <<~MSG - Message #{msg_type} is already registered to be handled by .reaction(#{msg_type}). - Sourced::Actor classes can only handle the same message type either as a reaction, or a command. - MSG - else - <<~MSG - Message #{msg_type} is already registered to be handled by .command(#{msg_type}). - Sourced::Actor classes can only handle the same message type either as a reaction, or a command. - MSG - end - super msg - end - end - - class DifferentStreamError < Error - def initialize(actor, event) - super <<~MSG - Actor instance #{actor.inspect} was initialized with id = '#{actor.id}', - but it was evolved with an event for stream_id '#{event.stream_id}', - - The event is #{event.inspect} - MSG - end - end - - class SmallerSequenceError < Error - def initialize(actor, event) - super <<~MSG - Actor instance #{actor.inspect} is currently at sequence number #{actor.seq}, - but it was evolved with an event at sequence #{event.seq}. - - The event is #{event.inspect} - MSG - end - end - - # An Actor class has its own Command and Event - # subclasses that are used to define inine commands and events. - # These classes serve as message registry for the Actor's inline messages. - class Command < Sourced::Command; end - class Event < Sourced::Event; end - - BLANK_HISTORY = [].freeze - - class << self - def inherited(subclass) - super - subclass.const_set(:Command, Class.new(const_get(:Command))) - subclass.const_set(:Event, Class.new(const_get(:Event))) - handled_commands.each do |cmd_type| - subclass.handled_commands << cmd_type - end - end - - # Access a Actor's Command or Event classes by name (e.g. :some_command or :some_event) - # @param message_name [Symbol] - # @return [Class] - # @raise [ArgumentError] if the message is not defined - def resolve_message_class(message_name) - message_type = __message_type(message_name) - msg_class = self::Event.registry[message_type] || self::Command.registry[message_type] - - raise UndefinedMessageError, "Message not found: #{message_name}" unless msg_class - msg_class - end - - alias [] resolve_message_class - - # Interface expected by React::StreamDispatcher - # so that this works in reaction blocks - # @example - # stream = stream_for(SomeActor) - # stream.command :do_something - # - # @return [String] - def stream_id - Sourced.new_stream_id - end - - def handled_commands - @handled_commands ||= [] - end - - # Register as a Reactor - # @return [Array] - def handled_messages = self.handled_commands + self.handled_messages_for_react - - # Define a command class, register a command handler - # and define a method to send the command - # Example: - # command :add_item, name: String do |state, cmd| - # event(ItemAdded, item_id: SecureRandom.uuid, name: cmd.payload.name) - # end - # - # # The exmaple above will define a command class `AddItem` in the current namespace: - # AddItem = Message.define('namespace.add_item', payload_schema: { name: String }) - # - # Payload schema is a Plumb Hash schema. - # See: https://github.com/ismasan/plumb#typeshash - # - def command(*args, &block) - raise ArgumentError, 'command block expects signature (state, command)' unless block.arity == 2 - - case args - in [Symbol => cmd_name, Hash => payload_schema] - __register_named_command_handler(cmd_name, payload_schema, &block) - in [Symbol => cmd_name] - __register_named_command_handler(cmd_name, &block) - in [Class => cmd_type] if cmd_type < Sourced::Message - __register_class_command_handler(cmd_type, &block) - else - raise ArgumentError, "Invalid arguments for #{self}.command" - end - end - - # Support defining event handlers with a symbol and a payload schema - # Or a class. - # - # @example - # - # event SomethingHappened, field1: String do |state, event| - # state[:status] = 'done' - # end - # - # event :something_happened, field1: String do |state, event| - # state[:status] = 'done' - # end - # - def event(*args, &block) - case args - in [Symbol => event_name, Hash => payload_schema] - __register_named_event_handler(event_name, payload_schema).tap do |event_class| - super(event_class, &block) - end - in [Symbol => event_name] - __register_named_event_handler(event_name).tap do |event_class| - super(event_class, &block) - end - in [Class => foo] - super - else - raise ArgumentError, "event expects a Symbol or Event class. Got: #{args.inspect}" - end - end - - # The Reactor interface - # Message can be: - # - A command, present in .handled_commands - # - A reaction, present in .handled_messages_for_react - # @param message [Sourced::Message] - # @option history [Enumerable] past messages in the stream - # @return [Sourced::Actions] - def handle(message, history: [], replaying: false) - instance = new(id: identity_from(message)) - instance.handle(message, history:, replaying:) - end - - # Batch processing for actors. Per-message iteration internally. - # Replay messages return OK immediately. Live messages create instance and handle. - # @param batch [Array<[Message, Boolean]>] array of [message, replaying] pairs - # @param history [Array] full stream history - # @return [Array<[actions, source_message]>] action pairs - def handle_batch(batch, history: BLANK_HISTORY) - each_with_partial_ack(batch) do |message, replaying| - if replaying - [Actions::OK, message] - else - instance = new(id: identity_from(message)) - actions = instance.handle(message, history:) - [actions, message] - end - end - end - - def __message_type(msg_name) - [__message_type_prefix, msg_name].join('.').downcase - end - - # Override this in subclasses - # to make an actor take it's @id from an arbitrary - # field in the message - # TODO: the danger here is that not all messages might - # support this arbitrary field. - # it would be nice if we could statically - # analyse messages handled by the Actor, and raise an error - # if not all of them support the specified field - # - # @param message [Sourced::Message] - # @return [Object] - def identity_from(message) = message.stream_id - - private - - def __register_named_command_handler(cmd_name, payload_schema = nil, &block) - cmd_class = self::Command.define(__message_type(cmd_name), payload_schema:) - klass_name = cmd_name.to_s.split('_').map(&:capitalize).join - const_set(klass_name, cmd_class) - __register_class_command_handler(cmd_class, &block) - end - - def __register_class_command_handler(cmd_type, &block) - raise DualMessageRegistrationError.new(cmd_type, :reaction) if handled_messages_for_react.include?(cmd_type) - - handled_commands << cmd_type - define_method(Sourced.message_method_name(PREFIX, cmd_type.name), &block) - end - - def __register_named_event_handler(event_name, payload_schema = nil) - klass_name = event_name.to_s.split('_').map(&:capitalize).join - event_class = self::Event.define(__message_type(event_name), payload_schema:) - const_set(klass_name, event_class) - end - - # Override this method defined in React mixin. - # Given a symbol for a message type, resolve the message class - # @return [Class, nil] - def __resolve_message_class(message_symbol) - self::Event.registry[__message_type(message_symbol)] - end - - # Override the default namespace for commands and events - # defined inline - # @example - # - # def message_namespace = 'my_app.messages.' - # - # @return [String] - def message_namespace - Types::ModuleToMessageType.parse(name.to_s) - end - - def __message_type_prefix - @__message_type_prefix ||= message_namespace - end - - # Override the no-op hook in Sourced::React - # We want to make sure that a message handled as a command - # cannot also be registered as a reaction. - # These are the command/event semantics that Actor adds on top - # of the underlying messaging infrastructure. - def __validate_message_for_reaction!(event_class) - raise DualMessageRegistrationError.new(event_class, :command) if handled_commands.include?(event_class) - end - end - - # Instance methods - - attr_reader :id, :seq, :uncommitted_events - - def initialize(id: Sourced.new_stream_id, logger: Sourced.config.logger) - @id = id - @seq = 0 - @uncommitted_events = [] - @logger = logger - @__current_command = Sourced::Command.new(stream_id: id) - end - - def inspect - %(<#{self.class} id:#{id} seq:#{seq}>) - end - - def handle(message, history:, replaying: false) - return Actions::OK if replaying - - evolve(history) - if handles_command?(message) - events = decide(message) - actions = [Actions::AppendAfter.new(id, events)] - actions + sync_actions_with(command: message, events:, state:) - elsif reacts_to?(message) - Actions.build_for(react(message)) - else - Actions::OK - end - end - - # Does this actor handle this message as a command? - # TODO: ATM I'm just doing .handled_commands.include? - # it would be more efficient to have an O(1) lookup - def handles_command?(message) - self.class.handled_commands.include?(message.class) - end - - # Route a command to its defined command handler, and run it. - # @param command [Sourced::Command] - # @return [Array] - def decide(command) - command = __set_current_command(command) - send(Sourced.message_method_name(PREFIX, command.class.name), state, command) - @__current_command = nil - @uncommitted_events.slice!(0..) - end - - # Apply an event from within a command handler - # @example - # - # command DoSomething do |state, cmd| - # event SomethingHappened, field1: 'foo', field2: 'bar' - # end - # - # Or, with symbol pointing to an event class defined with .event - # command DoSomething do |state, cmd| - # event :something_happened, field1: 'foo', field2: 'bar' - # end - # - # @param event_name [Symbol, Class] the event name or class - # @param payload [Hash] the event payload - # @return [Any] the - def event(event_name, payload = {}) - return apply(event_name, payload) unless event_name.is_a?(Symbol) - - event_class = Event.registry[self.class.__message_type(event_name)] - raise ArgumentError, "Event not found: #{event_name}" unless event_class - - apply(event_class, payload) - end - - private - - attr_reader :__current_command, :logger - - # Override Evolve#__update_on_evolve - def __update_on_evolve(event) - raise DifferentStreamError.new(self, event) if id != event.stream_id - raise SmallerSequenceError.new(self, event) if seq >= event.seq - - @seq = event.seq - end - - # TODO: in the new arch, commands - # already exist in the event stream - # so we don't append them again as part of uncommitted_events - # and we don't increment @seq - # However, when handling commands asynchronously, - # we DO also want to append the command with the produced events - # Think about this later. - def __set_current_command(command) - @__current_command = command - end - - def __next_sequence - @seq + 1 - end - - # Instantiate an event class and apply it to the state - # by running registered evolve blocks. - # Also store the event in the uncommitted events list, - # and keep track of the sequence number. - # To be used inside a .decide block. - # @example - # - # apply SomeEvent, field1: 'foo', field2: 'bar' - # - # @param event_class [Sourced::Event] - # @param payload [Hash] the event payload - # @return [Any] the new state - def apply(event_class, payload = {}) - evt = __current_command.follow_with_attributes( - event_class, - attrs: { seq: __next_sequence }, - # TODO: the infra sets this now - # metadata: { producer: self.class.consumer_info.group_id }, - payload: - ) - uncommitted_events << evt - evolve([evt]) - end - end -end diff --git a/lib/sourced/backends/pg_backend.rb b/lib/sourced/backends/pg_backend.rb deleted file mode 100644 index 2f3fc63b..00000000 --- a/lib/sourced/backends/pg_backend.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/backends/sequel_backend' - -module Sourced - module Backends - # Explicit PostgreSQL backend. Inherits all behaviour from SequelBackend - # and overrides only the adapter setup to skip auto-detection. - # - # @example - # db = Sequel.postgres('myapp') - # backend = Sourced::Backends::PGBackend.new(db) - class PGBackend < SequelBackend - private - - def setup_adapter - @notifier = PGNotifier.new(db: @db) - end - end - end -end diff --git a/lib/sourced/backends/sequel_backend.rb b/lib/sourced/backends/sequel_backend.rb deleted file mode 100644 index 3eba63b9..00000000 --- a/lib/sourced/backends/sequel_backend.rb +++ /dev/null @@ -1,980 +0,0 @@ -# frozen_string_literal: true - -require 'sequel' -require 'json' -require 'sourced/message' -require 'sourced/inline_notifier' - -Sequel.extension :pg_json if defined?(PG) - -module Sourced - module Backends - # Production backend implementation using Sequel ORM for PostgreSQL and SQLite. - # This backend provides persistent storage for messages, and consumer state - # with support for concurrency control, pub/sub notifications, and transactional processing. - # - # The SequelBackend handles: - # - Message storage and retrieval with ordering guarantees - # - Message scheduling and dispatching - # - Consumer group management and offset tracking - # - Concurrency control via database locks and claims - # - Pub/sub notifications for real-time message processing - # - # @example Basic setup with PostgreSQL - # db = Sequel.connect('postgres://localhost/myapp') - # backend = Sourced::Backends::SequelBackend.new(db) - # backend.install unless backend.installed? - # - # @example Configuration in Sourced - # Sourced.configure do |config| - # config.backend = Sequel.connect(ENV['DATABASE_URL']) - # end - class SequelBackend - # Consumer group status indicating active processing - ACTIVE = 'active' - # Consumer group status indicating stopped processing - STOPPED = 'stopped' - # Consumer group status indicating failed processing - FAILED = 'failed' - # Pre-allocated return for empty/failed batch claims - NO_BATCH = [nil, nil].freeze - - # Initialize a new Sequel backend instance. - # Automatically sets up database connection, table names, and pub/sub system. - # - # @param db [Sequel::Database, String] Database connection or connection string - # @param logger [Object] Logger instance for backend operations (defaults to configured logger) - # @param prefix [String] Table name prefix for Sourced tables (defaults to 'sourced') - # @param schema [String, nil] Optional PostgreSQL schema name. When set, all tables are - # created and queried within this schema (e.g. "my_app".sourced_messages). - # The schema is created automatically on #install if it doesn't exist. - # Defaults to nil (uses the database's default search_path, typically "public"). - # @example Initialize with existing connection - # backend = SequelBackend.new(Sequel.connect('postgres://localhost/mydb')) - # @example Initialize with custom prefix - # backend = SequelBackend.new(db, prefix: 'my_app') - # @example Initialize with a PostgreSQL schema - # backend = SequelBackend.new(db, schema: 'my_app') - # @example Combine schema and prefix - # backend = SequelBackend.new(db, schema: 'my_app', prefix: 'evt') - # # => tables: my_app.evt_messages, my_app.evt_streams, etc. - def initialize(db, logger: Sourced.config.logger, prefix: 'sourced', schema: nil) - @db = connect(db) - @logger = logger - @prefix = prefix - @schema = schema - setup_adapter - setup_tables - @setup = false - logger.info("Connected to #{@db}") - end - - # Data structure for backend statistics and monitoring information. - # @!attribute [r] stream_count - # @return [Integer] Total number of message streams - # @!attribute [r] max_global_seq - # @return [Integer] Highest global sequence number (latest message) - # @!attribute [r] groups - # @return [Array] Consumer group information with processing stats - Stats = Data.define(:stream_count, :max_global_seq, :groups) - - # Get comprehensive statistics about the backend state. - # Useful for monitoring, debugging, and understanding system health. - # - # @return [Stats] Statistics object with stream counts, sequences, and group info - # @example Get backend statistics - # stats = backend.stats - # puts "Total streams: #{stats.stream_count}" - # puts "Latest message: #{stats.max_global_seq}" - # stats.groups.each { |g| puts "Group #{g[:id]}: #{g[:status]}" } - def stats - stream_count = db[streams_table].count - max_global_seq = db[messages_table].max(:global_seq) - groups = db.fetch(sql_for_consumer_stats).all - Stats.new(stream_count, max_global_seq, groups) - end - - # Data structure representing a stream with its metadata. - # @!attribute [r] stream_id - # @return [String] Unique identifier for the stream - # @!attribute [r] seq - # @return [Integer] Latest sequence number in the stream - # @!attribute [r] updated_at - # @return [Time] Timestamp of the most recent message in the stream - Stream = Data.define(:stream_id, :seq, :updated_at) - - # Retrieve a list of recently active streams, ordered by most recent activity. - # This method is useful for diagnostics, monitoring, and debugging to understand - # which streams have been most active in the system. - # - # The query is optimized with a database index on updated_at for efficient sorting - # and uses LIMIT to minimize data transfer. - # - # @param limit [Integer] Maximum number of streams to return (defaults to 10, must be >= 0) - # @return [Array] Array of Stream objects ordered by updated_at descending - # @raise [ArgumentError] if limit is negative - # @example Get the 5 most recently active streams - # recent = backend.recent_streams(limit: 5) - # recent.each do |stream| - # puts "Stream #{stream.stream_id}: seq #{stream.seq} at #{stream.updated_at}" - # end - # @example Monitor system activity - # streams = backend.recent_streams(limit: 20) - # active_count = streams.count { |s| s.updated_at > 1.hour.ago } - # puts "#{active_count} streams active in last hour" - def recent_streams(limit: 10) - # Input validation - return [] if limit == 0 - raise ArgumentError, "limit must be a positive integer" if limit < 0 - - # Optimized query with index on updated_at - query = db[streams_table] - .select(:stream_id, :seq, :updated_at) - .reverse(:updated_at) # More idiomatic Sequel than order(Sequel.desc(:updated_at)) - .limit(limit) - - query.map do |row| - Stream.new(**row) - end - end - - # Returns the backend's notifier for real-time message dispatch. - # Returns a {PGNotifier} for PostgreSQL, {InlineNotifier} otherwise. - # Initialized eagerly in the constructor for thread and CoW safety. - # - # @return [PGNotifier, InlineNotifier] - attr_reader :notifier - - def transaction(&) - db.transaction(&) - end - - def schedule_messages(messages, at:) - return false if messages.empty? - - now = Time.now - rows = messages.map do |m| - message_data = m.to_h - message_data[:metadata] = message_data[:metadata].merge(scheduled_at: now) - { created_at: now, available_at: at, message: JSON.dump(message_data) } - end - - transaction do - db[scheduled_messages_table].multi_insert(rows) - end - - true - end - - def update_schedule! - now = Time.now - - transaction do - rows = db[scheduled_messages_table] - .where { available_at <= now } - .order(:id) - .limit(100) - .for_update - .skip_locked - .all - - return 0 if rows.empty? - - # Process all messages - messages = rows.map do |r| - data = JSON.parse(r[:message], symbolize_names: true) - data[:created_at] = now - Message.from(data) - end - - # Append messages to each stream - messages.group_by(&:stream_id).each do |stream_id, stream_messages| - append_next_to_stream(stream_id, stream_messages) - end - - # Batch delete all processed messages - row_ids = rows.map { |m| m[:id] } - db[scheduled_messages_table].where(id: row_ids).delete - - rows.size - end - end - - # Record heartbeats for a set of worker IDs. - # Accepts a String or an Array. Returns number of rows touched. - def worker_heartbeat(worker_ids, at: Time.now) - ids = Array(worker_ids).uniq - return 0 if ids.empty? - - rows = ids.map { |id| { id:, last_seen: at } } - db.transaction do - db[workers_table] - .insert_conflict(target: :id, update: { last_seen: at }) - .multi_insert(rows) - end - ids.size - end - - # Release stale claims for workers that have not heartbeated within ttl_seconds. - # Returns the number of offsets rows updated. - def release_stale_claims(ttl_seconds: 120) - cutoff = Time.now - ttl_seconds - - db.transaction do - stale_workers = db[workers_table] - .where { last_seen <= cutoff } - .select(:id) - - ds = db[offsets_table] - .where(claimed: true) - .where { claimed_at <= cutoff } - .where(claimed_by: stale_workers) - - ds.update(claimed: false, claimed_at: nil, claimed_by: nil) - end - end - - # Append one or more messages to a stream, creating the stream if it doesn't exist. - # Also automatically increments the sequence number for the stream and messages, - # without relying on the client passing the right sequence numbers. - # This is used for appending messages in order without client-side optimistic concurrency control. - # For example commands that will be picked up and handled by a reactor, later. - # @param stream_id [String] Unique identifier for the message stream - # @param messages [Sourced::Message, Array] Message(s) to append to the stream - # @option max_retries [Integer] Maximum number of retries on unique constraint violation (default: 3) - def append_next_to_stream(stream_id, messages, max_retries: 3) - # Handle both single message and array of messages - messages_array = Array(messages) - return true if messages_array.empty? - - retries = 0 - begin - db.transaction do - # Update or create a stream. - # If updating, increment seq by the number of messages being added. - messages_count = messages_array.size - stream_record = db[streams_table] - .insert_conflict( - target: :stream_id, - update: { - seq: Sequel.qualify(streams_table, :seq) + messages_count, - updated_at: Time.now - } - ) - .returning(:id, :seq) - .insert(stream_id:, seq: messages_count) - .first - - # Calculate the starting sequence number for this batch - # If stream existed, seq will be the new max seq after increment - # If stream was created, seq will be messages_count - starting_seq = stream_record[:seq] - messages_count + 1 - - # Prepare rows for bulk insert with incrementing sequence numbers - rows = messages_array.map.with_index do |msg, index| - row = serialize_message(msg, stream_record[:id]) - row[:seq] = starting_seq + index - row - end - - # Bulk insert all messages - db[messages_table].multi_insert(rows) - - # NOTIFY delivered on commit — atomically correct - notifier.notify_new_messages(messages_array.map(&:type)) - end - - true - rescue Sequel::UniqueConstraintViolation => e - retries += 1 - if retries <= max_retries - sleep(0.001 * retries) # Brief backoff - retry - else - raise Sourced::ConcurrentAppendError, e.message - end - end - end - - def append_to_stream(stream_id, messages) - # Handle both single message and array of messages - messages_array = Array(messages) - return false if messages_array.empty? - - if messages_array.map(&:stream_id).uniq.size > 1 - raise ArgumentError, 'Messages must all belong to the same stream' - end - - db.transaction do - seq = messages_array.last.seq - id = db[streams_table].insert_conflict(target: :stream_id, update: { seq:, updated_at: Time.now }).insert(stream_id:, seq:) - rows = messages_array.map { |e| serialize_message(e, id) } - db[messages_table].multi_insert(rows) - - notifier.notify_new_messages(messages_array.map(&:type)) - end - - true - rescue Sequel::UniqueConstraintViolation => e - raise Sourced::ConcurrentAppendError, e.message - end - - # Reserve next message(s) for a reactor, based on the reactor's #handled_messages list. - # This fetches the next un-acknowledged message(s) for the reactor, and processes them in a transaction - # which acquires a lock on the message's stream_id and the reactor's group_id. - # So that no other reactor instance in the same group can process the same stream concurrently. - # This way, messages for the same stream are guaranteed to be processed sequentially for each reactor group. - # - # When batch_size > 1, fetches multiple messages from the same stream in a single - # lock cycle, reducing per-message overhead for catch-up scenarios. - # - # The block is yielded once with the full batch and optional history. - # All-or-nothing ACK semantics: on success all actions are executed and the last - # message is ACKed; on RETRY the offset is released without ACK. - # - # @example - # backend.reserve_next_for_reactor(reactor, batch_size: 50) do |batch, history| - # # batch is Array of [message, replaying] pairs - # reactor.handle_batch(batch, history:) - # end - # - # @param reactor [Sourced::ReactorInterface] - # @param batch_size [Integer] Number of messages to fetch per lock cycle (default: 1) - # @param with_history [Boolean] Whether to fetch full stream history - # @param worker_id [String] - # @yieldparam batch [Array<[Sourced::Message, Boolean]>] array of [message, replaying] pairs - # @yieldparam history [Array, nil] full stream history if with_history is true - # @yieldreturn [Array<[actions, source_message]>, Sourced::Actions::RETRY] action pairs or RETRY - def reserve_next_for_reactor(reactor, batch_size: 1, with_history: false, worker_id: nil, &block) - worker_id ||= [Process.pid, Thread.current.object_id, Fiber.current.object_id].join('-') - group_id = reactor.consumer_info.group_id - handled_messages = reactor.handled_messages.map(&:type).uniq - now = Time.now - - bootstrap_offsets_for(group_id) - - start_from = reactor.consumer_info.start_from.call - rows, history = claim_and_fetch_batch(group_id, handled_messages, now, start_from, worker_id, batch_size, with_history:) - return unless rows - - reserve_batch(group_id, rows, history:, &block) - end - - private def release_offset(offset_id) - db[offsets_table].where(id: offset_id).update(claimed: false, claimed_at: nil, claimed_by: nil) - end - - # Process a batch of claimed messages through the reactor and handle the result. - # - # This is the core processing loop for batch message handling. It follows a - # yield-once contract: the caller (Router) receives the full batch and returns - # action pairs describing what side effects to execute. - # - # ## Flow - # - # 1. Deserialize raw DB rows into [Message, replaying] pairs - # 2. Yield the batch + optional history to the caller (reactor.handle_batch) - # 3. Handle the return value: - # - Actions::RETRY → release the claimed offset, no ACK (message will be retried) - # - Array of [actions, source_message] pairs → execute side effects and ACK - # - Empty array → release offset (nothing to do, avoid stuck claim) - # - # ## ACK semantics - # - # All-or-nothing per batch. On success, all action pairs are executed in a single - # DB transaction and the offset is advanced to the last message in the batch. - # ack_message() both advances the offset AND releases the claim in one update. - # - # Action pairs don't map 1:1 to batch messages. For example, a Projector returns - # one [sync_actions, last_msg] pair for the whole batch, plus separate - # [reaction_actions, triggering_msg] pairs. We ACK based on the last row in the - # batch regardless of which action pair triggered it. - # - # ## Error handling - # - # Any exception releases the claimed offset (so the batch can be retried by - # another worker) and re-raises for the Router to handle via on_exception. - # - # @param group_id [String] consumer group ID - # @param rows [Array] raw DB rows from claim_and_fetch_batch - # @param history [Array, nil] deserialized stream history, if requested - # @yield [batch, history] yields once to the caller for processing - # @return [Message, nil] the first message in the batch (used by Router for logging) - private def reserve_batch(group_id, rows, history: nil, &block) - first_row = rows.first - - begin - batch = rows.map { |row| [deserialize_message(row), row[:replaying]] } - - action_pairs = yield(batch, history) - - # RETRY: release offset without ACK. The batch will be picked up again. - if action_pairs == Actions::RETRY - release_offset(first_row[:offset_id]) - return batch.first&.first - end - - # Execute all side effects and ACK in a single transaction. - # This ensures actions (AppendNext, Sync, etc.) and the ACK are atomic. - db.transaction do - last_ack_row = nil - action_pairs.each do |(actions, source_message)| - should_ack = execute_actions(group_id, actions, source_message) - if should_ack - # Find the row matching this action_pair's source_message. - # For partial batches (PartialBatchError), this ACKs up to the - # last successfully processed message rather than the last batch row. - last_ack_row = rows.find { |r| r[:id] == source_message.id } || rows.last - end - end - - if last_ack_row - # Advance offset to last successfully processed message and release the claim - ack_message(group_id, last_ack_row[:stream_id_fk], last_ack_row[:global_seq]) - else - # No action pair triggered an ACK (e.g. empty action_pairs). - # Release the claim so the batch isn't stuck. - release_offset(first_row[:offset_id]) - end - end - - batch.first&.first - - rescue StandardError - # Release claim on any error so another worker can pick up the batch - release_offset(first_row[:offset_id]) - raise - end - end - - # Atomically claim an offset and fetch up to batch_size messages in a single CTE query. - # The CTE finds the first matching message (to identify the stream), claims the offset, - # and fetches all pending messages from that stream in one round-trip. - private def claim_and_fetch_batch(group_id, handled_messages, now, start_from, worker_id, batch_size, with_history: false) - sql = if start_from.is_a?(Time) - sql_for_claim_and_fetch_batch(handled_messages, group_id:, now:, worker_id:, batch_size:, start_from:, with_history:) - else - sql_for_claim_and_fetch_batch(handled_messages, group_id:, now:, worker_id:, batch_size:, with_history:) - end - - all_rows = db.fetch(sql).all - return NO_BATCH if all_rows.empty? - - if with_history - batch_rows = all_rows.select { |r| r[:in_batch] } - if batch_rows.empty? - # Claimed but no batch messages — release the claim - release_offset(all_rows.first[:offset_id]) - return NO_BATCH - end - # Deserialize all rows as history (also parses JSON for batch rows since they share objects) - history = all_rows.map { |row| deserialize_message(row) } - [batch_rows, history] - else - [all_rows, nil] - end - rescue Sequel::UniqueConstraintViolation - logger.debug "Batch claim for group #{group_id} already exists, skipping" - NO_BATCH - end - - # Execute action side effects without ACKing. - # Returns true if the message should be ACKed by the caller. - private def execute_actions(group_id, actions, message) - should_ack = false - actions = [actions] unless actions.is_a?(Array) - return true if actions.empty? # empty = implicit ACK - - actions.each do |action| - case action - when nil, Actions::OK - should_ack = true - - when Actions::Ack - ack_on(group_id, action.message_id) - - when Actions::RETRY - # Should not reach here (filtered by batch loop) - - else - action.execute(self, message) - should_ack = true - end - end - - should_ack - end - - # Insert missing offsets for a consumer group and all streams. - # This runs in advance of claiming messages for processing - # so that the claim query relies on existing offsets and FOR UPDATE SKIP LOCKED - # to prevent concurrent claims on the same stream by different workers. - private def bootstrap_offsets_for(group_id) - now = Time.now - db.transaction do - group = db[consumer_groups_table].where(group_id:, status: ACTIVE).first - return if group.nil? - - # Find streams that don't have offsets for this group - missing_streams = db[streams_table] - .left_join(offsets_table, - stream_id: :id, - group_id: group[:id]) - .where(Sequel[offsets_table][:id] => nil) - .select(Sequel[streams_table][:id], Sequel[streams_table][:stream_id]) - .limit(100) - - # Prepare offset records for bulk insert - offset_records = missing_streams.map do |stream| - { - group_id: group[:id], - stream_id: stream[:id], - global_seq: 0, - claimed: false, - created_at: now - } - end - - if offset_records.any? - db[offsets_table].insert_conflict.multi_insert(offset_records) - end - - offset_records.size - end - end - - - def register_consumer_group(group_id) - db[consumer_groups_table] - .insert_conflict(target: :group_id, update: nil) - .insert(group_id:, status: ACTIVE) - end - - # Fetch and update a consumer group in a transaction - # Used by Router to stop / retry consumer groups - # when handing exceptions. - # - # @example - # backend.updating_consumer_group(group_id) do |group| - # group.stop('some reason') - # end - # - # @param group_id [String] - # @yield [GroupUpdater] - def updating_consumer_group(group_id, &) - dataset = db[consumer_groups_table].where(group_id:) - group_row = dataset.for_update.first - raise ArgumentError, "Consumer group #{group_id} not found" unless group_row - - ctx = group_row[:error_context] ? parse_json(group_row[:error_context]) : {} - group_row[:error_context] = ctx - group = GroupUpdater.new(group_id, group_row, logger) - yield group - updates = group.updates - updates[:error_context] = JSON.dump(updates[:error_context]) - dataset.update(updates) - end - - # Start a consumer group that has been stopped or failed. - # Signals the notifier so workers pick up the reactor immediately. - # - # @param group_id [String] - def start_consumer_group(group_id) - group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) - dataset = db[consumer_groups_table].where(group_id: group_id) - dataset.update(status: ACTIVE, retry_at: nil, error_context: nil) - notifier.notify_reactor_resumed(group_id) - end - - # @param group_id [String] - # @param message [#to_s, NilClass] - def stop_consumer_group(group_id, message = nil) - group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) - updating_consumer_group(group_id) do |group| - group.stop(message:) - end - end - - # Reset offsets for all streams tracked by a consumer group. - # If the consumer group is active, this will make it re-process all messages - # - # @param group_id [String] - # @return [Boolean] - def reset_consumer_group(group_id) - group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) - db.transaction do - row = db[consumer_groups_table].where(group_id:).select(:id).first - return unless row - - id = row[:id] - db[offsets_table].where(group_id: id).delete - end - - true - end - - def ack_on(group_id, message_id, &) - db.transaction do - row = db.fetch(sql_for_ack_on, group_id, message_id).first - raise Sourced::ConcurrentAckError, "Stream for message #{message_id} is being concurrently processed by #{group_id}" unless row - - yield if block_given? - - ack_message(group_id, row[:stream_id_fk], row[:global_seq]) - end - end - - private def ack_message(group_id, stream_id, global_seq) - db.transaction do - # We could use an upsert here, but - # we create consumer groups on registration, or - # after the first update. - # So by this point, the consumer groups table - # will always have a record for the group_id. - # So we assume it's there and update it directly. - # If the group_id doesn't exist, it will be created below, - # but this should only happen once per group. - update_result = db[consumer_groups_table] - .where(group_id:) - .returning(:id) - .update( - updated_at: Sequel.function(:now), - highest_global_seq: Sequel.function( - :greatest, - :highest_global_seq, - global_seq - ) - ) - - # Here we do issue a separate INSERT, but this should only happen - # if the group record doesn't exist yet, for some reason. - if update_result.empty? # No rows were updated (record doesn't exist) - # Only then INSERT - update_result = db[consumer_groups_table] - .returning(:id) - .insert(group_id:, highest_global_seq: global_seq) - end - - group_id = update_result[0][:id] - - # Upsert offset - # NOTE: when using #reserve_next_for_reactor, - # an offset will always exist for an message - # but the synchronous ack_on method can be used with an message - # for whose group and stream no offset exists yet. - db[offsets_table] - .insert_conflict( - target: [:group_id, :stream_id], - update: { claimed: false, claimed_at: nil, claimed_by: nil, global_seq: } - ) - .insert( - group_id:, - stream_id:, - global_seq: - ) - end - end - - private def base_messages_query - db[messages_table] - .select( - Sequel[messages_table][:id], - Sequel[streams_table][:stream_id], - Sequel[messages_table][:seq], - Sequel[messages_table][:global_seq], - Sequel[messages_table][:type], - Sequel[messages_table][:created_at], - Sequel[messages_table][:causation_id], - Sequel[messages_table][:correlation_id], - Sequel[messages_table][:metadata], - Sequel[messages_table][:payload], - ) - .join(streams_table, id: :stream_id) - end - - def read_correlation_batch(message_id) - correlation_subquery = db[messages_table] - .select(:correlation_id) - .where(id: message_id) - - query = base_messages_query - .where(Sequel[messages_table][:correlation_id] => correlation_subquery) - .order(Sequel[messages_table][:global_seq]) - - query.map do |row| - deserialize_message(row) - end - end - - def read_stream(stream_id, after: nil, upto: nil) - _messages_table = messages_table # need local variable for Sequel block - - query = base_messages_query.where(Sequel[streams_table][:stream_id] => stream_id) - - query = query.where { Sequel[_messages_table][:seq] > after } if after - query = query.where { Sequel[_messages_table][:seq] <= upto } if upto - query.order(Sequel[_messages_table][:global_seq]).map do |row| - deserialize_message(row) - end - end - - # For tests only - def clear! - raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test' - # Truncate and restart global_seq increment first - db[messages_table].truncate(cascade: true, only: true, restart: true) - db[messages_table].delete - db[consumer_groups_table].delete - db[offsets_table].delete - db[streams_table].delete - db[scheduled_messages_table].delete - db[workers_table].delete - end - - # Called after Sourced.configure - def setup!(config) - return if @setup - if config.executor.is_a?(Sourced::AsyncExecutor) - Sequel.extension :fiber_concurrency - end - @setup = true - end - - # Check if all required database tables exist. - # This verifies that the backend has been properly installed with all necessary schema. - # - # @return [Boolean] true if all required tables exist, false otherwise - # @see #install - def installed? - installer.installed? - end - - def uninstall - installer.uninstall - self - end - - def install - installer.install - self - end - - def copy_migration_to(dir = nil, &block) - installer.copy_migration_to(dir, &block) - end - - private - - # Override in subclasses to configure the notifier. - # Default: PGNotifier for PG, InlineNotifier otherwise. - def setup_adapter - @notifier = @db.adapter_scheme == :postgres ? PGNotifier.new(db: @db) : InlineNotifier.new - end - - # Assign table name ivars and their literal SQL representations. - def setup_tables - @workers_table = table_name(:workers) - @scheduled_messages_table = table_name(:scheduled_messages) - @streams_table = table_name(:streams) - @offsets_table = table_name(:offsets) - @consumer_groups_table = table_name(:consumer_groups) - @messages_table = table_name(:messages) - # Literal SQL strings for use in raw SQL interpolation. - # Sequel qualified identifiers don't produce valid SQL via #to_s, - # so we pre-compute literal versions for the raw SQL methods. - @messages_table_literal = @db.literal(@messages_table) - @streams_table_literal = @db.literal(@streams_table) - @offsets_table_literal = @db.literal(@offsets_table) - @consumer_groups_table_literal = @db.literal(@consumer_groups_table) - end - - attr_reader :db, :logger, :prefix, :schema, :messages_table, :streams_table, :offsets_table, :consumer_groups_table, :scheduled_messages_table, :workers_table - - # Override in subclasses to use a different migration template. - def migration_template_name - '001_create_sourced_tables.rb.erb' - end - - def installer - @installer ||= Installer.new( - @db, - logger:, - schema:, - prefix:, - migration_template: migration_template_name - ) - end - - # CTE-based SQL that atomically claims an offset and fetches batch messages in one statement. - # 1. first_match: finds the first pending message to identify the stream/offset (FOR UPDATE SKIP LOCKED) - # 2. claim: claims the offset in the same statement (UPDATE ... RETURNING) - # 3. outer SELECT: fetches up to batch_size messages from the claimed stream - # - # When with_history: true, adds a `batch` CTE to identify batch rows, and the outer SELECT - # fetches ALL messages from the stream (for history), marking batch rows with `in_batch`. - # This saves a separate read_stream query for reactors that need history. - def sql_for_claim_and_fetch_batch(handled_messages, group_id:, now:, worker_id:, batch_size:, start_from: nil, with_history: false) - message_types = handled_messages.map { |e| "'#{e}'" } - now_literal = db.literal(now) - group_id_literal = db.literal(group_id) - worker_id_literal = db.literal(worker_id) - batch_size_literal = db.literal(batch_size) - message_types_sql = message_types.any? ? " AND e.type IN(#{message_types.join(',')})" : '' - time_window_sql = start_from ? " AND e.created_at > #{db.literal(start_from)}" : '' - - # Common CTEs: find first matching message, claim its offset - first_match_and_claim = <<~SQL - WITH first_match AS ( - SELECT - so.id as offset_id, ss.stream_id, e.stream_id as stream_id_fk, - cg.id as group_id_fk, - cg.highest_global_seq, - so.global_seq as offset_seq - FROM #{@messages_table_literal} e - JOIN #{@streams_table_literal} ss ON e.stream_id = ss.id - JOIN #{@consumer_groups_table_literal} cg ON cg.group_id = #{group_id_literal} - JOIN #{@offsets_table_literal} so ON cg.id = so.group_id AND ss.id = so.stream_id - WHERE e.global_seq > so.global_seq - AND so.claimed = FALSE - AND cg.status = 'active' - AND (cg.retry_at IS NULL OR cg.retry_at <= #{now_literal}) - #{message_types_sql}#{time_window_sql} - ORDER BY e.global_seq ASC - FOR UPDATE OF so SKIP LOCKED - LIMIT 1 - ), - claim AS ( - UPDATE #{@offsets_table_literal} - SET claimed = TRUE, claimed_at = #{now_literal}, claimed_by = #{worker_id_literal} - FROM first_match - WHERE #{@offsets_table_literal}.id = first_match.offset_id - RETURNING first_match.offset_id, first_match.stream_id, first_match.stream_id_fk, - first_match.group_id_fk, first_match.highest_global_seq, first_match.offset_seq - ) - SQL - - if with_history - # Fetch ALL stream messages + mark which are in the batch via LEFT JOIN. - # The batch CTE identifies the batch rows (after offset, matching types, limited). - # The outer SELECT returns every message in the stream for history. - first_match_and_claim + <<~SQL - , - batch AS ( - SELECT e.global_seq - FROM claim - JOIN #{@messages_table_literal} e ON e.stream_id = claim.stream_id_fk - WHERE e.global_seq > claim.offset_seq - #{message_types_sql}#{time_window_sql} - ORDER BY e.global_seq ASC - LIMIT #{batch_size_literal} - ) - SELECT e.global_seq, e.id, claim.stream_id, e.stream_id as stream_id_fk, - e.seq, e.type, e.created_at, e.causation_id, e.correlation_id, - e.metadata, e.payload, claim.offset_id, claim.group_id_fk, - (e.global_seq <= claim.highest_global_seq) AS replaying, - claim.highest_global_seq, - (batch.global_seq IS NOT NULL) AS in_batch - FROM claim - JOIN #{@messages_table_literal} e ON e.stream_id = claim.stream_id_fk - LEFT JOIN batch ON batch.global_seq = e.global_seq - ORDER BY e.global_seq ASC; - SQL - else - first_match_and_claim + <<~SQL - SELECT e.global_seq, e.id, claim.stream_id, e.stream_id as stream_id_fk, - e.seq, e.type, e.created_at, e.causation_id, e.correlation_id, - e.metadata, e.payload, claim.offset_id, claim.group_id_fk, - (e.global_seq <= claim.highest_global_seq) AS replaying, - claim.highest_global_seq - FROM claim - JOIN #{@messages_table_literal} e ON e.stream_id = claim.stream_id_fk - WHERE e.global_seq > claim.offset_seq - #{message_types_sql}#{time_window_sql} - ORDER BY e.global_seq ASC - LIMIT #{batch_size_literal}; - SQL - end - end - - def sql_for_ack_on - @sql_for_ack_on ||= <<~SQL - WITH candidate_rows AS ( - SELECT - e.global_seq, - e.stream_id AS stream_id_fk, - pg_try_advisory_xact_lock(hashtext(?::text), hashtext(s.id::text)) as lock_obtained - FROM #{@messages_table_literal} e - JOIN #{@streams_table_literal} s ON e.stream_id = s.id - WHERE e.id = ? - ) - SELECT * - FROM candidate_rows - WHERE lock_obtained = true - LIMIT 1; - SQL - end - - def sql_for_consumer_stats - @sql_for_consumer_stats ||= <<~SQL - SELECT - r.group_id, - r.status, - r.retry_at, - COALESCE(MIN(o.global_seq), 0) AS oldest_processed, - COALESCE(MAX(o.global_seq), 0) AS newest_processed, - COUNT(o.id) AS stream_count - FROM #{@consumer_groups_table_literal} r - LEFT JOIN #{@offsets_table_literal} o ON o.group_id = r.id AND o.global_seq > 0 - GROUP BY r.id, r.group_id, r.status, r.retry_at - ORDER BY r.group_id; - SQL - end - - def table_name(name) - t = [prefix, name].join('_').to_sym - schema ? Sequel[schema.to_sym][t] : t - end - - def parse_json(json) - return json unless json.is_a?(String) - - JSON.parse(json, symbolize_names: true) - end - - def upsert_consumer_group(group_id, status: ACTIVE) - db[consumer_groups_table] - .insert_conflict(target: :group_id, update: { status:, updated_at: Time.now }) - .insert(group_id:, status:) - end - - def serialize_message(message, stream_id) - row = message.to_h - row[:stream_id] = stream_id - row[:metadata] = JSON.dump(row[:metadata]) if row[:metadata] - row[:payload] = JSON.dump(row[:payload]) if row[:payload] - row - end - - def deserialize_message(row) - row[:payload] = parse_json(row[:payload]) if row[:payload] - row[:metadata] = parse_json(row[:metadata]) if row[:metadata] - Message.from(row) - end - - def connect(db) - case db - when Sequel::Database - db - when String, Hash - Sequel.connect(db) - else - raise ArgumentError, "Invalid database connection: #{db.inspect}" - end - end - end - end -end - -require 'sourced/backends/sequel_backend/installer' -require 'sourced/backends/sequel_backend/group_updater' -require 'sourced/backends/sequel_backend/pg_notifier' diff --git a/lib/sourced/backends/sequel_backend/group_updater.rb b/lib/sourced/backends/sequel_backend/group_updater.rb deleted file mode 100644 index 96d993bd..00000000 --- a/lib/sourced/backends/sequel_backend/group_updater.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module Backends - class SequelBackend - class GroupUpdater - attr_reader :group_id, :updates, :error_context - - def initialize(group_id, row, logger) - @group_id = group_id - @row = row - @logger = logger - @error_context = row[:error_context] - @updates = { error_context: @error_context.dup } - end - - def stop(message: nil) - @logger.error "stopping consumer group #{group_id}" - @updates[:status] = STOPPED - @updates[:retry_at] = nil - @updates[:updated_at] = Time.now - @updates[:error_context][:message] = message if message - end - - def fail(exception: nil) - @logger.error "failing consumer group #{group_id}" - @updates[:status] = FAILED - @updates[:retry_at] = nil - @updates[:updated_at] = Time.now - if exception - @updates[:error_context][:exception_class] = exception.class.to_s - @updates[:error_context][:exception_message] = exception.message - end - end - - def retry(time, ctx = {}) - @logger.warn "retrying consumer group #{group_id} at #{time}" - @updates[:updated_at] = Time.now - @updates[:retry_at] = time - @updates[:error_context].merge!(ctx) - end - end - end - end -end diff --git a/lib/sourced/backends/sequel_backend/installer.rb b/lib/sourced/backends/sequel_backend/installer.rb deleted file mode 100644 index bedac1d4..00000000 --- a/lib/sourced/backends/sequel_backend/installer.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -require 'sequel' -require 'sequel/extensions/migration' -require 'erb' - -module Sourced - module Backends - class SequelBackend - class Installer - def initialize(db, logger:, schema: nil, prefix: 'sourced', migration_template: '001_create_sourced_tables.rb.erb') - raise ArgumentError, "invalid prefix: #{prefix}" unless prefix.match?(/\A[a-zA-Z_]\w*\z/) - raise ArgumentError, "invalid schema: #{schema}" if schema && !schema.match?(/\A[a-zA-Z_]\w*\z/) - - @db = db - @logger = logger - @schema = schema - @prefix = prefix - @migration_template = migration_template - end - - # Eval the rendered migration and apply :up directly. - # No Migrator, no schema_info tracking table. - def install - migration.apply(db, :up) - logger.info("Sourced tables installed (prefix: #{prefix}, schema: #{schema || 'default'})") - end - - # Check that all expected tables exist. - def installed? - all_table_names.all? { |t| db.table_exists?(t) } - end - - # Apply :down on the migration to drop tables. - def uninstall - raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test' - - migration.apply(db, :down) - if schema - db.run("DROP SCHEMA IF EXISTS #{db.literal(schema.to_sym)}") - end - end - - # Render the migration to a file for use with the host app's Sequel::Migrator. - # - # installer.copy_migration_to("db/migrations") - # installer.copy_migration_to { "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_sourced_tables.rb" } - # - def copy_migration_to(dir = nil, &block) - path = block ? block.call : File.join(dir, '001_create_sourced_tables.rb') - File.write(path, rendered_migration) - logger.info("Copied Sourced migration to #{path}") - path - end - - private - - attr_reader :db, :logger, :schema, :prefix - - def migration - @migration ||= eval(rendered_migration) # rubocop:disable Security/Eval - end - - def rendered_migration - @rendered_migration ||= begin - template_path = File.join(__dir__, 'migrations', @migration_template) - ERB.new(File.read(template_path)).result(binding) - end - end - - # Used in ERB template to produce table name expressions. - # Returns Ruby source code strings, e.g. `:sourced_streams` or `Sequel[:my_schema][:sourced_streams]` - def table_name(name) - t = :"#{prefix}_#{name}" - schema ? "Sequel[:#{schema}][:#{t}]" : ":#{t}" - end - - # Returns actual Sequel identifiers for use in installed? checks. - def all_table_names - %i[streams consumer_groups offsets messages scheduled_messages workers].map do |name| - t = :"#{prefix}_#{name}" - schema ? Sequel[schema.to_sym][t] : t - end - end - end - end - end -end diff --git a/lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables.rb.erb b/lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables.rb.erb deleted file mode 100644 index ff0e3e92..00000000 --- a/lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables.rb.erb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -Sequel.migration do - change do -<% if schema %> - run 'CREATE SCHEMA IF NOT EXISTS "<%= schema %>"' -<% end %> - - create_table(<%= table_name(:streams) %>) do - primary_key :id - String :stream_id, null: false, unique: true - Time :updated_at, null: false, default: Sequel.function(:now) - Bignum :seq, null: false - - index :updated_at, name: 'idx_<%= prefix %>_streams_updated_at' - end - - create_table(<%= table_name(:consumer_groups) %>) do - primary_key :id - String :group_id, null: false, unique: true - Bignum :highest_global_seq, null: false, default: 0 - String :status, null: false, default: 'active', index: true - column :error_context, :jsonb - Time :retry_at, null: true, index: true - Time :created_at, null: false, default: Sequel.function(:now) - Time :updated_at, null: false, default: Sequel.function(:now) - - index :group_id, unique: true - end - - create_table(<%= table_name(:offsets) %>) do - primary_key :id - foreign_key :group_id, <%= table_name(:consumer_groups) %>, on_delete: :cascade - foreign_key :stream_id, <%= table_name(:streams) %>, on_delete: :cascade - Bignum :global_seq, null: false - Time :created_at, null: false, default: Sequel.function(:now) - TrueClass :claimed, null: false, default: false - Time :claimed_at, null: true - String :claimed_by, null: true - - index %i[group_id stream_id], unique: true - index :claimed, where: { claimed: false }, name: 'idx_<%= prefix %>_offsets_unclaimed' - index [:claimed, :claimed_by, :claimed_at], where: { claimed: true }, name: 'idx_<%= prefix %>_offsets_claimed_claimer' - index %i[group_id global_seq], name: 'idx_<%= prefix %>_offsets_group_seq_covering' - end - - create_table(<%= table_name(:messages) %>) do - primary_key :global_seq, type: :Bignum - column :id, :uuid, unique: true - foreign_key :stream_id, <%= table_name(:streams) %> - Bignum :seq, null: false - String :type, null: false - Time :created_at, null: false - column :causation_id, :uuid, index: true - column :correlation_id, :uuid, index: true - column :metadata, :jsonb - column :payload, :jsonb - - index %i[stream_id seq], unique: true - index :type - index :created_at - index %i[type global_seq] - index %i[stream_id global_seq] - end - - create_table(<%= table_name(:scheduled_messages) %>) do - primary_key :id - Time :created_at, null: false - Time :available_at, null: false - column :message, :jsonb - - index :available_at - end - - create_table(<%= table_name(:workers) %>) do - String :id, primary_key: true, null: false - Time :last_seen, null: false, index: true - String :pid, null: true - String :host, null: true - column :info, :jsonb - end - end -end diff --git a/lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables_sqlite.rb.erb b/lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables_sqlite.rb.erb deleted file mode 100644 index 8398c6b2..00000000 --- a/lib/sourced/backends/sequel_backend/migrations/001_create_sourced_tables_sqlite.rb.erb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -Sequel.migration do - change do - create_table(<%= table_name(:streams) %>) do - primary_key :id - String :stream_id, null: false, unique: true - Time :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP - Bignum :seq, null: false - - index :updated_at, name: 'idx_<%= prefix %>_streams_updated_at' - end - - create_table(<%= table_name(:consumer_groups) %>) do - primary_key :id - String :group_id, null: false, unique: true - Bignum :highest_global_seq, null: false, default: 0 - String :status, null: false, default: 'active', index: true - String :error_context - Time :retry_at, null: true, index: true - Time :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP - Time :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP - - index :group_id, unique: true - end - - create_table(<%= table_name(:offsets) %>) do - primary_key :id - foreign_key :group_id, <%= table_name(:consumer_groups) %>, on_delete: :cascade - foreign_key :stream_id, <%= table_name(:streams) %>, on_delete: :cascade - Bignum :global_seq, null: false - Time :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP - TrueClass :claimed, null: false, default: false - Time :claimed_at, null: true - String :claimed_by, null: true - - index %i[group_id stream_id], unique: true - index :claimed, name: 'idx_<%= prefix %>_offsets_unclaimed' - index [:claimed, :claimed_by, :claimed_at], name: 'idx_<%= prefix %>_offsets_claimed_claimer' - index %i[group_id global_seq], name: 'idx_<%= prefix %>_offsets_group_seq_covering' - end - - create_table(<%= table_name(:messages) %>) do - primary_key :global_seq, type: :Bignum - String :id, unique: true - foreign_key :stream_id, <%= table_name(:streams) %> - Bignum :seq, null: false - String :type, null: false - Time :created_at, null: false - String :causation_id, index: true - String :correlation_id, index: true - String :metadata - String :payload - - index %i[stream_id seq], unique: true - index :type - index :created_at - index %i[type global_seq] - index %i[stream_id global_seq] - end - - create_table(<%= table_name(:scheduled_messages) %>) do - primary_key :id - Time :created_at, null: false - Time :available_at, null: false - String :message - - index :available_at - end - - create_table(<%= table_name(:workers) %>) do - String :id, primary_key: true, null: false - Time :last_seen, null: false, index: true - String :pid, null: true - String :host, null: true - String :info - end - end -end diff --git a/lib/sourced/backends/sequel_backend/pg_notifier.rb b/lib/sourced/backends/sequel_backend/pg_notifier.rb deleted file mode 100644 index a5b97c88..00000000 --- a/lib/sourced/backends/sequel_backend/pg_notifier.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module Backends - class SequelBackend - # PostgreSQL LISTEN/NOTIFY pub/sub transport for real-time message dispatch. - # - # Events are multiplexed over a single PG channel. The wire format is - # +event_name:value+ (split on the first colon). - # - # Implements the same interface as {Sourced::InlineNotifier}. - # - # @example Wiring in a Dispatcher - # pg_notifier = PGNotifier.new(db: Sequel.postgres('mydb')) - # pg_notifier.subscribe(queuer) - # - # # In a dedicated fiber: - # pg_notifier.start # blocks, listening for notifications - # - # # From the append path (inside a transaction): - # pg_notifier.notify_new_messages(['orders.created', 'orders.shipped']) - # - # # From start_consumer_group: - # pg_notifier.notify_reactor_resumed('OrderReactor') - class PGNotifier - # @return [String] PG NOTIFY channel name - CHANNEL = 'sourced_new_messages' - - # @param db [Sequel::Database] a PostgreSQL Sequel database connection. - # The LISTEN connection should be separate from the one used for writes - # to avoid blocking. - def initialize(db:) - @db = db - @subscribers = [] - @listening = false - end - - # Register a subscriber to receive all published events. - # - # @param callable [#call] receives +(event_name, value)+ where both are Strings - # @return [void] - def subscribe(callable) - @subscribers << callable - end - - # Publish an event via PG NOTIFY. Should be called inside a transaction - # so the notification is delivered atomically on commit. - # Wire format: +event_name:value+. - # - # @param event_name [String] event name - # @param value [String] event payload - # @return [void] - def publish(event_name, value) - @db.run(Sequel.lit("SELECT pg_notify('#{CHANNEL}', ?)", "#{event_name}:#{value}")) - end - - # Notify that new messages were appended. - # - # @param types [Array] message type strings - # @return [void] - def notify_new_messages(types) - publish('messages_appended', types.uniq.join(',')) - end - - # Notify that a stopped reactor has been resumed. - # - # @param group_id [String] consumer group ID of the resumed reactor - # @return [void] - def notify_reactor_resumed(group_id) - publish('reactor_resumed', group_id) - end - - # Block on PG LISTEN, dispatching events to all subscribers. - # Loops with a 2-second timeout so {#stop} is checked periodically. - # - # @return [void] - def start - @listening = true - while @listening - @db.listen(CHANNEL, timeout: 2) do |_ch, _pid, payload| - event_name, value = payload.split(':', 2) - @subscribers.each { |s| s.call(event_name, value) } - end - end - end - - # Signal the LISTEN loop to exit after the current timeout cycle. - # - # @return [void] - def stop - @listening = false - end - end - end - end -end diff --git a/lib/sourced/backends/sqlite_backend.rb b/lib/sourced/backends/sqlite_backend.rb deleted file mode 100644 index 82e7602f..00000000 --- a/lib/sourced/backends/sqlite_backend.rb +++ /dev/null @@ -1,454 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/backends/sequel_backend' - -module Sourced - module Backends - # SQLite-specific backend. Inherits shared behaviour from {SequelBackend} - # and overrides methods that rely on PG-specific features: - # - # - No CTEs with +UPDATE ... RETURNING+ — uses separate SELECT/UPDATE steps - # - No +FOR UPDATE SKIP LOCKED+ — relies on SQLite's transaction-level write lock - # - No advisory locks — +ack_on+ runs inside a plain transaction - # - No +TRUNCATE+ — +clear!+ uses +DELETE+ - # - No +greatest()+ — uses +max()+ instead - # - # Uses {InlineNotifier} for synchronous worker dispatch (no PG +LISTEN/NOTIFY+). - # The {CatchUpPoller} provides a safety-net poll for startup catch-up and missed signals. - # - # Best suited for single-process deployments, development, scripts, and tests. - # - # @example File-based database - # db = Sequel.sqlite('myapp.db') - # backend = Sourced::Backends::SQLiteBackend.new(db) - # backend.install unless backend.installed? - # - # @example In-memory database (useful for scripts and tests) - # db = Sequel.sqlite - # backend = Sourced::Backends::SQLiteBackend.new(db) - # - # @see SequelBackend - class SQLiteBackend < SequelBackend - # Move scheduled messages whose +available_at+ has passed into the main message log. - # - # Unlike the PG version, this does not use +FOR UPDATE SKIP LOCKED+ to prevent - # concurrent workers from selecting the same rows. Instead, SQLite's database-level - # write lock serializes concurrent callers. - # - # @return [Integer] number of scheduled messages promoted - def update_schedule! - now = Time.now - - transaction do - rows = db[scheduled_messages_table] - .where { available_at <= now } - .order(:id) - .limit(100) - .all - - return 0 if rows.empty? - - messages = rows.map do |r| - data = JSON.parse(r[:message], symbolize_names: true) - data[:created_at] = now - Message.from(data) - end - - messages.group_by(&:stream_id).each do |stream_id, stream_messages| - append_next_to_stream(stream_id, stream_messages) - end - - row_ids = rows.map { |m| m[:id] } - db[scheduled_messages_table].where(id: row_ids).delete - - rows.size - end - end - - # Append messages to a stream, auto-incrementing sequence numbers. - # - # SQLite doesn't support +RETURNING+ on +INSERT ... ON CONFLICT+, so the stream - # record is fetched with a separate +SELECT+ after the upsert. Retries on - # +UniqueConstraintViolation+ with linear backoff up to +max_retries+. - # - # @param stream_id [String] the stream to append to - # @param messages [Message, Array] messages to append - # @param max_retries [Integer] retry limit for constraint violations (default: 3) - # @return [true] - # @raise [Sourced::ConcurrentAppendError] after exhausting retries - def append_next_to_stream(stream_id, messages, max_retries: 3) - messages_array = Array(messages) - return true if messages_array.empty? - - retries = 0 - begin - db.transaction do - messages_count = messages_array.size - - # Upsert stream — SQLite doesn't support RETURNING on INSERT - db[streams_table] - .insert_conflict( - target: :stream_id, - update: { - seq: Sequel.qualify(streams_table, :seq) + messages_count, - updated_at: Time.now - } - ) - .insert(stream_id:, seq: messages_count) - - stream_record = db[streams_table].where(stream_id:).select(:id, :seq).first - - starting_seq = stream_record[:seq] - messages_count + 1 - - rows = messages_array.map.with_index do |msg, index| - row = serialize_message(msg, stream_record[:id]) - row[:seq] = starting_seq + index - row - end - - db[messages_table].multi_insert(rows) - - notifier.notify_new_messages(messages_array.map(&:type)) - end - - true - rescue Sequel::UniqueConstraintViolation => e - retries += 1 - if retries <= max_retries - sleep(0.001 * retries) - retry - else - raise Sourced::ConcurrentAppendError, e.message - end - end - end - - # Append messages to a stream with optimistic locking (sequence check). - # - # Messages must carry pre-assigned +seq+ values. A +UniqueConstraintViolation+ - # on the +[stream_id, seq]+ index signals a concurrent append. - # - # @param stream_id [String] the stream to append to - # @param messages [Message, Array] messages with pre-assigned sequences - # @return [true] - # @raise [Sourced::ConcurrentAppendError] on sequence conflict - def append_to_stream(stream_id, messages) - messages_array = Array(messages) - return false if messages_array.empty? - - if messages_array.map(&:stream_id).uniq.size > 1 - raise ArgumentError, 'Messages must all belong to the same stream' - end - - db.transaction do - seq = messages_array.last.seq - db[streams_table] - .insert_conflict(target: :stream_id, update: { seq:, updated_at: Time.now }) - .insert(stream_id:, seq:) - - # Separate SELECT — SQLite insert_conflict doesn't reliably return the id - stream_row = db[streams_table].where(stream_id:).select(:id).first - id = stream_row[:id] - - rows = messages_array.map { |e| serialize_message(e, id) } - db[messages_table].multi_insert(rows) - - notifier.notify_new_messages(messages_array.map(&:type)) - end - - true - rescue Sequel::UniqueConstraintViolation => e - raise Sourced::ConcurrentAppendError, e.message - end - - # Yield a {GroupUpdater} for the given consumer group inside a transaction. - # - # The PG version uses +FOR UPDATE+ to row-lock the group. SQLite relies on its - # database-level transaction lock instead. - # - # @param group_id [String] consumer group identifier - # @yield [GroupUpdater] group updater for setting retry_at, stop, etc. - # @raise [ArgumentError] if the consumer group is not found - def updating_consumer_group(group_id, &) - db.transaction do - dataset = db[consumer_groups_table].where(group_id:) - group_row = dataset.first - raise ArgumentError, "Consumer group #{group_id} not found" unless group_row - - ctx = group_row[:error_context] ? parse_json(group_row[:error_context]) : {} - group_row[:error_context] = ctx - group = GroupUpdater.new(group_id, group_row, logger) - yield group - updates = group.updates - updates[:error_context] = JSON.dump(updates[:error_context]) - dataset.update(updates) - end - end - - # Acknowledge a message for a consumer group, running an optional block - # inside the same transaction. - # - # The PG version uses +pg_try_advisory_xact_lock+ to prevent concurrent - # processing of the same stream by the same group. SQLite relies on the - # database-level write lock instead, so a second concurrent caller will - # block until the first commits. - # - # @param group_id [String] consumer group identifier - # @param message_id [String] UUID of the message to acknowledge - # @yield optional block to run inside the transaction before ACKing - # @raise [Sourced::ConcurrentAckError] if the message is not found - def ack_on(group_id, message_id, &) - db.transaction do - row = db[messages_table] - .join(streams_table, id: :stream_id) - .where(Sequel[messages_table][:id] => message_id) - .select( - Sequel[messages_table][:global_seq], - Sequel[messages_table][:stream_id].as(:stream_id_fk) - ) - .first - - raise Sourced::ConcurrentAckError, "Message #{message_id} not found for group #{group_id}" unless row - - yield if block_given? - - ack_message(group_id, row[:stream_id_fk], row[:global_seq]) - end - end - - # Clear all data. For tests only. - # - # Uses +DELETE+ instead of +TRUNCATE+ (not supported by SQLite). - # Also resets the +sqlite_sequence+ table so +global_seq+ auto-increment - # restarts from 1. - # - # @raise [RuntimeError] if +ENV['ENVIRONMENT']+ is not +'test'+ - # @return [void] - def clear! - raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test' - db[offsets_table].delete - db[messages_table].delete - db[consumer_groups_table].delete - db[streams_table].delete - db[scheduled_messages_table].delete - db[workers_table].delete - # Reset auto-increment counters so global_seq starts from 1 - db.run("DELETE FROM sqlite_sequence") if db.table_exists?(:sqlite_sequence) - end - - private - - # Configure SQLite-specific PRAGMAs and set up the {InlineNotifier}. - # - # - +foreign_keys = ON+ — enforce FK constraints (off by default in SQLite) - # - +journal_mode = WAL+ — allow concurrent reads during writes - # - +busy_timeout = 5000+ — wait up to 5 s for a write lock before raising - # - # @return [void] - def setup_adapter - @notifier = InlineNotifier.new - @db.run('PRAGMA foreign_keys = ON') - @db.run('PRAGMA journal_mode = WAL') - @db.run('PRAGMA busy_timeout = 5000') - end - - private - - # @return [String] filename of the SQLite-specific migration ERB template - def migration_template_name - '001_create_sourced_tables_sqlite.rb.erb' - end - - # Acknowledge a message by advancing the consumer group's offset. - # - # Uses +max()+ instead of PG's +greatest()+ to update +highest_global_seq+. - # Falls back to +INSERT+ if the consumer group row doesn't exist yet. - # - # @param group_id [String] consumer group identifier - # @param stream_id [Integer] stream FK (internal integer ID, not the string stream_id) - # @param global_seq [Integer] global sequence number of the acknowledged message - # @return [void] - def ack_message(group_id, stream_id, global_seq) - db.transaction do - # Update consumer group — use max() instead of greatest() - updated = db[consumer_groups_table] - .where(group_id:) - .update( - updated_at: Time.now, - highest_global_seq: Sequel.function(:max, :highest_global_seq, global_seq) - ) - - if updated == 0 - db[consumer_groups_table].insert(group_id:, highest_global_seq: global_seq) - end - - group_id_fk = db[consumer_groups_table].where(group_id:).select(:id).first[:id] - - # Upsert offset - db[offsets_table] - .insert_conflict( - target: [:group_id, :stream_id], - update: { claimed: false, claimed_at: nil, claimed_by: nil, global_seq: } - ) - .insert( - group_id: group_id_fk, - stream_id:, - global_seq: - ) - end - end - - # Decompose the PG CTE (claim + fetch in one statement) into - # separate SELECT, UPDATE, SELECT steps within a transaction. - # - # SQLite has no +FOR UPDATE SKIP LOCKED+, so two concurrent workers - # contend for the database-level write lock rather than skipping locked rows. - # - # @param group_id [String] consumer group identifier - # @param handled_messages [Array] message type strings to filter - # @param now [Time] current time for claim timestamps and retry_at checks - # @param start_from [Time, nil] optional time window lower bound - # @param worker_id [String] identifier for the claiming worker - # @param batch_size [Integer] maximum messages to fetch - # @param with_history [Boolean] whether to include full stream history - # @return [Array(Array, Array?), NO_BATCH] batch rows and optional history - def claim_and_fetch_batch(group_id, handled_messages, now, start_from, worker_id, batch_size, with_history: false) - message_types_sql = handled_messages.any? ? " AND e.type IN(#{handled_messages.map { |e| db.literal(e) }.join(',')})" : '' - time_window_sql = start_from.is_a?(Time) ? " AND e.created_at > #{db.literal(start_from)}" : '' - - db.transaction do - # Step 1: Find first unclaimed offset with a matching pending message - first_match = db.fetch(<<~SQL).first - SELECT - so.id as offset_id, ss.stream_id, e.stream_id as stream_id_fk, - cg.id as group_id_fk, - cg.highest_global_seq, - so.global_seq as offset_seq - FROM #{@messages_table_literal} e - JOIN #{@streams_table_literal} ss ON e.stream_id = ss.id - JOIN #{@consumer_groups_table_literal} cg ON cg.group_id = #{db.literal(group_id)} - JOIN #{@offsets_table_literal} so ON cg.id = so.group_id AND ss.id = so.stream_id - WHERE e.global_seq > so.global_seq - AND so.claimed = 0 - AND cg.status = 'active' - AND (cg.retry_at IS NULL OR cg.retry_at <= #{db.literal(now)}) - #{message_types_sql}#{time_window_sql} - ORDER BY e.global_seq ASC - LIMIT 1 - SQL - - return NO_BATCH unless first_match - - # Step 2: Claim the offset - claimed = db[offsets_table] - .where(id: first_match[:offset_id], claimed: false) - .update(claimed: true, claimed_at: now, claimed_by: worker_id) - - return NO_BATCH if claimed == 0 - - # Step 3: Fetch messages - if with_history - fetch_batch_with_history(first_match, message_types_sql, time_window_sql, batch_size) - else - fetch_batch(first_match, message_types_sql, time_window_sql, batch_size) - end - end - rescue Sequel::UniqueConstraintViolation - logger.debug "Batch claim for group #{group_id} already exists, skipping" - NO_BATCH - end - - # Fetch a batch of messages for the claimed offset. - # - # @param first_match [Hash] the claimed offset row - # @param message_types_sql [String] SQL fragment filtering message types - # @param time_window_sql [String] SQL fragment filtering by time - # @param batch_size [Integer] maximum messages to fetch - # @return [Array(Array, nil), NO_BATCH] - def fetch_batch(first_match, message_types_sql, time_window_sql, batch_size) - rows = db.fetch(<<~SQL).all - SELECT e.global_seq, e.id, #{db.literal(first_match[:stream_id])} as stream_id, - e.stream_id as stream_id_fk, - e.seq, e.type, e.created_at, e.causation_id, e.correlation_id, - e.metadata, e.payload, #{db.literal(first_match[:offset_id])} as offset_id, - #{db.literal(first_match[:group_id_fk])} as group_id_fk, - (e.global_seq <= #{db.literal(first_match[:highest_global_seq])}) AS replaying, - #{db.literal(first_match[:highest_global_seq])} as highest_global_seq - FROM #{@messages_table_literal} e - WHERE e.stream_id = #{db.literal(first_match[:stream_id_fk])} - AND e.global_seq > #{db.literal(first_match[:offset_seq])} - #{message_types_sql}#{time_window_sql} - ORDER BY e.global_seq ASC - LIMIT #{db.literal(batch_size)} - SQL - - return NO_BATCH if rows.empty? - - coerce_booleans!(rows) - [rows, nil] - end - - # Fetch a batch of messages along with full stream history. - # - # @param first_match [Hash] the claimed offset row - # @param message_types_sql [String] SQL fragment filtering message types - # @param time_window_sql [String] SQL fragment filtering by time - # @param batch_size [Integer] maximum messages to fetch - # @return [Array(Array, Array), NO_BATCH] - def fetch_batch_with_history(first_match, message_types_sql, time_window_sql, batch_size) - # Fetch all stream messages for history - all_rows = db.fetch(<<~SQL).all - SELECT e.global_seq, e.id, #{db.literal(first_match[:stream_id])} as stream_id, - e.stream_id as stream_id_fk, - e.seq, e.type, e.created_at, e.causation_id, e.correlation_id, - e.metadata, e.payload, #{db.literal(first_match[:offset_id])} as offset_id, - #{db.literal(first_match[:group_id_fk])} as group_id_fk, - (e.global_seq <= #{db.literal(first_match[:highest_global_seq])}) AS replaying, - #{db.literal(first_match[:highest_global_seq])} as highest_global_seq - FROM #{@messages_table_literal} e - WHERE e.stream_id = #{db.literal(first_match[:stream_id_fk])} - ORDER BY e.global_seq ASC - SQL - - return NO_BATCH if all_rows.empty? - - # Identify which rows are in the batch - batch_seqs = db.fetch(<<~SQL).map { |r| r[:global_seq] } - SELECT e.global_seq - FROM #{@messages_table_literal} e - WHERE e.stream_id = #{db.literal(first_match[:stream_id_fk])} - AND e.global_seq > #{db.literal(first_match[:offset_seq])} - #{message_types_sql}#{time_window_sql} - ORDER BY e.global_seq ASC - LIMIT #{db.literal(batch_size)} - SQL - - batch_set = batch_seqs.to_set - all_rows.each { |r| r[:in_batch] = batch_set.include?(r[:global_seq]) } - - coerce_booleans!(all_rows) - - batch_rows = all_rows.select { |r| r[:in_batch] } - if batch_rows.empty? - release_offset(first_match[:offset_id]) - return NO_BATCH - end - - history = all_rows.map { |row| deserialize_message(row) } - [batch_rows, history] - end - - # Convert SQLite integer booleans (0/1) to Ruby booleans. - # - # SQLite boolean expressions like +(expr) AS replaying+ return +0+ or +1+ - # rather than +true+/+false+. This normalizes the +:replaying+ field so - # downstream code works with Ruby booleans. - # - # @param rows [Array] rows to mutate in place - # @return [void] - def coerce_booleans!(rows) - rows.each { |r| r[:replaying] = (r[:replaying] == 1 || r[:replaying] == true) } - end - end - end -end diff --git a/lib/sourced/backends/test_backend.rb b/lib/sourced/backends/test_backend.rb deleted file mode 100644 index 331f6f6d..00000000 --- a/lib/sourced/backends/test_backend.rb +++ /dev/null @@ -1,277 +0,0 @@ -# frozen_string_literal: true - -require 'thread' -require 'sourced/inline_notifier' - -module Sourced - module Backends - class TestBackend - ACTIVE = 'active' - STOPPED = 'stopped' - FAILED = 'failed' - - def initialize - clear! - @mutex = Mutex.new - @in_tx = false - @tx_id = nil - end - - def messages = @state.messages - - def inspect - %(<#{self.class} messages:#{messages.size} streams:#{@state.messages_by_stream_id.size}>) - end - - def clear! - @state = State.new - @notifier = InlineNotifier.new - end - - # Returns the backend's notifier for real-time message dispatch. - # Always returns an {InlineNotifier} since TestBackend has no PG connection. - # Eagerly initialized in {#clear!} for thread and CoW safety. - # - # @return [InlineNotifier] - attr_reader :notifier - - def installed? = true - - def handling_reactor_exceptions(_reactor, &) - yield - end - - def reserve_next_for_reactor(reactor, batch_size: 1, with_history: false, worker_id: Process.pid.to_s, &) - group_id = reactor.consumer_info.group_id - start_from = reactor.consumer_info.start_from.call - transaction do - group = @state.groups[group_id] - if group.active? && (group.retry_at.nil? || group.retry_at <= Time.now) - group.reserve_next(reactor.handled_messages, start_from, method(:process_actions), batch_size:, with_history:, &) - end - end - end - - private def process_actions(group_id, actions, ack, event, offset) - should_ack = false - actions = [actions] unless actions.is_a?(Array) - # Empty actions is assumed to be an ACK - return ack.() if actions.empty? - - actions.each do |action| - case action - when nil, Actions::OK - should_ack = true - - when Actions::Ack - offset.locked = false - ack_on(group_id, action.message_id) - - when Actions::RETRY - # Don't ack - - else - action.execute(self, event) - should_ack = true - end - end - - ack.() if should_ack - end - - def ack_on(group_id, event_id, &) - transaction do - group = @state.groups[group_id] - group.ack_on(event_id, &) - end - end - - def register_consumer_group(group_id) - transaction do - @state.groups[group_id] - end - end - - def updating_consumer_group(group_id, &) - transaction do - group = @state.groups[group_id] - yield group - end - end - - # Start a consumer group that has been stopped or failed. - # Signals the notifier so workers pick up the reactor immediately. - # - # @param group_id [String] - def start_consumer_group(group_id) - group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) - transaction do - group = @state.groups[group_id] - group.error_context = {} - group.status = ACTIVE - group.retry_at = nil - end - notifier.notify_reactor_resumed(group_id) - end - - def stop_consumer_group(group_id, message = nil) - group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) - transaction do - group = @state.groups[group_id] - group.stop(message:) - end - end - - def reset_consumer_group(group_id) - group_id = group_id.consumer_info.group_id if group_id.respond_to?(:consumer_info) - transaction do - group = @state.groups[group_id] - group.reset! - end - true - end - - def schedule_messages(messages, at: Time.now) - @state.schedule_messages(messages, at:) - true - end - - def update_schedule! - count = 0 - transaction do - @state.next_scheduled_messages do |scheduled_messages| - scheduled_messages.group_by(&:stream_id).each do |stream_id, stream_messages| - append_next_to_stream(stream_id, stream_messages) - end - count = scheduled_messages.size - end - count - end - end - - Stats = Data.define(:stream_count, :max_global_seq, :groups) - - def stats - stream_count = @state.messages_by_stream_id.size - max_global_seq = messages.size - groups = @state.groups.values.map(&:to_h) - Stats.new(stream_count, max_global_seq, groups) - end - - # Retrieve a list of recently active streams, ordered by most recent activity. - # This is the in-memory implementation that maintains stream metadata during testing. - # - # @param limit [Integer] Maximum number of streams to return (defaults to 10) - # @return [Array] Array of Stream objects ordered by updated_at descending - # @see SequelBackend#recent_streams - def recent_streams(limit: 10) - # Input validation (consistent with SequelBackend) - return [] if limit == 0 - raise ArgumentError, "limit must be a positive integer" if limit < 0 - - @state.streams.values.sort_by(&:updated_at).reverse.take(limit) - end - - def transaction(&) - if @in_tx - yield - else - @mutex.synchronize do - @in_tx = true - @state_snapshot = @state.copy - result = yield - @in_tx = false - @state_snapshot = nil - result - end - end - rescue StandardError => e - @in_tx = false - @state = @state_snapshot if @state_snapshot - raise - end - - # @param stream_id [String] Unique identifier for the event stream - # @param messages [Sourced::Message, Array] Event(s) to append to the stream - # @option max_retries [Integer] Not used in this backend, but kept for interface compatibility - def append_next_to_stream(stream_id, messages, max_retries: 3) - # Handle both single event and array of messages - messages_array = Array(messages) - return true if messages_array.empty? - - transaction do - last_message = @state.messages_by_stream_id[stream_id].last - last_seq = last_message ? last_message.seq : 0 - - messages_with_seq = messages_array.map.with_index do |message, index| - message.with(seq: last_seq + index + 1) - end - - append_to_stream(stream_id, messages_with_seq) - end - end - - def append_to_stream(stream_id, messages) - # Handle both single event and array of events - messages_array = Array(messages) - return false if messages_array.empty? - - transaction do - check_unique_seq!(messages_array) - - messages_array.each do |message| - @state.messages_by_correlation_id[message.correlation_id] << message - @state.messages_by_stream_id[stream_id] << message - @state.messages << message - @state.stream_id_seq_index[seq_key(stream_id, message)] = true - @state.upsert_stream(stream_id, message.seq) - end - end - @state.groups.each_value(&:reindex) - notifier.notify_new_messages(messages_array.map(&:type)) - true - end - - def read_correlation_batch(message_id) - message = @state.messages.find { |e| e.id == message_id } - return [] unless message - @state.messages_by_correlation_id[message.correlation_id] - end - - def read_stream(stream_id, after: nil, upto: nil) - messages = @state.messages_by_stream_id[stream_id] - messages = messages.select { |e| e.seq > after } if after - messages = messages.select { |e| e.seq <= upto } if upto - messages - end - - # No-op heartbeats for test backend - def worker_heartbeat(worker_ids, at: Time.now) - Array(worker_ids).size - end - - # No-op stale claim release for test backend - def release_stale_claims(ttl_seconds: 120) - 0 - end - - private - - def check_unique_seq!(messages) - duplicate = messages.find do |message| - @state.stream_id_seq_index[seq_key(message.stream_id, message)] - end - if duplicate - raise Sourced::ConcurrentAppendError, "Duplicate stream_id/seq: #{duplicate.stream_id}/#{duplicate.seq}" - end - end - - def seq_key(stream_id, message) - [stream_id, message.seq] - end - end - end -end - -require 'sourced/backends/test_backend/group' -require 'sourced/backends/test_backend/state' diff --git a/lib/sourced/backends/test_backend/group.rb b/lib/sourced/backends/test_backend/group.rb deleted file mode 100644 index e31eff74..00000000 --- a/lib/sourced/backends/test_backend/group.rb +++ /dev/null @@ -1,180 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module Backends - class TestBackend - class Group - attr_reader :group_id - attr_accessor :status, :error_context, :retry_at - - Offset = Struct.new(:stream_id, :index, :locked) - - def initialize(group_id, backend) - @group_id = group_id - @backend = backend - @status = :active - @error_context = {} - @retry_at = nil - @highest_index = -1 - reset! - end - - def active? = @status == :active - - def stop(message: nil) - @error_context[:message] = message if message - @status = :stopped - end - - def fail(exception: nil) - if exception - @error_context[:exception_class] = exception.class.to_s - @error_context[:exception_message] = exception.message - end - @status = :failed - end - - def reset! - @offsets = {} - reindex - end - - def retry(time, ctx = {}) - @error_context.merge!(ctx) - @retry_at = time - end - - def to_h - active_offsets = @offsets.values.select { |o| o.index >= 0 } - oldest_processed = (active_offsets.min_by(&:index)&.index || -1) + 1 - newest_processed = (active_offsets.max_by(&:index)&.index || -1) + 1 - stream_count = active_offsets.size - - { - group_id:, - status: @status.to_s, - oldest_processed:, - newest_processed:, - stream_count:, - retry_at: - } - end - - def reindex - backend.messages.each do |e| - @offsets[e.stream_id] ||= Offset.new(e.stream_id, -1, false) - end - end - - def ack_on(message_id, &) - global_seq = backend.messages.find_index { |e| e.id == message_id } - return unless global_seq - - evt = backend.messages[global_seq] - offset = @offsets[evt.stream_id] - if offset.locked - raise Sourced::ConcurrentAckError, "Stream for message #{message_id} is being concurrently processed by #{group_id}" - else - offset.locked = true - yield if block_given? - offset.index = global_seq - @highest_index = global_seq if global_seq > @highest_index - offset.locked = false - end - end - - NOOP_FILTER = ->(_) { true } - - def reserve_next(handled_messages, time_window, process_actions, batch_size: 1, with_history: false, &block) - time_filter = time_window.is_a?(Time) ? ->(e) { e.created_at > time_window } : NOOP_FILTER - evt = nil - offset = nil - index = -1 - - backend.messages.each.with_index do |e, idx| - offset = @offsets[e.stream_id] - if offset.locked # stream locked by another consumer in the group - next - elsif idx > offset.index && handled_messages.include?(e.class) && time_filter.call(e) # new message for the stream - evt = e - offset.locked = true - index = idx - break - else # messages already consumed - end - end - - return unless evt - - reserve_batch(evt, index, offset, handled_messages, time_filter, process_actions, batch_size, with_history:, &block) - end - - private - - def reserve_batch(first_evt, first_index, offset, handled_messages, time_filter, process_actions_callback, batch_size, with_history: false, &block) - stream_id = first_evt.stream_id - raw_batch = [[first_evt, first_index]] - - # Find additional messages from same stream - backend.messages.each.with_index do |e, idx| - break if raw_batch.size >= batch_size - next if idx <= first_index - next unless e.stream_id == stream_id - next unless handled_messages.include?(e.class) - next unless time_filter.call(e) - - raw_batch << [e, idx] - end - - # Build history if requested: all messages from this stream - history = if with_history - backend.messages.select { |e| e.stream_id == stream_id } - end - - # Build batch of [message, replaying] pairs - batch = raw_batch.map { |msg, idx| [msg, @highest_index >= idx] } - - # Yield batch + history once, get back action_pairs or RETRY - action_pairs = yield(batch, history) - - if action_pairs == Actions::RETRY - offset.locked = false - return first_evt - end - - # Execute all action pairs - if action_pairs.empty? - # Nothing to process — release the claim so another worker can retry - offset.locked = false - return first_evt - end - - noop_ack = -> {} - action_pairs.each do |actions, source_message| - process_actions_callback.(group_id, actions, noop_ack, source_message, offset) - end - - # ACK once for the last successfully processed message. - # Find the raw_batch entry matching the last action_pair's source_message, - # so partial batches ACK up to the last successful message (not the last in the batch). - last_source_message = action_pairs.last&.last - last_entry = raw_batch.find { |msg, _idx| msg.id == last_source_message&.id } || raw_batch.last - last_idx = last_entry[1] - ack(offset, last_idx) if last_idx > offset.index - - offset.locked = false - first_evt - end - - def ack(offset, index) - # ACK reactor/message - offset.index = index - @highest_index = index if index > @highest_index - end - - attr_reader :backend - end - - end - end -end diff --git a/lib/sourced/backends/test_backend/state.rb b/lib/sourced/backends/test_backend/state.rb deleted file mode 100644 index c6371ff8..00000000 --- a/lib/sourced/backends/test_backend/state.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module Backends - class TestBackend - class State - attr_reader :messages, :groups, :messages_by_correlation_id, :messages_by_stream_id, :stream_id_seq_index, :streams, :scheduled_messages - - def initialize( - messages: [], - groups: Hash.new { |h, k| h[k] = Group.new(k, self) }, - messages_by_correlation_id: Hash.new { |h, k| h[k] = [] }, - messages_by_stream_id: Hash.new { |h, k| h[k] = [] }, - stream_id_seq_index: {}, - streams: {}, - scheduled_messages: [] - ) - - @messages = messages - @groups = groups - @messages_by_correlation_id = messages_by_correlation_id - @messages_by_stream_id = messages_by_stream_id - @stream_id_seq_index = stream_id_seq_index - @streams = streams - @scheduled_messages = scheduled_messages - end - - ScheduledMessageRecord = Data.define(:message, :at, :position) do - def <=>(other) - self.position <=> other.position - end - end - - Stream = Data.define(:stream_id, :seq, :updated_at) do - def hash = stream_id - def eql?(other) = other.is_a?(Stream) && stream_id == other.stream_id - end - - def upsert_stream(stream_id, seq) - str = Stream.new(stream_id, seq, Time.now) - @streams[stream_id] = str - end - - def schedule_messages(messages, at: Time.now) - counter = @scheduled_messages.size - records = messages.map do |a| - counter += 1 - ScheduledMessageRecord.new(a, at, [at, counter]) - end - @scheduled_messages += records - @scheduled_messages.sort! - end - - def next_scheduled_messages(&) - now = Time.now - next_records, @scheduled_messages = @scheduled_messages.partition do |r| - r.at <= now - end - next_messages = next_records.map(&:message) - yield next_messages if next_messages.any? - next_messages - end - - def copy - self.class.new( - messages: messages.dup, - groups: deep_dup(groups), - messages_by_correlation_id: deep_dup(messages_by_correlation_id), - messages_by_stream_id: deep_dup(messages_by_stream_id), - stream_id_seq_index: deep_dup(stream_id_seq_index), - streams: streams.dup, - scheduled_messages: scheduled_messages.dup - ) - end - - private - - def deep_dup(hash) - hash.each.with_object(hash.dup.clear) do |(k, v), new_hash| - new_hash[k] = v.dup - end - end - end - end - end -end diff --git a/lib/sourced/ccc.rb b/lib/sourced/ccc.rb deleted file mode 100644 index 73d87418..00000000 --- a/lib/sourced/ccc.rb +++ /dev/null @@ -1,274 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/injector' - -module Sourced - module CCC - # @return [Configuration] the global CCC configuration instance - def self.config - @config ||= Configuration.new - end - - # Configure the CCC module. Stores the block for re-running after fork - # (see {.setup!}), then runs it immediately. - # @yieldparam config [Configuration] - def self.configure(&block) - @configure_block = block - setup! - end - - # Run (or re-run) the configure block on a fresh Configuration. - # Safe to call after a process fork to re-establish database connections. - # @return [void] - def self.setup! - @config = Configuration.new - @configure_block&.call(@config) - @config.setup! - @config.freeze - end - - # Register a reactor class with the global router. - # Triggers setup! if not already done. - # @param reactor [Class] a CCC reactor class - def self.register(reactor) - config.setup! - config.router.register(reactor) - end - - # @return [CCC::Store] the global store (triggers setup! if needed) - def self.store - config.setup! - config.store - end - - # @return [CCC::Router] the global router (triggers setup! if needed) - def self.router - config.setup! - config.router - end - - # Stop a consumer group and invoke the reactor's +on_stop+ callback. - # Delegates to {Router#stop_consumer_group}. - # - # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string - # @param message [String, nil] optional reason for stopping - # @return [void] - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - # @see Router#stop_consumer_group - def self.stop_consumer_group(reactor_or_id, message = nil) - config.router.stop_consumer_group(reactor_or_id, message) - end - - # Reset a consumer group and invoke the reactor's +on_reset+ callback. - # Delegates to {Router#reset_consumer_group}. - # - # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string - # @return [void] - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - # @see Router#reset_consumer_group - def self.reset_consumer_group(reactor_or_id) - config.router.reset_consumer_group(reactor_or_id) - end - - # Start a consumer group and invoke the reactor's +on_start+ callback. - # Delegates to {Router#start_consumer_group}. - # - # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string - # @return [void] - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - # @see Router#start_consumer_group - def self.start_consumer_group(reactor_or_id) - config.router.start_consumer_group(reactor_or_id) - end - - # Reset the global configuration. For test teardown. - def self.reset! - @config = nil - @configure_block = nil - @topology = nil - end - - # Build and cache the topology graph from all reactors registered with - # the global {.router}. The result is memoized; call {.reset_topology} - # to force a rebuild after registering new reactors. - # - # @return [Array] - # flat array of topology node structs - # - # @example Inspect the global topology - # Sourced::CCC.register(MyDecider) - # Sourced::CCC.register(MyProjector) - # - # Sourced::CCC.topology.each do |node| - # puts "#{node.type}: #{node.name} (#{node.id})" - # end - # - # @see CCC::Topology.build - def self.topology - @topology ||= CCC::Topology.build(router.reactors) - end - - # Clear the cached topology so it is rebuilt on next access to {.topology}. - # Useful after registering additional reactors at runtime. - # - # @return [nil] - # - # @example - # Sourced::CCC.register(LateAddedDecider) - # Sourced::CCC.reset_topology - # Sourced::CCC.topology # now includes LateAddedDecider - def self.reset_topology - @topology = nil - end - - # Returned by {.handle!} with command, reactor instance, and new events. - # Supports array destructuring: +cmd, reactor, events = CCC.handle!(cmd, MyDecider)+ - HandleResult = Data.define(:command, :reactor, :events) do - def to_ary = [command, reactor, events] - end - - # Handle a command synchronously: validate, load history, decide, append, and ACK. - # - # 1. Validates the command via +command.valid?+ - # 2. If invalid, returns immediately with the command, an uninitialized reactor, and empty events - # 3. Loads the reactor's history from the command's partition attributes - # 4. Evolves the reactor from history and runs the decider - # 5. Appends the command and correlated events to the store with optimistic concurrency - # 6. Advances consumer group offsets for registered reactors so background workers skip - # the already-handled command - # - # @param reactor_class [Class] a CCC::Decider (or any reactor extending Consumer + Evolve) - # @param command [CCC::Command] the command to handle (must respond to +valid?+) - # @param store [CCC::Store, nil] the store to use (defaults to CCC.store) - # @return [HandleResult] supports destructuring: +cmd, reactor, events = result+ - # @raise [Sourced::ConcurrentAppendError] if conflicting messages found after history read - # @raise [RuntimeError] if the decider raises a domain error (invariant violation) - # - # @example - # cmd = CourseApp::CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) - # cmd, decider, events = Sourced::CCC.handle!(CourseApp::CourseDecider, cmd) - # if cmd.valid? - # # events were appended, offsets advanced - # else - # # cmd.errors has validation details - # end - def self.handle!(reactor_class, command, store: nil) - store ||= self.store - - partition_attrs = extract_partition_attrs(command, reactor_class) - values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } - instance = reactor_class.new(values) - - unless command.valid? - return HandleResult.new(command: command, reactor: instance, events: []) - end - - # Load history if the reactor needs it (Deciders always do) - needs_history = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) - if needs_history - instance, read_result = load(reactor_class, store: store, **partition_attrs) - end - - # Decide - raw_events = instance.decide(command) - correlated_events = raw_events.map { |e| command.correlate(e) } - - # Append command + events in one transaction with consistency guard - guard = read_result&.guard - to_append = [command] + correlated_events - last_position = store.append(to_append, guard: guard) - - # Advance offsets for registered consumer groups - advance_registered_offsets(store, reactor_class, partition_attrs, last_position) - - HandleResult.new(command: command, reactor: instance, events: correlated_events) - end - - # Load a reactor instance from its event history using AND-filtered partition reads. - # Returns the evolved instance and a ReadResult (with .messages and .guard). - # - # Uses {Store#read_partition} which filters at the SQL level: a message is - # included only when every partition attribute it declares matches the given - # value. Messages that don't declare a partition attribute pass through - # (e.g. CourseCreated with only +course_id+ is included even when - # +student_id+ is in the partition). - # - # @param reactor_class [Class] a CCC reactor class (Decider, Projector, or any class - # extending CCC::Consumer that includes CCC::Evolve) - # @param store [CCC::Store, nil] the store to read from (defaults to CCC.store) - # @param values [Hash{Symbol => String}] partition attribute values - # @return [Array(reactor_instance, ReadResult)] - # - # @example - # decider, read_result = Sourced::CCC.load(MyDecider, course_id: 'Algebra', student_id: 'joe') - # decider.state # evolved state - # read_result.guard # ConsistencyGuard for subsequent appends - def self.load(reactor_class, store: nil, **values) - store ||= self.store - partition_attrs = reactor_class.partition_keys.to_h { |k| [k, values[k]] } - handled_types = reactor_class.handled_messages_for_evolve.map(&:type).uniq - read_result = store.read_partition(partition_attrs, handled_types:) - instance = reactor_class.new(values) - - instance.evolve(read_result.messages) - - [instance, read_result] - end - - # Extract partition attribute values from a command's payload, - # scoped to the reactor's declared partition_keys. - # - # @param command [CCC::Command] - # @param reactor_class [Class] - # @return [Hash{Symbol => String}] - private_class_method def self.extract_partition_attrs(command, reactor_class) - reactor_class.partition_keys.each_with_object({}) do |key, h| - value = command.payload&.respond_to?(key) ? command.payload.send(key) : nil - h[key] = value if value - end - end - - # Advance consumer group offsets for all reactors registered in the global router - # that handle the given reactor_class's messages, so background workers skip - # the already-handled command. - # - # @param store [CCC::Store] - # @param reactor_class [Class] - # @param partition_attrs [Hash{Symbol => String}] - # @param position [Integer] - private_class_method def self.advance_registered_offsets(store, reactor_class, partition_attrs, position) - return unless config.router - - partition = partition_attrs.transform_keys(&:to_s) - - config.router.reactors.each do |registered_reactor| - next unless registered_reactor == reactor_class - - store.advance_offset( - registered_reactor.group_id, - partition: partition, - position: position - ) - end - end - end -end - -require 'sourced/ccc/configuration' -require 'sourced/ccc/message' -require 'sourced/ccc/actions' -require 'sourced/ccc/consumer' -require 'sourced/ccc/evolve' -require 'sourced/ccc/react' -require 'sourced/ccc/sync' -require 'sourced/ccc/decider' -require 'sourced/ccc/projector' -require 'sourced/ccc/router' -require 'sourced/ccc/worker' -require 'sourced/ccc/stale_claim_reaper' -require 'sourced/ccc/dispatcher' -require 'sourced/ccc/command_context' -require 'sourced/ccc/topology' -require 'sourced/ccc/supervisor' -require 'sourced/ccc/durable_workflow' diff --git a/lib/sourced/ccc/README.md b/lib/sourced/ccc/README.md deleted file mode 100644 index 3df14ed8..00000000 --- a/lib/sourced/ccc/README.md +++ /dev/null @@ -1,967 +0,0 @@ -# Sourced::CCC — Stream-less Event Sourcing - -CCC ("Command Context Consistency") is an experimental module for aggregateless, stream-less event sourcing. Events go into a flat, globally-ordered log. Consistency context is assembled dynamically by querying relevant facts via key-value pairs extracted from event payloads, rather than being pre-assigned to fixed streams. - -## Core Concepts - -- **No streams or aggregates** — all messages share a single append-only log with auto-increment positions. -- **Partitioning by attributes** — reactors declare which payload attributes define their consistency boundary (e.g. `partition_by :course_id`). The store uses these to build query conditions and claim work. -- **Decide → Evolve → React** — same pattern as core Sourced, but without stream IDs or sequence numbers. -- **Optimistic concurrency** — reads return a `ConsistencyGuard` that detects conflicting writes at append time. - -## Messages - -CCC has its own message hierarchy, separate from `Sourced::Message`. Messages have no `stream_id` or `seq` — they get a global `position` when stored. - -```ruby -# Define base classes for your domain (optional but recommended) -class MyEvent < Sourced::CCC::Event; end -class MyCommand < Sourced::CCC::Command; end - -# Define typed messages with payload attributes -CreateCourse = MyCommand.define('courses.create') do - attribute :course_id, String - attribute :course_name, String -end - -CourseCreated = MyEvent.define('courses.created') do - attribute :course_id, String - attribute :course_name, String -end -``` - -### Message features - -- **Auto-generated UUIDs** for `id`, `causation_id`, and `correlation_id` -- **Causal tracing** via `#correlate(other_message)` — sets `causation_id` and `correlation_id` -- **Auto-indexed keys** — `#extracted_keys` returns `[["course_id", "c1"], ["course_name", "Algebra"]]` from payload attributes, used by the store to index messages for querying -- **Own registry** — `CCC::Message.registry` is separate from `Sourced::Message.registry`. Use `.from(type: "courses.created", payload: {...})` to instantiate from a type string. -- **Typed payloads** — uses the same Plumb/Types DSL as core Sourced for attribute coercion and validation - -## Store - -`CCC::Store` is an SQLite-backed store (via Sequel) providing the flat message log, key-pair indexing, and consumer group management. - -```ruby -require 'sequel' - -db = Sequel.sqlite('my_app.db') -store = Sourced::CCC::Store.new(db) -store.install! # creates tables (idempotent) -``` - -### Appending messages - -```ruby -cmd = CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) -position = store.append(cmd) # returns the assigned position -``` - -### Reading with query conditions - -```ruby -# Build conditions for a specific course -conditions = CourseCreated.to_conditions(course_id: 'c1') -# => [QueryCondition('courses.created', attrs: { course_id: 'c1' })] - -# Read matching messages -result = store.read(conditions) -result.messages # => [PositionedMessage, ...] -result.guard # => ConsistencyGuard (for optimistic concurrency) -``` - -### Optimistic concurrency - -```ruby -result = store.read(conditions) - -# ... later, append with conflict detection -store.append(new_events, guard: result.guard) -# raises Sourced::ConcurrentAppendError if conflicting messages -# were appended after the read -``` - -### Partition reads - -`read_partition` uses AND semantics — a message is included only when every partition attribute it declares matches the given value. - -```ruby -result = store.read_partition( - { course_id: 'c1' }, - handled_types: ['courses.created', 'courses.enrolled'] -) -``` - -### Browsing the global log - -`read_all` paginates the entire message log, without requiring query conditions or partition attributes. It returns a `ReadAllResult` with `messages` and `last_position` (the current max position in the store), so clients know whether more pages exist. - -```ruby -# First page (default limit: 50, ascending order) -result = store.read_all(limit: 20) -result.messages # => [PositionedMessage, ...] -result.last_position # => 100 (max position in the store) - -# Next page — pass the last message's position as cursor -result = store.read_all(from_position: result.messages.last.position, limit: 20) - -# Check if there are more pages -has_more = result.messages.any? && result.messages.last.position < result.last_position - -# Destructuring is also supported -messages, last_position = store.read_all(limit: 20) -``` - -Use `order: :desc` for reverse-chronological browsing (newest first). Pagination works the same way — `from_position` fetches messages *before* the given position. - -```ruby -result = store.read_all(order: :desc, limit: 20) - -# Next page of older messages -result = store.read_all(from_position: result.messages.last.position, order: :desc, limit: 20) -``` - -#### Iterating all messages with `to_enum` - -`ReadAllResult#to_enum` returns a lazy `Enumerator` that transparently fetches subsequent pages as you iterate, using the same `order` and `limit` from the original query. - -```ruby -# Iterate all messages in pages of 50 -store.read_all(limit: 50).to_enum.each do |msg| - puts "#{msg.position}: #{msg.type}" -end - -# Works with Enumerable methods -store.read_all(order: :desc, limit: 100).to_enum.map(&:type) - -# Supports lazy enumeration — stops fetching pages once satisfied -store.read_all(limit: 20).to_enum.lazy.select { |m| - m.type == 'courses.created' -}.first(5) -``` - -### Database setup - -`Store#install!` creates all required tables directly (useful for scripts, tests, and quick prototyping). For production apps using Sequel migrations, the store can export a migration file instead. - -#### Quick setup (e.g. scripts, tests) - -```ruby -db = Sequel.sqlite('my_app.db') -store = Sourced::CCC::Store.new(db) -store.install! -``` - -#### Exporting a Sequel migration - -Use `Store#copy_migration_to` to generate a migration file compatible with `Sequel::Migrator`: - -```ruby -db = Sequel.sqlite('my_app.db') -store = Sourced::CCC::Store.new(db) - -# Option 1: pass a directory (uses a default filename) -store.copy_migration_to('db/migrations') - -# Option 2: pass a block for full control over the path -store.copy_migration_to do - "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_ccc_tables.rb" -end -``` - -Then run your migrations as usual: - -```bash -sequel -m db/migrations sqlite://my_app.db -``` - -#### Custom table prefix - -By default, tables are prefixed with `sourced_` (e.g. `sourced_messages`, `sourced_consumer_groups`). Pass a `prefix:` to `Store.new` to customise this — for example when running multiple CCC stores in the same database: - -```ruby -store = Sourced::CCC::Store.new(db, prefix: 'billing') -store.install! -# Creates: billing_messages, billing_key_pairs, billing_consumer_groups, ... -``` - -The prefix is carried through to exported migrations automatically. - -#### Using the Installer directly - -The installer is also available as a standalone object, which is useful for Rake tasks or setup scripts: - -```ruby -installer = Sourced::CCC::Installer.new(db, logger: Logger.new($stdout), prefix: 'sourced') -installer.install # create tables -installer.installed? # check if tables exist -installer.uninstall # drop tables (test env only) -installer.copy_migration_to('db/migrations') -``` - -## Deciders - -Deciders handle commands, enforce invariants, and produce events. They rebuild state from event history before each decision. - -```ruby -class CourseDecider < Sourced::CCC::Decider - # Defines the consistency boundary - partition_by :course_name - - # Initial state factory (receives partition values hash) - state do |_partition_values| - { name_taken: false } - end - - # Evolve state from events (rebuilds history) - evolve CourseCreated do |state, _event| - state[:name_taken] = true - end - - # Command handler — enforce invariants, then produce events - command CreateCourse do |state, cmd| - raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] - - event CourseCreated, - course_id: cmd.payload.course_id, - course_name: cmd.payload.course_name - end -end -``` - -### Synchronous command handling - -`CCC.handle!` loads history, runs the decider, appends the command + events, and advances consumer group offsets — all in one call. Designed for web controllers. - -```ruby -cmd = CreateCourse.new(payload: { course_id: 'c1', course_name: 'Algebra' }) -cmd, decider, events = Sourced::CCC.handle!(CourseDecider, cmd) - -if cmd.valid? - # Success — events were appended -else - # Validation failure — cmd.errors has details -end -``` - -Raises `Sourced::ConcurrentAppendError` on conflicts, or `RuntimeError` on domain invariant violations (e.g. "Course already exists"). - -### CommandContext - -`CCC::CommandContext` is a factory for building CCC commands from raw Hash attributes (e.g. HTTP params), injecting defaults like `metadata`. It mirrors `Sourced::CommandContext` but without `stream_id`, since CCC messages are stream-less. - -```ruby -# In a web controller, build a context with shared metadata -ctx = Sourced::CCC::CommandContext.new( - metadata: { user_id: session[:user_id] } -) - -# Build from a type string + payload hash (e.g. from JSON params) -cmd = ctx.build(type: 'courses.create', payload: { course_id: 'c1', course_name: 'Algebra' }) -cmd.metadata[:user_id] # => session[:user_id] - -# Or pass an explicit command class -cmd = ctx.build(CreateCourse, payload: { course_id: 'c1', course_name: 'Algebra' }) -``` - -String keys are automatically symbolized, so `ctx.build('type' => '...', 'payload' => { ... })` works too. - -#### Callback hooks (`on` and `any`) - -Subclass `CommandContext` and register class-level hooks to enrich or transform commands at build time — e.g. injecting session data or adding metadata from the request scope. - -- **`on(MessageClass, ...)`** — runs for one or more command types. Multiple `on` calls for the same class accumulate (all blocks run in registration order). -- **`any`** — runs for all commands (multiple blocks allowed, executed in order) - -Both receive the `app` scope and the command, and must return the (possibly modified) command. `on` blocks run before `any` blocks. Blocks are evaluated in the context of the `CommandContext` instance, so they can call private helper methods defined on the subclass. - -```ruby -class AppCommandContext < Sourced::CCC::CommandContext - # Enrich a specific command with data from the app scope - on CreateCourse do |app, cmd| - cmd.with_payload(created_by: app.current_user.id) - end - - # Same block for multiple command types - on EnrolStudent, DropStudent do |app, cmd| - cmd.with_metadata(campus: app.current_campus) - end - - # Additional block for EnrolStudent — both blocks run in order - on EnrolStudent do |app, cmd| - cmd.with_metadata(enrolment_source: 'web') - end - - # Add metadata to every command - any do |app, cmd| - cmd.with_metadata( - request_id: app.request_id, - session_id: app.session_id - ) - end -end -``` - -Pass the request-scoped `app` object at construction time: - -```ruby -# In a web controller -ctx = AppCommandContext.new( - metadata: { user_id: session[:user_id] }, - app: self # e.g. Sinatra app instance, Rack env wrapper, etc. -) - -cmd = ctx.build(type: 'courses.create', payload: { course_id: 'c1', course_name: 'Algebra' }) -cmd.metadata[:request_id] # => set by the `any` hook -``` - -`app` defaults to `nil`, so existing callers without hooks are unaffected. Hooks are inherited by subclasses. - -Since blocks run in instance context, you can extract shared logic into private methods: - -```ruby -class AppCommandContext < Sourced::CCC::CommandContext - on CreateCourse do |app, cmd| - cmd.with_metadata(user_id: build_user_id(app)) - end - - private - - def build_user_id(app) - "user-#{app.session_id}" - end -end -``` - -#### Scoping to a command subset - -By default, `CommandContext` looks up types in `CCC::Command.registry`. Pass a `scope:` to restrict lookups to a specific command subclass — attempts to build commands outside the scope raise `Sourced::UnknownMessageError`. - -```ruby -class PublicCommand < Sourced::CCC::Command; end - -CreateCourse = PublicCommand.define('courses.create') do - attribute :course_id, String - attribute :course_name, String -end - -# Only PublicCommand subclasses are allowed -ctx = Sourced::CCC::CommandContext.new(scope: PublicCommand) -ctx.build(type: 'courses.create', payload: { ... }) # OK -ctx.build(type: 'admin.delete_all', payload: {}) # raises UnknownMessageError -``` - -### Loading a decider's state - -```ruby -decider, read_result = Sourced::CCC.load(CourseDecider, course_name: 'Algebra') -decider.state # => { name_taken: true } -``` - -## Projectors - -Projectors consume events to build read models. Two flavours: - -### EventSourced projector - -Rebuilds state from full history on every batch (like deciders). - -```ruby -class CourseCatalogProjector < Sourced::CCC::Projector::EventSourced - partition_by :course_id - - state do |_partition_values| - { course_id: nil, course_name: nil, students: [] } - end - - evolve CourseCreated do |state, event| - state[:course_id] = event.payload.course_id - state[:course_name] = event.payload.course_name - end - - evolve StudentEnrolled do |state, event| - state[:students] << event.payload.student_id - end - - # Sync block runs within the store transaction after evolving - sync do |state:, messages:, **| - next unless state[:course_id] - # Write projection to disk, database, cache, etc. - File.write("projections/#{state[:course_id]}.json", state.to_json) - end - - # After-sync block runs after the transaction commits. - # Use for side effects that should only happen on successful commit - # (e.g. sending emails, HTTP calls, pushing to external queues). - after_sync do |state:, messages:, **| - NotificationService.notify("Course #{state[:course_name]} updated") - end -end -``` - -### StateStored projector - -Loads persisted state via the `state` block, evolves only new (unprocessed) messages. - -```ruby -class MyProjector < Sourced::CCC::Projector::StateStored - partition_by :course_id - - state do |partition_values| - # Load existing state from your storage - existing = MyDB.find(partition_values[:course_id]) - existing || { course_id: nil, students: [] } - end - - evolve StudentEnrolled do |state, event| - state[:students] << event.payload.student_id - end - - sync do |state:, messages:, **| - MyDB.upsert(state) - end -end -``` - -## Reactions - -Both deciders and projectors can react to events to produce new commands or events, enabling workflow orchestration. - -```ruby -class EnrolmentDecider < Sourced::CCC::Decider - partition_by :course_id - - # ... evolve and command handlers ... - - # React to an event by producing new messages - reaction StudentEnrolled do |state, event| - NotifyStudent.new(payload: { student_id: event.payload.student_id }) - end -end -``` - -Reactions are skipped during replay (when `replaying: true`), so side effects don't re-fire. - -## Sync and After-Sync Blocks - -Both deciders and projectors support `sync` and `after_sync` blocks for running side effects during message processing. - -- **`sync`** blocks run **inside** the store transaction, alongside event persistence and offset acknowledgement. Use them for writes that must be atomic with the event append (e.g. updating a database projection). -- **`after_sync`** blocks run **after** the transaction commits. Use them for side effects that should only happen if the commit succeeds (e.g. sending emails, HTTP calls, pushing to external queues). - -Both receive the same keyword arguments as the reactor's action-building step: - -| Reactor type | Keyword arguments | -|--------------|---------------------------------------| -| Decider | `state:`, `messages:`, `events:` | -| Projector | `state:`, `messages:`, `replaying:` | - -```ruby -class OrderDecider < Sourced::CCC::Decider - partition_by :order_id - - # ... evolve / command handlers ... - - sync do |state:, messages:, events:| - # Runs inside the transaction - OrderCache.update(state[:order_id], state) - end - - after_sync do |state:, messages:, events:| - # Runs after successful commit - Mailer.send_confirmation(state[:order_id]) if events.any? { |e| e.is_a?(OrderPlaced) } - end -end -``` - -Multiple `sync` and `after_sync` blocks can be registered; they execute in registration order. Blocks are inherited by subclasses. - -## Configuration - -```ruby -require 'sourced/ccc' - -Sourced::CCC.configure do |c| - # Pass a Sequel SQLite connection or a CCC::Store instance - c.store = Sequel.sqlite('my_app.db') - - # Optional settings - c.worker_count = 4 # background worker fibers (default: 2) - c.batch_size = 50 # messages per claim (default: 50) - c.catchup_interval = 5 # seconds between catch-up polls (default: 5) - c.max_drain_rounds = 10 # max drain iterations per pickup (default: 10) - c.claim_ttl_seconds = 120 # stale claim threshold (default: 120) - c.housekeeping_interval = 30 # heartbeat/reap cycle (default: 30) -end -``` - -## Failure handling and retries - -CCC already supports consumer-group retries on failure. - -- On reactor errors, `Router#handle_next_for` calls the reactor's `on_exception` hook. -- By default, that hook uses `CCC.config.error_strategy`. -- The default `Sourced::ErrorStrategy` marks the consumer group as failed immediately. -- If you configure a retrying error strategy, CCC stores the next retry time in the consumer group's `retry_at` column and skips claiming work for that group until that time has passed. - -So retries are built in already, but they are opt-in via the error strategy configuration. - -### Example: exponential backoff retries - -```ruby -require 'sourced/ccc' - -Sourced::CCC.configure do |c| - c.store = Sequel.sqlite('my_app.db') - - c.error_strategy = Sourced::ErrorStrategy.new do |s| - s.retry( - times: 5, - after: 2, - backoff: ->(retry_after, retry_count) { retry_after * (2**(retry_count - 1)) } - ) - - s.on_retry do |retry_count, exception, message, later| - LOGGER.warn( - "CCC retry ##{retry_count} for #{message.type} (#{message.id}) " \ - "at #{later}: #{exception.class}: #{exception.message}" - ) - end - - s.on_fail do |exception, message| - LOGGER.error( - "CCC failing consumer group after retries for #{message.type} (#{message.id}): " \ - "#{exception.class}: #{exception.message}" - ) - end - end -end -``` - -With the configuration above, failures retry after: - -- retry 1: 2 seconds -- retry 2: 4 seconds -- retry 3: 8 seconds -- retry 4: 16 seconds -- retry 5: 32 seconds - -After the configured retries are exhausted, the consumer group is marked as failed. - -## Registering reactors - -```ruby -Sourced::CCC.register(CourseDecider) -Sourced::CCC.register(EnrolmentDecider) -Sourced::CCC.register(CourseCatalogProjector) -``` - -This registers the reactor's consumer group with the store and adds it to the global router. - -## Background processing - -### Falcon (recommended) - -`CCC::Falcon` provides a ready-made Falcon service that runs both the web server and CCC background workers as sibling fibers. No separate worker process needed. - -```ruby -# falcon.rb -#!/usr/bin/env falcon-host -require_relative 'domain' -require_relative 'app' -require 'sourced/ccc/falcon' - -service "my-app" do - include Sourced::CCC::Falcon::Environment - include Falcon::Environment::Rackup - - url "http://localhost:9292" - count 1 -end -``` - -Start with: - -```bash -bundle exec falcon host -``` - -The service automatically calls `CCC.setup!` in each forked process, which replays the `CCC.configure` block to create fresh database connections. This is necessary because SQLite connections are not fork-safe. - -#### How it works - -- `CCC::Falcon::Environment` — mixin that sets the `service_class` to `CCC::Falcon::Service`. Include it in your Falcon service definition alongside `Falcon::Environment::Rackup`. -- `CCC::Falcon::Service` — extends `Falcon::Service::Server`. On `run`, it calls `CCC.setup!`, starts the web server, and spawns a `CCC::Dispatcher` with all settings from `CCC.config`. On `stop`, it shuts down the dispatcher before the server. -- No separate HouseKeeper fibers are needed — the `StaleClaimReaper` is embedded in the CCC Dispatcher. - -### Supervisor (standalone) - -For running workers without a web server, the supervisor starts workers that claim partitions, process messages, and ack offsets. - -```ruby -# Start blocking (handles INT/TERM signals for graceful shutdown) -Sourced::CCC::Supervisor.start - -# Or create and start manually -supervisor = Sourced::CCC::Supervisor.new( - router: Sourced::CCC.router, - count: 4 -) -supervisor.start -``` - -### How it works - -1. **Store** appends messages and notifies listeners of new message types -2. **Dispatcher** routes notifications to a `WorkQueue`, mapping message types to interested reactors -3. **Workers** pop reactors from the queue, claim a partition via `Router#handle_next_for`, process messages, and ack -4. **CatchUpPoller** periodically pushes all reactors as a safety net (handles missed notifications) -5. **ScheduledMessagePoller** promotes due delayed messages into the main CCC log -6. **StaleClaimReaper** releases claims held by dead workers - -### Router (direct usage) - -The router can also be used directly for testing or scripting: - -```ruby -router = Sourced::CCC.router - -# Process one batch for a specific reactor -router.handle_next_for(CourseDecider) - -# Drain all pending work across all reactors -router.drain -``` - -## Consumer groups - -Each reactor class is a consumer group. The store tracks per-partition offsets so multiple reactors process the same events independently. - -The lifecycle methods (`stop_consumer_group`, `start_consumer_group`, `reset_consumer_group`, `consumer_group_active?`) accept either a String group ID or any object responding to `#group_id` (e.g. a reactor class). - -```ruby -store = Sourced::CCC.store - -# Pass reactor classes directly -store.stop_consumer_group(CourseDecider) -store.start_consumer_group(CourseDecider) -store.reset_consumer_group(CourseDecider) # reprocess from beginning -store.consumer_group_active?(CourseDecider) # => true/false - -# Or use plain strings -store.stop_consumer_group('CourseApp::CourseDecider') -``` - -When retries are configured via `CCC.config.error_strategy`, failed consumer groups remain active but paused until their `retry_at` time. Once that time passes, they become claimable again automatically. - -### Lifecycle hooks via Router - -The Router provides lifecycle methods that wrap the Store operations and invoke optional callbacks on the reactor class. This lets reactors run cleanup or setup logic when their consumer group is stopped, reset, or started. - -```ruby -# Accept a reactor class or a string group_id -Sourced::CCC.stop_consumer_group(CourseDecider, 'maintenance window') -Sourced::CCC.reset_consumer_group(CourseDecider) -Sourced::CCC.start_consumer_group(CourseDecider) - -# String group_id works too — the router resolves it to the registered class -Sourced::CCC.stop_consumer_group('CourseApp::CourseDecider') -``` - -These delegate to `Router#stop_consumer_group`, `Router#reset_consumer_group`, and `Router#start_consumer_group`, which: - -1. Resolve the argument to a registered reactor class (raising `ArgumentError` if the string doesn't match any registered reactor) -2. Call the corresponding `Store` method -3. Invoke the reactor's callback (`on_stop`, `on_reset`, `on_start`) - -#### Defining callbacks - -Override the no-op class methods on your reactor to hook into lifecycle events: - -```ruby -class CourseDecider < Sourced::CCC::Decider - partition_by :course_name - - # Called when the consumer group is stopped. - # `message` is the optional reason string passed to stop_consumer_group. - def self.on_stop(message = nil) - Rails.logger.info "CourseDecider stopped: #{message}" - end - - # Called when the consumer group is reset (offsets cleared). - def self.on_reset - Rails.cache.delete_matched('course_projections/*') - end - - # Called when the consumer group is started. - def self.on_start - Rails.logger.info 'CourseDecider started' - end -end -``` - -Reactors without custom callbacks work fine — the defaults are no-ops. - -## Monitoring - -`Store#stats` returns system-wide diagnostics for monitoring and debugging CCC deployments. - -```ruby -stats = store.stats -stats.max_position # => 42 (latest position in the message log) -stats.groups # => array of per-consumer-group hashes -``` - -Each group hash contains: - -| Key | Description | -|----------------------|----------------------------------------------------------------| -| `group_id` | Consumer group identifier (e.g. `"CourseDecider"`) | -| `status` | `"active"`, `"stopped"`, or `"failed"` | -| `retry_at` | `Time` of next retry, or `nil` | -| `error_context` | Hash with error details (`{}` when healthy, see below) | -| `oldest_processed` | `MIN(last_position)` across partitions where processing started | -| `newest_processed` | `MAX(last_position)` across partitions | -| `partition_count` | Number of offset rows (partitions) for this group | - -### `error_context` - -The `error_context` hash is empty (`{}`) for healthy groups. When a group is stopped or has failed, it may contain: - -| Key | Present when | Description | -|----------------------|--------------|--------------------------------------| -| `:message` | Stopped | Operator-supplied reason for stopping | -| `:exception_class` | Failed | Exception class name (e.g. `"RuntimeError"`) | -| `:exception_message` | Failed | Exception message string | - -When retries are configured, `error_context` also accumulates retry state set by `GroupUpdater#retry_later`. - -```ruby -stats = store.stats -stats.groups.each do |g| - puts "#{g[:group_id]}: #{g[:status]} (#{g[:partition_count]} partitions, up to position #{g[:newest_processed]})" - if g[:status] == 'failed' - puts " error: #{g[:error_context][:exception_class]}: #{g[:error_context][:exception_message]}" - end -end -``` - -### `Store#read_offsets` — inspecting partition offsets - -`read_offsets` lists individual consumer group offsets with optional filtering and cursor-based pagination. Useful for inspecting the progress and claim status of each partition. - -```ruby -result = store.read_offsets -result.offsets # => array of offset hashes -result.total_count # => total number of matching offsets (ignoring pagination) -``` - -#### Parameters - -| Parameter | Type | Default | Description | -|-------------|----------------|---------|----------------------------------------------------------| -| `group_id:` | `String`, `nil` | `nil` | Filter by consumer group. `nil` returns all groups. | -| `limit:` | `Integer` | `50` | Max offsets per page. | -| `from_id:` | `Integer`, `nil`| `nil` | Cursor — return offsets with `id >= from_id` (inclusive). | - -#### Offset hash fields - -Each offset in the result is a Hash with: - -| Key | Type | Description | -|------------------|---------------|------------------------------------------------------| -| `:id` | `Integer` | Offset primary key (used as pagination cursor) | -| `:group_name` | `String` | Consumer group identifier | -| `:group_status` | `String` | `"active"`, `"stopped"`, or `"failed"` | -| `:partition_key` | `String` | Partition identifier (e.g. `"device_id:dev-1"`) | -| `:last_position` | `Integer` | Highest acked position for this partition | -| `:claimed` | `Boolean` | Whether a worker currently holds this partition | -| `:claimed_at` | `String`, `nil`| ISO8601 timestamp of the claim | -| `:claimed_by` | `String`, `nil`| Worker ID holding the claim | - -#### Filtering by group - -```ruby -result = store.read_offsets(group_id: 'CourseDecider') -result.offsets.each do |o| - puts "#{o[:partition_key]}: position #{o[:last_position]}, claimed=#{o[:claimed]}" -end -``` - -#### Pagination - -```ruby -# First page -page1 = store.read_offsets(limit: 20) - -# Next page using cursor -page2 = store.read_offsets(limit: 20, from_id: page1.offsets.last[:id] + 1) -``` - -#### Auto-pagination with `to_enum` - -`OffsetsResult#to_enum` returns a lazy `Enumerator` that fetches subsequent pages automatically. - -```ruby -# Iterate all offsets in pages of 20 -store.read_offsets(limit: 20).to_enum.each do |offset| - puts "#{offset[:group_name]} / #{offset[:partition_key]}: #{offset[:last_position]}" -end - -# Works with Enumerable methods -behind = store.read_offsets(limit: 50).to_enum.lazy.select { |o| - o[:last_position] < store.latest_position - 100 -}.to_a -``` - -#### Array destructuring - -```ruby -offsets, total_count = store.read_offsets(group_id: 'CourseDecider') -``` - -## Testing - -CCC ships with RSpec helpers for Given-When-Then testing of deciders and projectors. The helpers call `handle_batch` directly — no store, router, or consumer group setup needed. - -```ruby -require 'sourced/ccc/testing/rspec' - -RSpec.configure do |config| - config.include Sourced::CCC::Testing::RSpec -end -``` - -### Testing deciders - -`with_reactor` takes a decider class and partition attributes, then chains `.given` (history), `.when` (command), and `.then` (expected outcomes). - -```ruby -RSpec.describe CourseDecider do - include Sourced::CCC::Testing::RSpec - - it 'creates a course' do - with_reactor(CourseDecider, course_name: 'Algebra') - .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') - .then(CourseCreated, course_id: 'c1', course_name: 'Algebra') - end - - it 'rejects duplicate course names' do - with_reactor(CourseDecider, course_name: 'Algebra') - .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') - .when(CreateCourse, course_id: 'c2', course_name: 'Algebra') - .then(RuntimeError, "Course 'Algebra' already exists") - end - - it 'produces no events for a no-op command' do - with_reactor(CourseDecider, course_name: 'Algebra') - .when(SomeNoopCommand, course_name: 'Algebra') - .then([]) - end -end -``` - -#### Multiple expected messages - -When a decider produces events and reactions, pass all expected messages as instances: - -```ruby -it 'produces event and reaction' do - with_reactor(EnrolmentDecider, course_id: 'c1') - .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') - .when(EnrolStudent, course_id: 'c1', student_id: 's1') - .then( - StudentEnrolled.new(payload: { course_id: 'c1', student_id: 's1' }), - NotifyStudent.new(payload: { student_id: 's1' }) - ) -end -``` - -#### Block form - -Pass a block to `.then` to receive the raw action pairs for custom assertions: - -```ruby -it 'inspects action pairs' do - with_reactor(CourseDecider, course_name: 'Algebra') - .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') - .then { |pairs| - actions, source_msg = pairs.first - append = Array(actions).find { |a| a.is_a?(Sourced::CCC::Actions::Append) } - expect(append.messages.first).to be_a(CourseCreated) - } -end -``` - -#### `.then!` — run sync and after_sync actions - -Use `.then!` instead of `.then` to execute both `sync` and `after_sync` actions before assertions: - -```ruby -it 'runs sync block' do - with_reactor(CourseDecider, course_name: 'Algebra') - .when(CreateCourse, course_id: 'c1', course_name: 'Algebra') - .then! { |pairs| ... } -end -``` - -### Testing projectors - -Projectors use `.given` (events to evolve) and `.then` with a block that receives the projected state. `.when` is not supported — projectors don't handle commands. - -#### StateStored - -```ruby -RSpec.describe ItemProjector do - include Sourced::CCC::Testing::RSpec - - it 'builds state from events' do - with_reactor(ItemProjector, list_id: 'L1') - .given(ItemAdded, list_id: 'L1', name: 'Apple') - .given(ItemAdded, list_id: 'L1', name: 'Banana') - .then { |state| expect(state[:items]).to eq(['Apple', 'Banana']) } - end - - it 'handles removal' do - with_reactor(ItemProjector, list_id: 'L1') - .given(ItemAdded, list_id: 'L1', name: 'Apple') - .and(ItemArchived, list_id: 'L1', name: 'Apple') - .then { |state| expect(state[:items]).to eq([]) } - end - - it 'runs sync actions with then!' do - with_reactor(ItemProjector, list_id: 'L1') - .given(ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |state| expect(state[:synced]).to be true } - end -end -``` - -#### EventSourced - -Same API — the helper creates an instance, evolves from all given messages, and yields state: - -```ruby -RSpec.describe CatalogProjector do - include Sourced::CCC::Testing::RSpec - - it 'rebuilds state from full history' do - with_reactor(CatalogProjector, course_id: 'c1') - .given(CourseCreated, course_id: 'c1', course_name: 'Algebra') - .given(StudentEnrolled, course_id: 'c1', student_id: 's1') - .then { |state| expect(state[:students]).to eq(['s1']) } - end -end -``` - -### Message matching - -`.then` compares messages by **class** and **payload** only. Fields like `id`, `created_at`, `causation_id`, `correlation_id`, and `metadata` are ignored, so tests don't need to match auto-generated values. - -## Full example - -See `examples/ccc_app/` for a complete Sinatra application with: -- Two deciders (course creation with name uniqueness, student enrolment with capacity limits) -- An event-sourced projector writing JSON files -- Synchronous command handling via `CCC.handle!` in HTTP endpoints -- Background worker processing via Falcon diff --git a/lib/sourced/ccc/actions.rb b/lib/sourced/ccc/actions.rb deleted file mode 100644 index 455b622c..00000000 --- a/lib/sourced/ccc/actions.rb +++ /dev/null @@ -1,145 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Action builders and executable action types for CCC reactors. - module Actions - OK = :ok - RETRY = :retry - - # Split produced messages into immediate append actions and delayed schedule actions. - # - # @param messages [CCC::Message, Array] messages produced by a reactor - # @param guard [ConsistencyGuard, nil] optional concurrency guard for immediate appends - # @param source [CCC::Message, nil] source message used for correlation when executing - # @param correlated [Boolean] whether +messages+ are already correlated - # @return [Array] executable actions in append/schedule groups - def self.build_for(messages, guard: nil, source: nil, correlated: false) - actions = [] - messages = Array(messages) - return actions if messages.empty? - - now = Time.now - to_schedule, to_append = messages.partition { |message| message.created_at > now } - - actions << Append.new(to_append, guard:, source:, correlated:) if to_append.any? - to_schedule.group_by(&:created_at).each do |at, scheduled_messages| - actions << Schedule.new(scheduled_messages, at:, source:, correlated:) - end - - actions - end - - # Append messages to the CCC store with optional consistency guard. - # Auto-correlates messages with the source message at execution time. - # - # When +source:+ is provided, it overrides the runtime's source_message - # for correlation (e.g. reactions correlated with the event, not the command). - # - # When +correlated: true+, messages are assumed to be already correlated - # and are appended as-is without re-correlation. - class Append - attr_reader :messages, :guard, :source - - # @param messages [CCC::Message, Array] messages to append - # @param guard [ConsistencyGuard, nil] optional optimistic concurrency guard - # @param source [CCC::Message, nil] explicit correlation source - # @param correlated [Boolean] whether +messages+ are already correlated - def initialize(messages, guard: nil, source: nil, correlated: false) - @messages = Array(messages) - @guard = guard - @source = source - @correlated = correlated - end - - # @return [Boolean] whether messages should be appended without re-correlation - def correlated? = @correlated - - # @param store [CCC::Store] - # @param source_message [CCC::Message] default message to correlate from - # @return [Array] correlated messages that were appended - def execute(store, source_message) - to_append = if @correlated - messages - else - correlate_from = @source || source_message - messages.map { |m| correlate_from.correlate(m) } - end - store.append(to_append, guard: guard) - to_append - end - end - - # Schedule messages for future promotion into the main CCC log. - class Schedule - attr_reader :messages, :at, :source - - # @param messages [CCC::Message, Array] messages to schedule - # @param at [Time] when the messages should become available for promotion - # @param source [CCC::Message, nil] explicit correlation source - # @param correlated [Boolean] whether +messages+ are already correlated - def initialize(messages, at:, source: nil, correlated: false) - @messages = Array(messages) - @at = at - @source = source - @correlated = correlated - end - - # @return [Boolean] whether messages should be scheduled without re-correlation - def correlated? = @correlated - - # @param store [CCC::Store] - # @param source_message [CCC::Message] default message to correlate from - # @return [Array] correlated messages that were scheduled - def execute(store, source_message) - to_schedule = if @correlated - messages - else - correlate_from = @source || source_message - messages.map { |message| correlate_from.correlate(message) } - end - store.schedule_messages(to_schedule, at: at) - to_schedule - end - end - - # Execute a synchronous side effect within the current transaction. - class Sync - # @param work [#call] callable to execute - def initialize(work) - @work = work - end - - # @return [Object] the callable's return value - def call = @work.call - - # @param _store [Object] unused - # @param _source_message [Object] unused - # @return [nil] - def execute(_store, _source_message) - call - nil - end - end - - # Execute a side effect after the transaction commits. - class AfterSync - # @param work [#call] callable to execute - def initialize(work) - @work = work - end - - # @return [Object] the callable's return value - def call = @work.call - - # @param _store [Object] unused - # @param _source_message [Object] unused - # @return [nil] - def execute(_store, _source_message) - call - nil - end - end - end - end -end diff --git a/lib/sourced/ccc/command_context.rb b/lib/sourced/ccc/command_context.rb deleted file mode 100644 index dabaa4c5..00000000 --- a/lib/sourced/ccc/command_context.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/types' - -module Sourced - module CCC - # Builds CCC command instances with shared default metadata. - # - # @example Build a command from a type string - # ctx = CommandContext.new(metadata: { user_id: 10 }) - # cmd = ctx.build(type: 'orders.place', payload: { item: 'hat' }) - # cmd.metadata[:user_id] # => 10 - # - # @example Scope to a custom command subclass - # scope = Class.new(CCC::Command) - # MyCmd = scope.define('my.cmd') { attribute :name, String } - # ctx = CommandContext.new(scope: scope) - # cmd = ctx.build(type: 'my.cmd', payload: { name: 'hello' }) - class CommandContext - class << self - # Register a block to run when building specific command type(s). - # The block receives the app scope and the command, and must return the (possibly modified) command. - # - # @param message_classes [Class] one or more command classes to match - # @yield [app, cmd] transformation block - # @return [void] - def on(*message_classes, &block) - message_classes.each { |klass| (message_blocks[klass] ||= []) << block } - end - - # Register a block to run for all commands. - # The block receives the app scope and the command, and must return the (possibly modified) command. - # - # @yield [app, cmd] transformation block - # @return [void] - def any(&block) - any_blocks << block - end - - # @api private - def message_blocks - @message_blocks ||= {} - end - - # @api private - def any_blocks - @any_blocks ||= [] - end - - # @api private - def inherited(subclass) - super - message_blocks.each { |k, v| subclass.message_blocks[k] = v.dup } - any_blocks.each { |blk| subclass.any_blocks << blk } - end - end - - # @param metadata [Hash] default metadata merged into every command built by this context - # @param scope [Class] message class whose registry is used to resolve type strings (default: {CCC::Command}) - # @param app [Object, nil] request-scoped object passed to callback blocks - # - # @example - # ctx = CommandContext.new(metadata: { user_id: 42 }, app: rack_app) - def initialize(metadata: Plumb::BLANK_HASH, scope: CCC::Command, app: nil) - @defaults = { metadata: }.freeze - @scope = scope - @app = app - end - - # Build a command instance, merging in default metadata. - # - # @overload build(attrs) - # Resolve the command class from the +type+ key in +attrs+ via the scope's registry. - # @param attrs [Hash] must include +:type+ and +:payload+ keys - # @return [CCC::Message] - # @example - # ctx.build(type: 'orders.place', payload: { item: 'hat' }) - # - # @overload build(klass, attrs) - # Use the given command class directly. - # @param klass [Class] a CCC::Command subclass - # @param attrs [Hash] must include +:payload+ key - # @return [CCC::Message] - # @example - # ctx.build(PlaceOrder, payload: { item: 'hat' }) - # - # @raise [ArgumentError] if arguments don't match either form - # @raise [Sourced::UnknownMessageError] if the type string is not registered in the scope - def build(*args) - cmd = case args - in [Class => klass, Hash => attrs] - attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) - klass.parse(attrs) - in [Hash => attrs] - attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) - scope.from(attrs) - else - raise ArgumentError, "Invalid arguments: #{args.inspect}" - end - run_pipeline(cmd) - end - - private - - EMPTY_ARRAY = [].freeze - - attr_reader :defaults, :scope, :app - - def run_pipeline(cmd) - blocks = self.class.message_blocks[cmd.class] || EMPTY_ARRAY - steps = blocks + self.class.any_blocks - steps.reduce(cmd) { |c, st| instance_exec(app, c, &st) } - end - end - end -end diff --git a/lib/sourced/ccc/configuration.rb b/lib/sourced/ccc/configuration.rb deleted file mode 100644 index c6e30842..00000000 --- a/lib/sourced/ccc/configuration.rb +++ /dev/null @@ -1,77 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - class Configuration - StoreInterface = Types::Interface[ - :installed?, - :install!, - :append, - :read, - :read_partition, - :claim_next, - :ack, - :release, - :register_consumer_group, - :worker_heartbeat, - :release_stale_claims, - :notifier - ] - - attr_accessor :logger, :worker_count, :batch_size, - :catchup_interval, :max_drain_rounds, - :claim_ttl_seconds, :housekeeping_interval - - attr_reader :store, :router - - def initialize - @logger = Sourced.config.logger - @worker_count = 2 - @batch_size = 50 - @catchup_interval = 5 - @max_drain_rounds = 10 - @claim_ttl_seconds = 120 - @housekeeping_interval = 30 - @store = nil - @router = nil - @error_strategy = nil - @setup = false - end - - # Accepts either a CCC::Store instance or a Sequel::SQLite::Database connection. - # When given a DB connection, wraps it in CCC::Store.new(db). - # Accepts a CCC::Store, a Sequel::SQLite::Database (auto-wrapped), - # or any object implementing StoreInterface. - def store=(s) - @store = case s.class.name - when 'Sequel::SQLite::Database' - require 'sourced/ccc/store' - Store.new(s) - else StoreInterface.parse(s) - end - end - - def error_strategy=(strategy) - raise ArgumentError, 'Must respond to #call' unless strategy.respond_to?(:call) - - @error_strategy = strategy - end - - def error_strategy - @error_strategy || Sourced.config.error_strategy - end - - def setup! - return if @setup - - unless @store - require 'sourced/ccc/store' - @store = Store.new(Sequel.sqlite) - end - @store.install! - @router ||= Router.new(store: @store) - @setup = true - end - end - end -end diff --git a/lib/sourced/ccc/consumer.rb b/lib/sourced/ccc/consumer.rb deleted file mode 100644 index 549e484e..00000000 --- a/lib/sourced/ccc/consumer.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Accumulates mutations to a consumer group row for atomic persistence. - # Mirrors Sourced::Backends::SequelBackend::GroupUpdater. - # Used by {Store#updating_consumer_group}. - class GroupUpdater - attr_reader :group_id, :updates, :error_context - - def initialize(group_id, row, logger) - @group_id = group_id - @logger = logger - @error_context = row[:error_context] - @updates = { error_context: @error_context.dup } - end - - def stop(message: nil) - @logger.error "CCC: stopping consumer group #{group_id}" - @updates[:status] = Store::STOPPED - @updates[:retry_at] = nil - @updates[:updated_at] = Time.now.iso8601 - @updates[:error_context][:message] = message if message - end - - def fail(exception: nil) - @logger.error "CCC: failing consumer group #{group_id}. #{exception&.class}: #{exception&.message}" - @updates[:status] = Store::FAILED - @updates[:retry_at] = nil - @updates[:updated_at] = Time.now.iso8601 - if exception - @updates[:error_context][:exception_class] = exception.class.to_s - @updates[:error_context][:exception_message] = exception.message - end - end - - def retry(time, **ctx) - @logger.warn "CCC: retrying consumer group #{group_id} at #{time}" - @updates[:updated_at] = Time.now.iso8601 - @updates[:retry_at] = time.iso8601 - @updates[:error_context].merge!(ctx) - end - end - - # Shared consumer configuration for CCC reactors. - # Extended (not included) onto reactor classes. - module Consumer - def self.extended(base) - super - base.extend ClassMethods - end - - def partition_keys - @partition_keys ||= [] - end - - def partition_by(*keys) - @partition_keys = keys.flatten.map(&:to_sym) - end - - def group_id - @group_id ||= name - end - - def consumer_group(id) - @group_id = id - end - - # Message types this consumer evolves from. Used by {#context_for} - # to build query conditions for history reads. - # Defaults to empty; overridden by CCC::Evolve mixin. - def handled_messages_for_evolve - @handled_messages_for_evolve ||= [] - end - - # Build query conditions from partition attributes and handled evolve types. - # Override in reactor for custom per-command conditions. - def context_for(partition_attrs) - handled_messages_for_evolve.flat_map { |klass| - klass.to_conditions(**partition_attrs) - } - end - - def on_exception(exception, message, group) - CCC.config.error_strategy.call(exception, message, group) - end - - # Called by {Router#stop_consumer_group} after the group is marked as stopped. - # Override in reactor classes to run cleanup logic on stop. - # - # @param message [String, nil] optional reason for stopping - # @return [void] - def on_stop(message = nil) - # no-op by default - end - - # Called by {Router#reset_consumer_group} after the group's offsets are cleared. - # Override in reactor classes to run cleanup logic on reset - # (e.g. clearing caches or projections). - # - # @return [void] - def on_reset - # no-op by default - end - - # Called by {Router#start_consumer_group} after the group is marked as active. - # Override in reactor classes to run setup logic on start. - # - # @return [void] - def on_start - # no-op by default - end - - # Iterate messages collecting [actions, message] pairs. - # On mid-batch failure, raises PartialBatchError with pairs collected so far. - # If the first message fails, re-raises the original error. - def each_with_partial_ack(messages) - results = [] - messages.each do |msg| - pair = yield(msg) - results << pair if pair - rescue StandardError => e - raise e if results.empty? - raise Sourced::PartialBatchError.new(results, msg, e) - end - results - end - - module ClassMethods - # Resolve a CCC message class from a symbol or type-like string. - # - # Symbols are normalized by replacing dots with underscores before - # matching against registered message types. For example, - # +:course_created+ matches "course.created" and - # "course_created". - # - # @param message_symbol [Symbol, String] symbolic message identifier - # @return [Class, nil] matching CCC message class, or +nil+ if none found - # - # @example - # CourseDecider[:courses_created] - # # => CourseCreated - def [](message_symbol) - normalized = message_symbol.to_s.tr('.', '_') - find_registered_message_class(normalized) - end - - private - - def find_registered_message_class(normalized_name, base = CCC::Message) - base.registry.keys.each do |type| - klass = base.registry[type] - return klass if type.tr('.', '_') == normalized_name - end - - base.subclasses.each do |subclass| - klass = find_registered_message_class(normalized_name, subclass) - return klass if klass - end - - nil - end - end - end - end -end diff --git a/lib/sourced/ccc/decider.rb b/lib/sourced/ccc/decider.rb deleted file mode 100644 index dc72adca..00000000 --- a/lib/sourced/ccc/decider.rb +++ /dev/null @@ -1,134 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Reactor base class for command-handling workflows in CCC. - class Decider - include CCC::Evolve - include CCC::React - include CCC::Sync - extend CCC::Consumer - - class << self - # @return [Array] command message classes handled by this decider - def handled_commands - @handled_commands ||= [] - end - - # Messages to claim: commands to decide on + events to react to. - # Evolve types are NOT included — they are only for context_for(). - # - # @return [Array] command and reaction message classes - def handled_messages - handled_commands + handled_messages_for_react - end - - # Register a command handler. - # - # @param message_class [Class] CCC command class to handle - # @yield [state, message] command handler block - # @return [void] - def command(message_class, &block) - handled_commands << message_class - define_method(Sourced.message_method_name('ccc_decide', message_class.to_s), &block) - end - - def handle_batch(partition_values, new_messages, history:, replaying: false) - instance = new(partition_values) - instance.evolve(history.messages) - - each_with_partial_ack(new_messages) do |msg| - if handled_commands.include?(msg.class) - raw_events = instance.decide(msg) - correlated_events = raw_events.map { |e| msg.correlate(e) } - actions = [] - actions.concat( - Actions.build_for(correlated_events, guard: history.guard, correlated: true) - ) - - correlated_events.each do |evt| - next unless instance.reacts_to?(evt) - reaction_msgs = Array(instance.react(evt)) - actions.concat(Actions.build_for(reaction_msgs, source: evt)) - end - - actions += instance.collect_actions( - state: instance.state, messages: [msg], events: raw_events - ) - - [actions, msg] - else - [Actions::OK, msg] - end - end - end - - # Build executable actions for a claimed batch. - # - # @param claim [ClaimResult] claimed partition batch - # @param history [ReadResult] event history for the partition - # @return [Array, PositionedMessage)>] action/source pairs - def handle_claim(claim, history:) - values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } - handle_batch(values, claim.messages, history:) - end - - # Copy registered command handlers into subclasses. - # - # @param subclass [Class] subclass being created - # @return [void] - def inherited(subclass) - super - handled_commands.each do |cmd_class| - subclass.handled_commands << cmd_class - end - end - end - - attr_reader :partition_values - - # @param partition_values [Hash{Symbol => String}] partition key-value pairs - def initialize(partition_values = {}) - @partition_values = partition_values - @uncommitted_events = [] - end - - # Decide a command against the decider's current in-memory state. - # - # @param command [CCC::Message] command to handle - # @return [Array] newly produced events - def decide(command) - @uncommitted_events = [] - method_name = Sourced.message_method_name('ccc_decide', command.class.to_s) - send(method_name, state, command) if respond_to?(method_name) - @uncommitted_events.dup - end - - # Produce a new event from within a command handler and apply it - # to the decider's in-memory state immediately. - # - # Accepts either a CCC message class or a symbol resolved via .[]. - # - # @param event_class [Class, Symbol] event class or symbolic event name - # @param payload [Hash] payload attributes for the event - # @return [CCC::Message] the newly built event - # - # @example Produce by class - # command RegisterDevice do |_state, cmd| - # event DeviceRegistered, device_id: cmd.payload.device_id - # end - # - # @example Produce by symbol - # command RegisterDevice do |_state, cmd| - # event :device_registered, device_id: cmd.payload.device_id - # end - def event(event_class, payload = {}) - event_class = self.class[event_class] if event_class.is_a?(Symbol) - evt = event_class.new(payload: payload) - @uncommitted_events << evt - evolve([evt]) - evt - end - end - end -end diff --git a/lib/sourced/ccc/dispatcher.rb b/lib/sourced/ccc/dispatcher.rb deleted file mode 100644 index a4138c85..00000000 --- a/lib/sourced/ccc/dispatcher.rb +++ /dev/null @@ -1,223 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/work_queue' -require 'sourced/catchup_poller' -require 'sourced/ccc/worker' -require 'sourced/ccc/scheduled_message_poller' -require 'sourced/ccc/stale_claim_reaper' - -module Sourced - module CCC - # Orchestrator that wires together the signal-driven dispatch pipeline for CCC: - # {WorkQueue}, {NotificationQueuer}, {CatchUpPoller}, store notifier, and CCC {Worker}s. - # - # Mirrors {Sourced::Dispatcher} but uses CCC-specific interfaces: - # - +router.reactors+ instead of +router.async_reactors+ - # - +reactor.group_id+ instead of +reactor.consumer_info.group_id+ - # - +router.store.notifier+ instead of +router.backend.notifier+ - # - # Does not own the process lifecycle — the caller provides the task/fiber - # context via {#spawn_into}, and triggers shutdown via {#stop}. - # - # @example Usage with a task runner - # dispatcher = CCC::Dispatcher.new(router: ccc_router, worker_count: 4) - # executor.start do |task| - # dispatcher.spawn_into(task) - # end - # dispatcher.stop - # - # @example With custom queue for testing - # queue = WorkQueue.new(max_per_reactor: 2, queue: Queue.new) - # dispatcher = CCC::Dispatcher.new(router: ccc_router, work_queue: queue) - class Dispatcher - # Subscriber for the store notifier. Routes events to the {WorkQueue} - # by resolving message types or group IDs to reactor classes. - # - # Handles two events: - # - +'messages_appended'+ — comma-separated type strings; - # maps types to interested reactors and pushes them - # - +'reactor_resumed'+ — a consumer group ID; - # looks up the reactor and pushes it directly - class NotificationQueuer - MESSAGES_APPENDED = 'messages_appended' - REACTOR_RESUMED = 'reactor_resumed' - - # @param work_queue [WorkQueue] queue to push signaled reactors onto - # @param reactors [Array] reactor classes whose +handled_messages+ - # define the type-to-reactor mapping - def initialize(work_queue:, reactors:) - @work_queue = work_queue - @type_to_reactors = build_type_lookup(reactors) - @group_id_to_reactor = build_group_id_lookup(reactors) - end - - # Dispatch a notifier event to the appropriate handler. - # - # @param event_name [String] event name - # @param value [String] event payload - # @return [void] - def call(event_name, value) - case event_name - when MESSAGES_APPENDED - types = value.split(',').map(&:strip) - reactors = types.flat_map { |t| @type_to_reactors.fetch(t, []) }.uniq - reactors.each { |r| @work_queue.push(r) } - when REACTOR_RESUMED - reactor = @group_id_to_reactor[value] - @work_queue.push(reactor) if reactor - end - end - - private - - # @return [Hash{String => Array}] mapping from type string to reactor classes - def build_type_lookup(reactors) - lookup = Hash.new { |h, k| h[k] = [] } - reactors.each do |reactor| - reactor.handled_messages.map(&:type).uniq.each do |type| - lookup[type] << reactor - end - end - lookup - end - - # @return [Hash{String => Class}] mapping from group_id to reactor class - def build_group_id_lookup(reactors) - reactors.each_with_object({}) do |reactor, lookup| - lookup[reactor.group_id] = reactor - end - end - end - - # @return [Array] worker instances managed by this dispatcher - attr_reader :workers - - def self.spawn_into(task) - config = CCC.config - dispatcher = CCC::Dispatcher.new( - router: CCC.router, - worker_count: config.worker_count, - batch_size: config.batch_size, - max_drain_rounds: config.max_drain_rounds, - catchup_interval: config.catchup_interval, - housekeeping_interval: config.housekeeping_interval, - claim_ttl_seconds: config.claim_ttl_seconds, - logger: config.logger - ).spawn_into(task) - end - - # @param router [CCC::Router] the CCC router providing reactors and store - # @param worker_count [Integer] number of worker fibers to spawn (default 2) - # @param batch_size [Integer] max messages per claim (default 50) - # @param max_drain_rounds [Integer] max drain iterations before re-enqueue (default 10) - # @param catchup_interval [Numeric] seconds between catch-up polls (default 5) - # @param housekeeping_interval [Numeric] seconds between heartbeat/reap cycles (default 30) - # @param claim_ttl_seconds [Integer] stale claim age threshold in seconds (default 120) - # @param work_queue [WorkQueue, nil] optional pre-built queue (useful for testing) - # @param logger [Object] logger instance - def initialize( - router:, - worker_count: 2, - batch_size: 50, - max_drain_rounds: 10, - catchup_interval: 5, - housekeeping_interval: 30, - claim_ttl_seconds: 120, - work_queue: nil, - logger: CCC.config.logger - ) - @logger = logger - @router = router - @workers = [] - - return if worker_count.zero? - - reactors = router.reactors.select { |r| r.handled_messages.any? }.to_a - - @work_queue = work_queue || WorkQueue.new(max_per_reactor: worker_count) - - @workers = worker_count.times.map do |i| - Worker.new( - work_queue: @work_queue, - router:, - name: "worker-#{i}", - batch_size:, - max_drain_rounds:, - logger: - ) - end - - notification_queuer = NotificationQueuer.new(work_queue: @work_queue, reactors: reactors) - @store_notifier = router.store.notifier - @store_notifier.subscribe(notification_queuer) - - @catchup_poller = CatchUpPoller.new( - work_queue: @work_queue, - reactors:, - interval: catchup_interval, - logger: - ) - - @scheduled_message_poller = ScheduledMessagePoller.new( - store: router.store, - interval: catchup_interval, - logger: - ) - - @stale_claim_reaper = StaleClaimReaper.new( - store: router.store, - interval: housekeeping_interval, - ttl_seconds: claim_ttl_seconds, - worker_ids_provider: -> { @workers.map(&:name) }, - logger: - ) - end - - # Spawn all component fibers into the caller's task context. - # Spawns: store notifier (e.g. PG LISTEN), catch-up poller, and N workers. - # - # @param task [Object] an executor task or Async::Task to spawn fibers into - # @return [void] - def spawn_into(task) - return if @workers.empty? - - s = task.respond_to?(:spawn) ? :spawn : :async - - # Store notifier (start — no-op for InlineNotifier) - task.send(s) { @store_notifier.start } - - # CatchUp poller - task.send(s) { @catchup_poller.run } - - # Scheduled message poller - task.send(s) { @scheduled_message_poller.run } - - # Stale claim reaper - task.send(s) { @stale_claim_reaper.run } - - # Workers - @workers.each do |w| - task.send(s) { w.run } - end - - self - end - - # Stop all components and close the work queue. - # - # @return [void] - def stop - return if @workers.empty? - - @logger.info "CCC::Dispatcher: stopping #{@workers.size} workers" - @store_notifier.stop - @catchup_poller.stop - @scheduled_message_poller.stop - @stale_claim_reaper.stop - @workers.each(&:stop) - @work_queue.close(@workers.size) - @logger.info 'CCC::Dispatcher: all components stopped' - end - end - end -end diff --git a/lib/sourced/ccc/durable_workflow.rb b/lib/sourced/ccc/durable_workflow.rb deleted file mode 100644 index 54837a47..00000000 --- a/lib/sourced/ccc/durable_workflow.rb +++ /dev/null @@ -1,400 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' - -module Sourced - module CCC - # Stream-less port of {Sourced::DurableWorkflow}. - # - # A workflow instance is identified by a +workflow_id+ string, which doubles - # as the partition key. All lifecycle events (WorkflowStarted, StepStarted, - # StepFailed, StepComplete, ContextUpdated, WaitStarted, WaitEnded, - # WorkflowComplete, WorkflowFailed) carry +workflow_id+ as their first - # payload attribute so {CCC::Message#extracted_keys} indexes them for - # partition queries. - # - # The +durable+ / +wait+ / +context+ / +execute+ DSL mirrors - # {Sourced::DurableWorkflow} 1:1. The step-memoisation mechanism - # (@lookup + catch(:halt)) is unchanged; only the - # persistence and dispatch layer differ. - class DurableWorkflow - extend CCC::Consumer - include CCC::Evolve - - partition_by :workflow_id - - UnknownMessageError = Class.new(StandardError) - - # Stable hash-based key for a given (method, args) pair. - def self.step_key(step_name, args) - [step_name, args].hash.to_s - end - - def self.inherited(child) - super - child.partition_by(:workflow_id) - cname = child.name.to_s.gsub(/::/, '.') - .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') - .gsub(/([a-z\d])([A-Z])/, '\1_\2') - .tr('-', '_') - .downcase - - child.const_set(:WorkflowStarted, CCC::Event.define("#{cname}.workflow.started") do - attribute :workflow_id, String - attribute :args, Sourced::Types::Array.default([].freeze) - end) - child.const_set(:ContextUpdated, CCC::Event.define("#{cname}.context.updated") do - attribute :workflow_id, String - attribute :context, Sourced::Types::Any - end) - child.const_set(:WorkflowComplete, CCC::Event.define("#{cname}.workflow.complete") do - attribute :workflow_id, String - attribute :output, Sourced::Types::Any - end) - child.const_set(:WorkflowFailed, CCC::Event.define("#{cname}.workflow.failed") do - attribute :workflow_id, String - end) - child.const_set(:StepStarted, CCC::Event.define("#{cname}.step.started") do - attribute :workflow_id, String - attribute :key, String - attribute :step_name, Sourced::Types::Lax::Symbol - attribute :args, Sourced::Types::Array.default([].freeze) - end) - child.const_set(:StepFailed, CCC::Event.define("#{cname}.step.failed") do - attribute :workflow_id, String - attribute :key, String - attribute :step_name, Sourced::Types::Lax::Symbol - attribute :error_message, String - attribute :error_class, String - attribute :backtrace, Sourced::Types::Array[String] - end) - child.const_set(:StepComplete, CCC::Event.define("#{cname}.step.complete") do - attribute :workflow_id, String - attribute :key, String - attribute :step_name, Sourced::Types::Lax::Symbol - attribute :output, Sourced::Types::Any - end) - child.const_set(:WaitStarted, CCC::Event.define("#{cname}.wait.started") do - attribute :workflow_id, String - attribute :count, Integer - attribute :at, Sourced::Types::Forms::Time - end) - child.const_set(:WaitEnded, CCC::Event.define("#{cname}.wait.ended") do - attribute :workflow_id, String - end) - - # Register all event classes so: - # - Router claims them on our consumer group (`handled_messages`). - # - `context_for(workflow_id:)` builds OR conditions for the partition - # read (one per event type, via `Message.to_conditions`). - [ - child::WorkflowStarted, child::ContextUpdated, child::WorkflowComplete, - child::WorkflowFailed, child::StepStarted, child::StepFailed, - child::StepComplete, child::WaitStarted, child::WaitEnded - ].each do |klass| - child.handled_messages_for_evolve << klass unless child.handled_messages_for_evolve.include?(klass) - end - end - - # Message types this consumer claims. Same set as evolve types because - # every workflow event both advances state and re-triggers the workflow. - def self.handled_messages - handled_messages_for_evolve - end - - # Define the initial context hash. Block receives no arguments. - def self.context(&block) - define_method :initial_context, &block - end - - # Wrap a method so the runtime memoises its result across workflow - # re-entries. Mirrors {Sourced::DurableWorkflow.durable}. - def self.durable(method_name, retries: nil) - source_method = :"__durable_source_#{method_name}" - alias_method source_method, method_name - define_method method_name do |*args| - key = self.class.step_key(method_name, args) - cached = @lookup[key] - - case cached&.status - when :complete - cached.output - when :started - begin - output = send(source_method, *args) - @new_events << self.class::StepComplete.new( - payload: { workflow_id: id, key:, step_name: method_name, output: } - ) - throw :halt - rescue StandardError => e - @new_events << self.class::StepFailed.new( - payload: { - workflow_id: id, - key:, - step_name: method_name, - error_message: e.inspect, - error_class: e.class.to_s, - backtrace: e.backtrace - } - ) - if retries && cached.attempts == retries - @new_events << self.class::WorkflowFailed.new(payload: { workflow_id: id }) - end - throw :halt - end - when :failed - @new_events << self.class::StepStarted.new( - payload: { workflow_id: id, key:, step_name: method_name, args: } - ) - throw :halt - when nil - @new_events << self.class::StepStarted.new( - payload: { workflow_id: id, key:, step_name: method_name, args: } - ) - throw :halt - end - end - end - - Step = Struct.new(:status, :backtrace, :output, :attempts) do - def self.build - new(:started, [], nil, 0) - end - - def start - self.status = :started - self.attempts += 1 - end - - def fail_with(backtrace) - self.status = :failed - self.backtrace = backtrace - self - end - - def complete_with(output) - self.status = :complete - self.output = output - self - end - end - - # Kick off a new workflow instance. Appends a WorkflowStarted event and - # returns a {Waiter} that can poll for completion. - # - # @param args [Array] positional args passed to the workflow's #execute - # @param store [CCC::Store] defaults to CCC.store - # @return [Waiter] - def self.execute(*args, store: CCC.store) - workflow_id = "workflow-#{SecureRandom.uuid}" - evt = self::WorkflowStarted.new(payload: { workflow_id:, args: }) - store.append([evt]) - Waiter.new(self, workflow_id, store:) - end - - # Router entry point. Drops claim.messages from the read history (the - # router's +store.read(conditions)+ returns the full partition, including - # messages being claimed) and delegates to {.handle_batch}. - def self.handle_claim(claim, history:) - claim_positions = claim.messages.map { |m| m.position if m.respond_to?(:position) }.compact.to_set - prior = history.messages.reject { |m| m.respond_to?(:position) && claim_positions.include?(m.position) } - prior_history = ReadResult.new(messages: prior, guard: history.guard) - values = claim.partition_value.transform_keys(&:to_sym) - handle_batch(values, claim.messages, history: prior_history) - end - - # GWT-compatible entry point. +history.messages+ must be disjoint from - # +new_messages+ — the caller owns that distinction. - def self.handle_batch(partition_values, new_messages, history:, replaying: false) - workflow_id = partition_values[:workflow_id] - instance = new([workflow_id]) - instance.__replay(history.messages) - - each_with_partial_ack(new_messages) do |msg| - instance.__evolve(msg) - actions = instance.__handle(msg, guard: history.guard) - [actions, msg] - end - end - - # Direct handler used by unit tests and by {Waiter}. Mirrors - # {Sourced::DurableWorkflow.handle}: +history+ should already contain - # +message+ as its last element. - def self.handle(message, history:) - from(history).__handle(message) - end - - # Rebuild a workflow instance by replaying +history+. - def self.from(history) - new.__replay(history) - end - - # Load a workflow instance for +workflow_id+ from the store. - def self.load(workflow_id, store: CCC.store) - _inst, _rr = CCC.load(self, store:, workflow_id: workflow_id) - end - - # Polls the store for terminal workflow events. - class Waiter - attr_reader :workflow_id, :instance - - def initialize(klass, workflow_id, store: CCC.store) - @klass = klass - @workflow_id = workflow_id - @store = store - @instance = klass.new([workflow_id]) - end - - def wait(timeout: nil) - deadline = timeout ? Time.now + timeout : nil - until @instance.status == :complete || @instance.status == :failed - raise 'DurableWorkflow wait timed out' if deadline && Time.now > deadline - - sleep 0.05 - load - end - @instance - end - - def load - handled_types = @klass.handled_messages_for_evolve.map(&:type).uniq - result = @store.read_partition({ workflow_id: @workflow_id }, handled_types:) - @instance = @klass.new([@workflow_id]) - @instance.__replay(result.messages) - @instance - end - end - - attr_reader :id, :context, :args, :output, :status - - # +partition_values+ may be: - # - an Array like +['wf-id']+ (from {.handle_claim}) - # - a Hash like +{ workflow_id: 'wf-id' }+ (from {CCC.load}) - # - a String +'wf-id'+ - # - nil - def initialize(partition_values = nil) - @id = case partition_values - when Array then partition_values.first - when Hash then partition_values[:workflow_id] - when String then partition_values - else nil - end - @status = :new - @args = [] - @output = nil - @lookup = {} - @new_events = [] - @wait_count = 0 - @waiters = [] - @context = initial_context - end - - def initial_context = nil - - def __replay(history) - Array(history).each { |m| __evolve(m) } - self - end - - # Override CCC::Evolve#evolve so {CCC.load} (which calls +instance.evolve+) - # applies workflow events via our manual dispatcher. - def evolve(messages) - __replay(messages) - end - - def __evolve(event) - case event - when self.class::ContextUpdated - @context = deep_dup(event.payload.context) - when self.class::WorkflowStarted - @id ||= event.payload.workflow_id - @args = event.payload.args - @status = :started - when self.class::WorkflowFailed - @status = :failed - when self.class::StepStarted - (@lookup[event.payload.key] ||= Step.build).start - when self.class::StepFailed - @lookup[event.payload.key].fail_with(event.payload.backtrace) - when self.class::WaitStarted - @waiters[event.payload.count] = true - @waiting = true - when self.class::WaitEnded - @waiting = false - when self.class::StepComplete - @lookup[event.payload.key].complete_with(event.payload.output) - when self.class::WorkflowComplete - @status = :complete - @output = event.payload.output - else - raise UnknownMessageError, "No idea how to handle #{event.inspect}" - end - end - - # Decide the next action given +message+. State is assumed to already - # reflect +message+ (caller replayed it). - def __handle(message, guard: nil) - return Actions::OK if @status == :complete || @status == :failed - - if message.is_a?(self.class::WaitStarted) - evt = self.class::WaitEnded.new(payload: { workflow_id: id }) - return Actions::Schedule.new([evt], at: message.payload.at) - end - - @initial_context = deep_dup(@context) - - completed = false - output = nil - - catch(:halt) do - output = execute(*@args) - completed = true - end - - if @context != @initial_context - @new_events << self.class::ContextUpdated.new( - payload: { workflow_id: id, context: deep_dup(@context) } - ) - end - - if completed - @new_events << self.class::WorkflowComplete.new( - payload: { workflow_id: id, output: } - ) - end - - events = @new_events - @new_events = [] - return Actions::OK if events.empty? - - Actions::Append.new(events, guard: guard) - end - - private - - def wait(seconds) - @wait_count += 1 - - if @waiters[@wait_count] - seconds - else - @new_events << self.class::WaitStarted.new( - payload: { workflow_id: id, count: @wait_count, at: Time.now + seconds } - ) - throw :halt - end - end - - def deep_dup(value) - case value - when Hash - value.each.with_object({}) { |(k, v), h| h[k] = deep_dup(v) } - when Array - value.map { |v| deep_dup(v) } - else - value.dup rescue value - end - end - end - end -end diff --git a/lib/sourced/ccc/evolve.rb b/lib/sourced/ccc/evolve.rb deleted file mode 100644 index f7bda036..00000000 --- a/lib/sourced/ccc/evolve.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Evolve mixin for CCC reactors. - # Adapted from Sourced::Evolve for CCC::Message (no stream_id/seq). - # State block receives partition values hash instead of stream id. - module Evolve - PREFIX = 'ccc_evolution' - - def self.included(base) - super - base.extend ClassMethods - end - - def init_state(_partition_values) - nil - end - - def state - @state ||= init_state(partition_values) - end - - def partition_values - @partition_values ||= {} - end - - # Apply messages to state via registered handlers. - # Skips messages without a registered handler. - def evolve(messages) - Array(messages).each do |msg| - method_name = Sourced.message_method_name(PREFIX, msg.class.to_s) - send(method_name, state, msg) if respond_to?(method_name) - end - state - end - - module ClassMethods - def inherited(subclass) - super - handled_messages_for_evolve.each do |klass| - subclass.handled_messages_for_evolve << klass - end - end - - def handled_messages_for_evolve - @handled_messages_for_evolve ||= [] - end - - # Define initial state factory. Block receives partition values hash. - def state(&blk) - define_method(:init_state, &blk) - end - - # Register an evolve handler for a CCC::Message subclass. - def evolve(message_class, &block) - handled_messages_for_evolve << message_class - define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) - end - end - end - end -end diff --git a/lib/sourced/ccc/falcon.rb b/lib/sourced/ccc/falcon.rb deleted file mode 100644 index 3154e735..00000000 --- a/lib/sourced/ccc/falcon.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -require 'falcon' -require_relative 'falcon/environment' -require_relative 'falcon/service' diff --git a/lib/sourced/ccc/falcon/environment.rb b/lib/sourced/ccc/falcon/environment.rb deleted file mode 100644 index e22f7f40..00000000 --- a/lib/sourced/ccc/falcon/environment.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - module Falcon - # Environment mixin for configuring a combined Falcon web server + CCC workers service. - # - # Include this module in a Falcon service definition to get CCC worker defaults - # alongside the standard Falcon server environment. All settings are read from - # {CCC.config} — no per-service config methods needed. - # - # The Service automatically calls {CCC.setup!} at the start of +run+ to - # re-establish database connections after Falcon forks (SQLite connections - # are not fork-safe). This replays the block passed to {CCC.configure}. - # - # @example falcon.rb - # #!/usr/bin/env falcon-host - # require 'sourced/ccc/falcon' - # require_relative 'config/environment' - # - # service "my-app" do - # include Sourced::CCC::Falcon::Environment - # include Falcon::Environment::Rackup - # - # url "http://[::]:9292" - # end - module Environment - include ::Falcon::Environment::Server - - def service_class = CCC::Falcon::Service - end - end - end -end diff --git a/lib/sourced/ccc/falcon/service.rb b/lib/sourced/ccc/falcon/service.rb deleted file mode 100644 index e749fee4..00000000 --- a/lib/sourced/ccc/falcon/service.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/ccc/dispatcher' - -module Sourced - module CCC - module Falcon - # A Falcon service that runs both the web server and CCC background workers - # as sibling fibers within the same Async reactor. - # - # Uses a CCC::Dispatcher for signal-driven worker dispatch. The Dispatcher - # already embeds the StaleClaimReaper, so no separate HouseKeeper is needed - # (unlike {Sourced::Falcon::Service}). - # - # All configuration is read from {CCC.config}. - class Service < ::Falcon::Service::Server - def run(instance, evaluator) - CCC.setup! - - server = evaluator.make_server(@bound_endpoint) - - Async do |task| - server.run - CCC::Dispatcher.spawn_into(task) - task.children.each(&:wait) - end - - server - end - - def stop(...) - @dispatcher&.stop - super - end - end - end - end -end diff --git a/lib/sourced/ccc/installer.rb b/lib/sourced/ccc/installer.rb deleted file mode 100644 index b4aca702..00000000 --- a/lib/sourced/ccc/installer.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -require 'sequel' -require 'sequel/extensions/migration' -require 'erb' - -module Sourced - module CCC - class Installer - TABLE_SUFFIXES = %i[messages key_pairs message_key_pairs scheduled_messages consumer_groups offsets offset_key_pairs workers].freeze - - attr_reader :messages_table, :key_pairs_table, :message_key_pairs_table, - :scheduled_messages_table, :consumer_groups_table, :offsets_table, - :offset_key_pairs_table, :workers_table - - def initialize(db, logger:, prefix: 'sourced', migration_template: '001_create_ccc_tables.rb.erb') - raise ArgumentError, "invalid prefix: #{prefix}" unless prefix.match?(/\A[a-zA-Z_]\w*\z/) - - @db = db - @logger = logger - @prefix = prefix - @migration_template = migration_template - - TABLE_SUFFIXES.each do |suffix| - instance_variable_set(:"@#{suffix}_table", :"#{prefix}_#{suffix}") - end - end - - # Eval the rendered migration and apply :up directly. - def install - migration.apply(db, :up) - logger.info("Sourced tables installed (prefix: #{prefix})") - end - - # Check that all expected tables exist. - def installed? - all_table_names.all? { |t| db.table_exists?(t) } - end - - # Apply :down on the migration to drop tables. - def uninstall - raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test' - - migration.apply(db, :down) - end - - # Render the migration to a file for use with the host app's Sequel::Migrator. - # - # installer.copy_migration_to("db/migrations") - # installer.copy_migration_to { "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_ccc_tables.rb" } - # - def copy_migration_to(dir = nil, &block) - path = block ? block.call : File.join(dir, '001_create_ccc_tables.rb') - File.write(path, rendered_migration) - logger.info("Copied CCC migration to #{path}") - path - end - - private - - attr_reader :db, :logger, :prefix - - def migration - @migration ||= eval(rendered_migration) # rubocop:disable Security/Eval - end - - def rendered_migration - @rendered_migration ||= begin - template_path = File.join(__dir__, 'migrations', @migration_template) - ERB.new(File.read(template_path)).result(binding) - end - end - - # Returns actual symbols for use in installed? checks. - def all_table_names - TABLE_SUFFIXES.map { |suffix| instance_variable_get(:"@#{suffix}_table") } - end - end - end -end diff --git a/lib/sourced/ccc/message.rb b/lib/sourced/ccc/message.rb deleted file mode 100644 index e8bc2c16..00000000 --- a/lib/sourced/ccc/message.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/types' - -module Sourced - module CCC - # A query condition for reading messages from the store. - # Matches on (message_type AND all attrs key-value pairs). - # Multiple conditions are OR'd when passed to {Store#read}. - QueryCondition = Data.define(:message_type, :attrs) - - # Returned by {Store#read} and {Store#claim_next} for optimistic concurrency. - # Pass to {Store#append} via +guard:+ to detect conflicting writes. - ConsistencyGuard = Data.define(:conditions, :last_position) - - # Base message class for CCC's stream-less event sourcing. - # Unlike {Sourced::Message}, CCC messages have no stream_id or seq - # — they go into a flat, globally-ordered log. - # - # Supports +causation_id+ and +correlation_id+ for tracing causal chains - # across messages, similar to {Sourced::Message}. - # - # Define message types via {.define}: - # - # CourseCreated = CCC::Message.define('course.created') do - # attribute :course_name, String - # end - # - class Message < Types::Data - EMPTY_ARRAY = [].freeze - - attribute :id, Types::AutoUUID - attribute :type, Types::String.present - attribute? :causation_id, Types::UUID::V4 - attribute? :correlation_id, Types::UUID::V4 - attribute :created_at, Types::Forms::Time.default { Time.now } - attribute :metadata, Types::Hash.default(Plumb::BLANK_HASH) - attribute :payload, Types::Static[nil] - - # Lookup table mapping type strings to message subclasses. - # Separate from {Sourced::Message}'s registry. - class Registry - # @param message_class [Class] the root message class for this registry - def initialize(message_class) - @message_class = message_class - @lookup = {} - end - - # @return [Array] registered type strings - def keys = @lookup.keys - - # @return [Array] direct subclasses of the root message class - def subclasses = message_class.subclasses - - # Register a message class under a type string. - # - # @param key [String] message type string - # @param klass [Class] message subclass - def []=(key, klass) - @lookup[key] = klass - end - - # Look up a message class by type string. - # Searches this registry first, then recurses into subclass registries. - # - # @param key [String] message type string - # @return [Class, nil] - def [](key) - klass = lookup[key] - return klass if klass - - subclasses.each do |c| - klass = c.registry[key] - return klass if klass - end - nil - end - - # All registered message classes across this registry and subclass registries. - # - # @return [Enumerator] if no block given - # @yield [Class] each registered message class - def all(&block) - return enum_for(:all) unless block - - lookup.each_value(&block) - subclasses.each { |c| c.registry.all(&block) } - end - - private - - attr_reader :lookup, :message_class - end - - # @return [Registry] the message type registry for this class - def self.registry - @registry ||= Registry.new(self) - end - - # Base class for typed message payloads. - class Payload < Types::Data - # @param key [Symbol] attribute name - # @return [Object] attribute value - def [](key) = attributes[key] - - # @see Hash#fetch - def fetch(...) = to_h.fetch(...) - end - - # Define a new message type. Registers it in the {.registry} and - # optionally defines a typed payload. - # - # @param type_str [String] unique message type identifier (e.g. 'course.created') - # @yield optional block to define payload attributes via +attribute+ DSL - # @return [Class] the new message subclass - # - # @example - # UserJoined = CCC::Message.define('user.joined') do - # attribute :course_name, String - # attribute :user_id, String - # end - def self.define(type_str, &payload_block) - type_str.freeze unless type_str.frozen? - - registry[type_str] = Class.new(self) do - def self.node_name = :data - define_singleton_method(:type) { type_str } - - attribute :type, Types::Static[type_str] - if block_given? - payload_class = Class.new(Payload, &payload_block) - const_set(:Payload, payload_class) - attribute :payload, payload_class - names = payload_class._schema.to_h.keys.map(&:to_sym).freeze - define_singleton_method(:payload_attribute_names) { names } - end - end - end - - # Instantiate the correct message subclass from a hash with a +:type+ key. - # - # @param attrs [Hash] must include +:type+ matching a registered type string - # @return [Message] instance of the appropriate subclass - # @raise [Sourced::UnknownMessageError] if the type string is not registered - def self.from(attrs) - klass = registry[attrs[:type]] - raise Sourced::UnknownMessageError, "Unknown message type: #{attrs[:type]}" unless klass - - klass.new(attrs) - end - - def initialize(attrs = {}) - attrs = attrs.merge(payload: {}) unless attrs[:payload] - super(attrs) - end - - # Identity implementation of the +to_message+ contract — see - # {.===} and {CCC::PositionedMessage#to_message}. - def to_message = self - - # Make +case/when+ transparent to {CCC::PositionedMessage} (or any - # wrapper implementing +#to_message+). Ruby's default +Module#===+ - # is implemented in C and ignores +is_a?+ overrides, so wrapped - # messages would otherwise fall through the +else+ branch. - def self.===(other) - return true if super - return false unless other.respond_to?(:to_message) - - unwrapped = other.to_message - !unwrapped.equal?(other) && super(unwrapped) - end - - def with_metadata(meta = {}) - return self if meta.empty? - - with(metadata: metadata.merge(meta)) - end - - def with_payload(attrs = {}) - hash = to_h - (hash[:payload] ||= {}).merge!(attrs) - self.class.new(hash) - end - - def at(datetime) - if datetime < created_at - raise Sourced::PastMessageDateError, "Message #{type} can't be delayed to a date in the past" - end - - with(created_at: datetime) - end - - # Set causation and correlation IDs on another message, establishing - # a causal link from this message to +message+. Merges metadata. - # - # @param message [Message] the message to correlate - # @return [Message] a copy of +message+ with causation/correlation set - # - # @example - # caused = source_event.correlate(SomeCommand.new(payload: { ... })) - # caused.causation_id # => source_event.id - # caused.correlation_id # => source_event.correlation_id - def correlate(message) - attrs = { - causation_id: id, - correlation_id: correlation_id, - metadata: metadata.merge(message.metadata || Plumb::BLANK_HASH) - } - message.with(attrs) - end - - # Returns the declared payload attribute names for this message class. - # Subclasses created via {.define} override this with a cached frozen array. - # - # @return [Array] attribute names (e.g. +[:course_name, :user_id]+) - def self.payload_attribute_names = EMPTY_ARRAY - - # Build a {QueryCondition} for the intersection of this message's declared - # attributes and the given key-value pairs. Attributes not declared on this - # message class are silently ignored. Returns an array with a single condition - # containing all matching attrs, or an empty array if none match. - # - # @param attrs [Hash{Symbol => String}] partition attribute values - # @return [Array] - # - # @example - # CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') - # # => [QueryCondition('course.created', { course_name: 'Algebra' })] - # # user_id ignored — CourseCreated doesn't declare it - def self.to_conditions(**attrs) - supported = payload_attribute_names - matched = attrs.select { |key, _| supported.include?(key) } - .transform_values(&:to_s) - return [] if matched.empty? - - [QueryCondition.new(message_type: type, attrs: matched)] - end - - # Auto-extract key-value pairs from all top-level payload attributes. - # Used by {Store#append} to index messages for querying. - # - # @return [Array] pairs of [name, value], skipping nils - def extracted_keys - return [] unless payload - - payload.to_h.filter_map { |k, v| - [k.to_s, v.to_s] unless v.nil? - } - end - - private - - # Hook called by Plumb after schema parsing, when +:id+ has been resolved. - # Defaults +causation_id+ and +correlation_id+ to the message's own +id+. - def prepare_attributes(attrs) - attrs[:correlation_id] = attrs[:id] unless attrs[:correlation_id] - attrs[:causation_id] = attrs[:id] unless attrs[:causation_id] - attrs - end - end - - class Command < Message; end - class Event < Message; end - end -end diff --git a/lib/sourced/ccc/projector.rb b/lib/sourced/ccc/projector.rb deleted file mode 100644 index a0bab8d5..00000000 --- a/lib/sourced/ccc/projector.rb +++ /dev/null @@ -1,87 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Reactor base class for CCC read-model projectors. - class Projector - include CCC::Evolve - include CCC::React - include CCC::Sync - extend CCC::Consumer - - class << self - # Projectors claim events they evolve from + events they react to. - # - # @return [Array] evolved and reacted-to message classes - def handled_messages - (handled_messages_for_evolve + handled_messages_for_react).uniq - end - - private - - def build_action_pairs(instance, messages, replaying:) - sync_actions = instance.collect_actions( - state: instance.state, messages: messages, replaying: replaying - ) - - reaction_pairs = if replaying - [] - else - each_with_partial_ack(messages) do |msg| - next unless instance.reacts_to?(msg) - reaction_msgs = Array(instance.react(msg)) - actions = Actions.build_for(reaction_msgs) - actions.any? ? [actions, msg] : nil - end - end - - reaction_pairs + [[sync_actions, messages.last]] - end - end - - attr_reader :partition_values - - # @param partition_values [Hash{Symbol => String}] partition key-value pairs - def initialize(partition_values = {}) - @partition_values = partition_values - end - - # Projector variant that evolves only the claimed messages on top of stored state. - class StateStored < self - class << self - def handle_batch(partition_values, new_messages, history: nil, replaying: false) - instance = new(partition_values) - instance.evolve(new_messages) - build_action_pairs(instance, new_messages, replaying: replaying) - end - - # @param claim [ClaimResult] claimed partition batch - # @return [Array, PositionedMessage)>] action/source pairs - def handle_claim(claim) - values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } - handle_batch(values, claim.messages, replaying: claim.replaying) - end - end - end - - # Projector variant that rebuilds state from full history each time. - class EventSourced < self - class << self - def handle_batch(partition_values, new_messages, history:, replaying: false) - instance = new(partition_values) - instance.evolve(history.messages) - build_action_pairs(instance, new_messages, replaying: replaying) - end - - # @param claim [ClaimResult] claimed partition batch - # @param history [ReadResult] full partition history - # @return [Array, PositionedMessage)>] action/source pairs - def handle_claim(claim, history:) - values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } - handle_batch(values, claim.messages, history:, replaying: claim.replaying) - end - end - end - end - end -end diff --git a/lib/sourced/ccc/react.rb b/lib/sourced/ccc/react.rb deleted file mode 100644 index f55adf2f..00000000 --- a/lib/sourced/ccc/react.rb +++ /dev/null @@ -1,177 +0,0 @@ -# frozen_string_literal: true - -require 'set' - -module Sourced - module CCC - # React mixin for CCC reactors. - # Supports the same dispatch-based reaction DSL as Sourced::React, - # adapted to CCC's stream-less messages. - module React - PREFIX = 'ccc_reaction' - EMPTY_ARRAY = [].freeze - - def self.included(base) - super - base.extend ClassMethods - end - - # Run reaction handlers for one or more messages. - # Supports both explicit message returns and dispatch(...) calls. - def react(messages) - __handling_reactions(Array(messages)) do |message| - method_name = Sourced.message_method_name(PREFIX, message.class.to_s) - if respond_to?(method_name) - Array(send(method_name, state, message)).compact - else - EMPTY_ARRAY - end - end - end - - def reacts_to?(message) - self.class.handled_messages_for_react.include?(message.class) - end - - private - - def __handling_reactions(messages) - messages.flat_map do |message| - @__reaction_dispatchers = [] - @__message_for_reaction = message - explicit = Array(yield(message)).compact.reject { |value| value.is_a?(Dispatcher) } - dispatched = @__reaction_dispatchers.map(&:message) - explicit + dispatched - end - ensure - @__reaction_dispatchers = [] - @__message_for_reaction = nil - end - - class Dispatcher - attr_reader :message - - def initialize(message) - @message = message - end - - def inspect = %(<#{self.class} #{@message}>) - - def at(datetime) - @message = @message.at(datetime) - self - end - - def with_metadata(attrs = {}) - @message = @message.with_metadata(attrs) - self - end - end - - # Queue a follow-up message from within a reaction block. - # - # The returned {Dispatcher} can be chained to delay the message - # or add metadata before it is appended. - # - # @param message_class [Class, Symbol] CCC message class, or a symbol - # resolved via .[] - # @param payload [Hash] message payload attributes - # @return [Dispatcher] chainable wrapper around the dispatched message - # - # @example Dispatch by class - # reaction StudentEnrolled do |_state, event| - # dispatch(NotifyStudent, student_id: event.payload.student_id) - # end - # - # @example Dispatch by symbol with delay and metadata - # reaction StudentEnrolled do |_state, event| - # dispatch(:notify_student, student_id: event.payload.student_id) - # .with_metadata(channel: 'email') - # .at(Time.now + 300) - # end - def dispatch(message_class, payload = {}) - message_class = self.class[message_class] if message_class.is_a?(Symbol) - message = @__message_for_reaction - .correlate(message_class.new(payload: payload)) - .with_metadata(producer: self.class.group_id) - - dispatcher = Dispatcher.new(message) - @__reaction_dispatchers << dispatcher - dispatcher - end - - module ClassMethods - def inherited(subclass) - super - handled_messages_for_react.each do |klass| - subclass.handled_messages_for_react << klass - end - catch_all_react_events.each do |klass| - subclass.catch_all_react_events << klass - end - end - - def handled_messages_for_react - @handled_messages_for_react ||= [] - end - - def catch_all_react_events - @catch_all_react_events ||= Set.new - end - - # Register a reaction handler for one or more CCC message types. - # - # Accepts message classes, symbols resolved via .[], - # multiple arguments, or no arguments for a catch-all reaction across - # all evolve types without an explicit handler. - # - # @example React to a specific event class - # reaction StudentEnrolled do |state, event| - # dispatch(NotifyStudent, student_id: event.payload.student_id) - # end - # - # @example React to a symbol-resolved message class - # reaction :student_enrolled do |state, event| - # dispatch(:notify_student, student_id: event.payload.student_id) - # end - def reaction(*args, &block) - case args - in [] - handled_messages_for_evolve.each do |message_class| - method_name = Sourced.message_method_name(PREFIX, message_class.to_s) - next if instance_methods.include?(method_name.to_sym) - - catch_all_react_events << message_class - reaction(message_class, &block) - end - - in [Symbol => message_symbol] - message_class = self[message_symbol].tap do |klass| - raise( - ArgumentError, - "Cannot resolve message symbol #{message_symbol.inspect} for #{self}.reaction" - ) unless klass - end - - reaction(message_class, &block) - in [Class => message_class] if message_class < CCC::Message - __validate_message_for_reaction!(message_class) - handled_messages_for_react << message_class - define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) if block_given? - in Array => values if values.none?(&:nil?) - values.each { |value| reaction(value, &block) } - else - raise( - ArgumentError, - "Invalid arguments #{args.inspect} for #{self}.reaction" - ) - end - end - - def __validate_message_for_reaction!(_message_class) - # no-op - end - end - end - end -end diff --git a/lib/sourced/ccc/router.rb b/lib/sourced/ccc/router.rb deleted file mode 100644 index 84d1274d..00000000 --- a/lib/sourced/ccc/router.rb +++ /dev/null @@ -1,180 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/injector' - -module Sourced - module CCC - class Router - attr_reader :store, :reactors - - def initialize(store:) - @store = store - @reactors = [] - @needs_history = {} - end - - def register(reactor_class) - @reactors << reactor_class - store.register_consumer_group( - reactor_class.group_id, - partition_by: reactor_class.partition_keys.map(&:to_s) - ) - @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) - end - - def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) - handled_types = reactor_class.handled_messages.map(&:type).uniq - - claim = store.claim_next( - reactor_class.group_id, - partition_by: reactor_class.partition_keys.map(&:to_s), - handled_types: handled_types, - worker_id: worker_id, - batch_size: batch_size - ) - return false unless claim - - begin - kwargs = {} - if @needs_history[reactor_class] - attrs = claim.partition_value.transform_keys(&:to_sym) - conditions = reactor_class.context_for(attrs) - kwargs[:history] = store.read(conditions) - end - - action_pairs = reactor_class.handle_claim(claim, **kwargs) - - if action_pairs == Actions::RETRY - store.release(reactor_class.group_id, offset_id: claim.offset_id) - return true - end - - execute_actions(action_pairs, claim, reactor_class.group_id) - true - - rescue Sourced::PartialBatchError => e - execute_actions(e.action_pairs, claim, reactor_class.group_id) - store.updating_consumer_group(reactor_class.group_id) do |group| - reactor_class.on_exception(e, e.failed_message, group) - end - true - rescue Sourced::ConcurrentAppendError - store.release(reactor_class.group_id, offset_id: claim.offset_id) - true - rescue StandardError => e - store.release(reactor_class.group_id, offset_id: claim.offset_id) - store.updating_consumer_group(reactor_class.group_id) do |group| - reactor_class.on_exception(e, claim.messages.first, group) - end - true - end - end - - # Stop a consumer group and invoke the reactor's {Consumer#on_stop} callback. - # - # Marks the group as stopped in the store so workers will no longer claim - # work for it, then calls +on_stop+ on the reactor class. - # - # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string - # @param message [String, nil] optional reason for stopping (persisted in the group's error_context) - # @return [void] - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - # - # @example Stop with a reactor class - # router.stop_consumer_group(CourseDecider, 'maintenance window') - # - # @example Stop with a string group_id - # router.stop_consumer_group('CourseDecider') - def stop_consumer_group(reactor_or_id, message = nil) - reactor_class = resolve_reactor_class(reactor_or_id) - store.stop_consumer_group(reactor_class.group_id, message) - reactor_class.on_stop(message) - end - - # Reset a consumer group and invoke the reactor's {Consumer#on_reset} callback. - # - # Clears all partition offsets and resets the discovery position to 0, - # so the group will reprocess messages from the beginning. Does not - # change the group's status (a stopped group remains stopped after reset). - # Then calls +on_reset+ on the reactor class. - # - # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string - # @return [void] - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - # - # @example - # router.reset_consumer_group(CourseDecider) - def reset_consumer_group(reactor_or_id) - reactor_class = resolve_reactor_class(reactor_or_id) - store.reset_consumer_group(reactor_class.group_id) - reactor_class.on_reset - end - - # Start a consumer group and invoke the reactor's {Consumer#on_start} callback. - # - # Marks the group as active in the store so workers can claim work for it - # again, then calls +on_start+ on the reactor class. - # - # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string - # @return [void] - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - # - # @example - # router.start_consumer_group(CourseDecider) - def start_consumer_group(reactor_or_id) - reactor_class = resolve_reactor_class(reactor_or_id) - store.start_consumer_group(reactor_class.group_id) - reactor_class.on_start - end - - def drain(limit = Float::INFINITY) - count = 0 - loop do - count += 1 - found_any = @reactors.any? { |r| handle_next_for(r) } - break unless found_any && count < limit - end - end - - private - - # Resolve a reactor class or group_id string to a registered reactor class. - # - # @param reactor_or_id [Class, String] a reactor class (returned as-is) or a +group_id+ string - # @return [Class] the matching registered reactor class - # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor - def resolve_reactor_class(reactor_or_id) - return reactor_or_id if reactor_or_id.is_a?(Module) - - @reactors.find { |r| r.group_id == reactor_or_id } || - raise(ArgumentError, "No reactor registered with group_id '#{reactor_or_id}'") - end - - def execute_actions(action_pairs, claim, group_id) - after_sync_actions = [] - - store.db.transaction do - last_position = nil - Array(action_pairs).each do |(actions, source_message)| - Array(actions).each do |action| - if action.is_a?(Actions::AfterSync) - after_sync_actions << action - elsif action != Actions::OK - action.execute(store, source_message) - end - end - last_position = source_message.position if source_message.respond_to?(:position) - end - - if last_position - store.ack(group_id, offset_id: claim.offset_id, position: last_position) - else - store.release(group_id, offset_id: claim.offset_id) - end - end - - after_sync_actions.each(&:call) - end - end - end -end diff --git a/lib/sourced/ccc/scheduled_message_poller.rb b/lib/sourced/ccc/scheduled_message_poller.rb deleted file mode 100644 index 93a8e62a..00000000 --- a/lib/sourced/ccc/scheduled_message_poller.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Periodically promotes due scheduled messages into the main CCC log. - class ScheduledMessagePoller - # @param store [CCC::Store] the CCC store containing scheduled messages - # @param interval [Numeric] polling interval in seconds - # @param logger [Object] logger instance - def initialize(store:, interval: 5, logger: CCC.config.logger) - @store = store - @interval = interval - @logger = logger - @running = false - end - - # Run the polling loop until {#stop} is called. - # - # @return [void] - def run - @running = true - while @running - promoted = @store.update_schedule! - @logger.info "CCC::ScheduledMessagePoller: appended #{promoted} scheduled messages" if promoted > 0 - sleep @interval - end - @logger.info 'CCC::ScheduledMessagePoller: stopped' - end - - # Signal the poller to stop after the current sleep cycle. - # - # @return [void] - def stop - @running = false - end - end - end -end diff --git a/lib/sourced/ccc/stale_claim_reaper.rb b/lib/sourced/ccc/stale_claim_reaper.rb deleted file mode 100644 index 2f729625..00000000 --- a/lib/sourced/ccc/stale_claim_reaper.rb +++ /dev/null @@ -1,77 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Periodic loop that heartbeats active workers and releases claims - # held by workers that have stopped heartbeating (crashed or killed). - # - # Combines heartbeating and reaping in one loop since CCC doesn't have - # a separate HouseKeeper like the main Sourced module. - # - # +worker_ids_provider+ is a proc that returns current worker names — - # injected by the {Dispatcher} which owns the Worker instances. - # - # @example - # reaper = StaleClaimReaper.new( - # store: store, - # interval: 30, - # ttl_seconds: 120, - # worker_ids_provider: -> { workers.map(&:name) }, - # logger: logger - # ) - # # In a fiber/thread: - # reaper.run # blocks, heartbeating + reaping every 30s - # # From another fiber/thread: - # reaper.stop # breaks the loop - class StaleClaimReaper - # @param store [CCC::Store] the CCC store - # @param interval [Numeric] seconds between heartbeat/reap cycles (default 30) - # @param ttl_seconds [Integer] age threshold for stale claims (default 120) - # @param worker_ids_provider [Proc] returns Array of active worker IDs - # @param logger [Object] logger instance - def initialize(store:, interval: 30, ttl_seconds: 120, worker_ids_provider: -> { [] }, logger: CCC.config.logger) - @store = store - @interval = interval - @ttl_seconds = ttl_seconds - @worker_ids_provider = worker_ids_provider - @logger = logger - @running = false - end - - # Run the heartbeat/reap loop. Blocks until {#stop} is called. - # Reaps on startup (from previous runs where workers were killed). - # - # @return [void] - def run - @running = true - reap # reap on startup for claims left by previously killed workers - while @running - sleep @interval - heartbeat if @running - reap if @running - end - @logger.info 'CCC::StaleClaimReaper: stopped' - end - - # Signal the reaper to stop after the current sleep cycle. - # - # @return [void] - def stop - @running = false - end - - private - - def heartbeat - ids = Array(@worker_ids_provider.call).uniq - count = @store.worker_heartbeat(ids) - @logger.debug "CCC::StaleClaimReaper: heartbeated #{count} workers" if count > 0 - end - - def reap - released = @store.release_stale_claims(ttl_seconds: @ttl_seconds) - @logger.info "CCC::StaleClaimReaper: released #{released} stale claims" if released > 0 - end - end - end -end diff --git a/lib/sourced/ccc/store.rb b/lib/sourced/ccc/store.rb deleted file mode 100644 index eca30e4b..00000000 --- a/lib/sourced/ccc/store.rb +++ /dev/null @@ -1,1305 +0,0 @@ -# frozen_string_literal: true - -require 'json' -require 'set' -require 'sourced/inline_notifier' -require 'sourced/ccc/installer' - -module Sourced - module CCC - # Wraps a Message with a storage position. Delegates all message methods. - class PositionedMessage < SimpleDelegator - attr_reader :position - - # @param message [CCC::Message] the wrapped message instance - # @param position [Integer] global log position - def initialize(message, position) - super(message) - @position = position - end - - def class = __getobj__.class - def is_a?(klass) = __getobj__.is_a?(klass) || super - def kind_of?(klass) = is_a?(klass) - def instance_of?(klass) = __getobj__.instance_of?(klass) - - # Unwrap to the underlying {CCC::Message}. Part of the +to_message+ - # contract honoured by {CCC::Message.===} so that +case/when+ works - # transparently across wrapped and unwrapped messages. - def to_message = __getobj__ - end - - # Returned by {Store#claim_next} with everything needed to process and ack a partition. - ClaimResult = Data.define(:offset_id, :key_pair_ids, :partition_key, :partition_value, :messages, :replaying, :guard) - - # Returned by {Store#read} with messages and a consistency guard. - # Supports array destructuring via #to_ary for backwards compatibility: - # messages, guard = store.read(conditions) - ReadResult = Data.define(:messages, :guard) do - def to_ary = [messages, guard] - end - - ReadAllResult = Data.define(:messages, :last_position, :fetcher) do - include Enumerable - - def to_ary = [messages, last_position] - - # Iterates messages in the current page. - def each(&block) = messages.each(&block) - - # Returns an Enumerator that lazily paginates through all messages, - # fetching subsequent pages as needed. - def to_enum - Enumerator.new do |y| - result = self - loop do - break if result.messages.empty? - - result.messages.each { |m| y << m } - result = result.fetcher.call(result.messages.last.position) - end - end - end - end - - Stats = Data.define(:max_position, :groups) - - OffsetsResult = Data.define(:offsets, :total_count, :fetcher) do - include Enumerable - - def to_ary = [offsets, total_count] - - # Iterates offsets in the current page. - def each(&block) = offsets.each(&block) - - # Returns an Enumerator that lazily paginates through all offsets, - # fetching subsequent pages as needed. - def to_enum - Enumerator.new do |y| - result = self - loop do - break if result.offsets.empty? - - result.offsets.each { |o| y << o } - result = result.fetcher.call(result.offsets.last[:id] + 1) - end - end - end - end - - # SQLite-backed store for CCC's flat, globally-ordered message log. - # Provides message storage with automatic key-pair indexing, - # consumer group management, and partition-based offset tracking - # for parallel background processing. - class Store - ACTIVE = 'active' - STOPPED = 'stopped' - FAILED = 'failed' - DISCOVERY_BATCH_SIZE = 100 - - # @return [Sequel::SQLite::Database] - attr_reader :db - - # @return [Sourced::InlineNotifier] - attr_reader :notifier - - # @return [Logger] - attr_reader :logger - - # @return [CCC::Installer] - attr_reader :installer - - # @param db [Sequel::SQLite::Database] a Sequel SQLite connection - # @param notifier [#notify_new_messages, #notify_reactor_resumed, nil] optional notifier for dispatch signals - # @param logger [Logger, nil] optional logger (defaults to Sourced.config.logger) - # @param prefix [String] table name prefix (default 'sourced') - def initialize(db, notifier: nil, logger: nil, prefix: 'sourced') - @db = db - @notifier = notifier || Sourced::InlineNotifier.new - @logger = logger || Sourced.config.logger - Sequel.extension(:fiber_concurrency) - @db.run('PRAGMA foreign_keys = ON') - @db.run('PRAGMA journal_mode = WAL') - @db.run('PRAGMA busy_timeout = 5000') - - @installer = Installer.new(db, logger: @logger, prefix: prefix) - - # Source table name symbols from the installer - @messages_table = @installer.messages_table - @key_pairs_table = @installer.key_pairs_table - @message_key_pairs_table = @installer.message_key_pairs_table - @scheduled_messages_table = @installer.scheduled_messages_table - @consumer_groups_table = @installer.consumer_groups_table - @offsets_table = @installer.offsets_table - @offset_key_pairs_table = @installer.offset_key_pairs_table - @workers_table = @installer.workers_table - - # Cache of registered consumer groups for eager offset creation in append. - # Populated by register_consumer_group. - # { group_id => { cg_id: Integer, partition_by: Array | nil } } - @registered_groups = {} - end - - # Whether all required tables exist. - # @return [Boolean] - def installed? - installer.installed? - end - - # Create all required tables and indexes. Idempotent. - # @return [void] - def install! - installer.install - end - - # Drop all tables. Test-only guard. - # @return [void] - def uninstall - installer.uninstall - end - - # Render the migration to a file for use with the host app's Sequel::Migrator. - # @see Installer#copy_migration_to - def copy_migration_to(dir = nil, &block) - installer.copy_migration_to(dir, &block) - end - - # Append messages to the store. Extracts and indexes key-value pairs - # from each message's payload automatically. - # - # When a {ConsistencyGuard} is provided, checks for conflicting messages - # before inserting (optimistic concurrency). - # - # @param messages [CCC::Message, Array] one or more messages to append - # @param guard [ConsistencyGuard, nil] optional guard for conflict detection - # @return [Integer] the last assigned position - # @raise [Sourced::ConcurrentAppendError] if conflicting messages found after guard position - def append(messages, guard: nil) - messages = Array(messages) - return latest_position if messages.empty? - - last_position = nil - - db.transaction do - if guard - conflicts = check_conflicts(guard.conditions, guard.last_position) - raise Sourced::ConcurrentAppendError, "Conflicting messages found after position #{guard.last_position}" if conflicts.any? - end - - messages.each do |msg| - payload_json = msg.payload ? JSON.dump(msg.payload.to_h) : '{}' - metadata_json = msg.metadata.empty? ? nil : JSON.dump(msg.metadata) - - # insert returns last_insert_rowid on SQLite — no need for a separate SELECT - last_position = db[@messages_table].insert( - message_id: msg.id, - message_type: msg.type, - causation_id: msg.causation_id, - correlation_id: msg.correlation_id, - payload: payload_json, - metadata: metadata_json, - created_at: msg.created_at.iso8601 - ) - - # Upsert key pairs and link to message in 2 statements (was 3): - # 1. INSERT OR IGNORE the key_pair - # 2. INSERT message_key_pair with key_pair_id resolved via subquery - msg.extracted_keys.each do |name, value| - db.run("INSERT OR IGNORE INTO #{@key_pairs_table} (name, value) VALUES (#{db.literal(name)}, #{db.literal(value)})") - db.run(<<~SQL) - INSERT INTO #{@message_key_pairs_table} (message_position, key_pair_id) - SELECT #{db.literal(last_position)}, id - FROM #{@key_pairs_table} - WHERE name = #{db.literal(name)} AND value = #{db.literal(value)} - SQL - end - end - - ensure_offsets_for_registered_groups(messages) - end - - notifier.notify_new_messages(messages.map(&:type).uniq) - - last_position - end - - # Persist messages for future promotion into the main CCC log. - # - # @param messages [CCC::Message, Array] one or more delayed messages - # @param at [Time] when the messages should become available - # @return [Boolean] false when no messages were provided, true otherwise - def schedule_messages(messages, at:) - messages = Array(messages) - return false if messages.empty? - - now = Time.now - rows = messages.map do |message| - data = message.to_h - data[:metadata] = message.metadata.merge(scheduled_at: now) - { - created_at: now.iso8601, - available_at: at.iso8601, - message: JSON.dump(data) - } - end - - db.transaction do - db[@scheduled_messages_table].multi_insert(rows) - end - - true - end - - # Promote due scheduled messages into the main CCC log. - # - # Appended messages are re-inserted through {#append} so they are indexed, - # assigned fresh positions, and announced through the store notifier. - # - # @return [Integer] number of scheduled messages promoted - def update_schedule! - now = Time.now - - db.transaction do - rows = db[@scheduled_messages_table] - .where { available_at <= now.iso8601 } - .order(:id) - .limit(100) - .all - - return 0 if rows.empty? - - messages = rows.map do |row| - data = JSON.parse(row[:message], symbolize_names: true) - data[:created_at] = now - Message.from(data) - end - - append(messages) - - row_ids = rows.map { |row| row[:id] } - db[@scheduled_messages_table].where(id: row_ids).delete - - rows.size - end - end - - # Paginate the global event log in position order. - # - # @example First page - # messages = store.read_all(limit: 20) - # - # @example Next page (using the last position from the previous page) - # messages = store.read_all(from_position: 20, limit: 20) - # - # @example Filtered by conditions (OR semantics, same as #read) - # messages = store.read_all(conditions: [cond1, cond2], limit: 20) - # - # @param from_position [Integer] return messages from this position, inclusive (default 0) - # @param conditions [Array, nil] optional conditions to filter by - # @param limit [Integer] max number of messages to return (default 50) - # @return [ReadAllResult] messages and last global position - def read_all(from_position: nil, conditions: [], limit: 50, order: :asc) - desc = order == :desc - conditions = Array(conditions).compact - after_position = from_position ? from_position - 1 : nil - - if conditions.any? - messages = query_messages(conditions, after_position:, limit:, order:) - else - ds = db[@messages_table] - ds = desc ? ds.where { position <= from_position } : ds.where { position >= from_position } if from_position - messages = ds.order(desc ? Sequel.desc(:position) : :position).limit(limit).map { |row| deserialize(row) } - end - - fetcher = ->(pos) { read_all(from_position: desc ? pos - 1 : pos + 1, conditions:, limit:, order:) } - ReadAllResult.new(messages:, last_position: latest_position, fetcher:) - end - - # Query messages by conditions. Each condition matches on - # (message_type AND key_name AND key_value). Multiple conditions are OR'd. - # - # @param conditions [QueryCondition, Array] query conditions - # @param after_position [Integer, nil] only return messages after this position (exclusive) - # @param limit [Integer, nil] max number of messages to return - # @return [ReadResult] messages and a guard - def read(conditions, after_position: nil, limit: nil) - conditions = Array(conditions) - if conditions.empty? - guard = ConsistencyGuard.new(conditions:, last_position: after_position || latest_position) - return ReadResult.new(messages: [], guard:) - end - - messages = query_messages(conditions, after_position:, limit:) - last_position = messages.any? ? messages.last.position : (after_position || latest_position) - guard = ConsistencyGuard.new(conditions:, last_position:) - ReadResult.new(messages:, guard:) - end - - # Read messages for a specific partition using AND semantics. - # A message is included only when every partition attribute it declares - # matches the given value. Messages that don't declare a partition - # attribute pass through (same logic as {#claim_next}). - # - # @example Single partition attribute - # result = store.read_partition( - # { device_id: 'dev-1' }, - # handled_types: ['device.registered', 'device.bound'] - # ) - # result.messages # => [#, ...] - # result.guard # => # - # - # @example Composite partition (AND semantics — messages must match all attributes they declare) - # result = store.read_partition( - # { course_name: 'Algebra', user_id: 'joe' }, - # handled_types: ['course.created', 'user.joined_course'] - # ) - # # Returns CourseCreated(course_name: 'Algebra') — matches on its only attribute - # # Returns UserJoinedCourse(course_name: 'Algebra', user_id: 'joe') — matches both - # # Excludes UserJoinedCourse(course_name: 'Algebra', user_id: 'jane') — user_id mismatch - # - # @example Resuming from a position (e.g. after processing a batch) - # result = store.read_partition( - # { device_id: 'dev-1' }, - # handled_types: ['device.registered'], - # after_position: 42 - # ) - # # Only returns messages with position > 42 - # - # @example Using the guard for optimistic concurrency on append - # result = store.read_partition( - # { device_id: 'dev-1' }, - # handled_types: ['device.registered', 'device.bound'] - # ) - # # ... build new events from result.messages ... - # store.append(new_events, guard: result.guard) - # # Raises Sourced::ConcurrentAppendError if conflicting writes occurred - # - # @param partition_attrs [Hash{Symbol|String => String}] partition attribute values - # @param handled_types [Array] message type strings to include - # @param after_position [Integer] fetch messages after this position (exclusive, default 0) - # @return [ReadResult] messages and a guard for optimistic concurrency - def read_partition(partition_attrs, handled_types:, after_position: 0) - # Resolve key_pair_ids for each partition attribute - key_pair_ids = partition_attrs.filter_map do |name, value| - db[@key_pairs_table].where(name: name.to_s, value: value.to_s).get(:id) - end - - # If any key pair doesn't exist in the store, no messages can match - if key_pair_ids.size < partition_attrs.size - guard = ConsistencyGuard.new(conditions: [], last_position: after_position) - return ReadResult.new(messages: [], guard:) - end - - messages = fetch_partition_messages(key_pair_ids, after_position, handled_types) - - # Build guard conditions from handled_types, scoped to partition attrs. - # These use OR semantics so the guard detects any concurrent write - # in the broader partition context (e.g. another student enrolling). - partition_sym = partition_attrs.transform_keys(&:to_sym) - guard_conditions = handled_types.filter_map do |type| - klass = Message.registry[type] - klass&.to_conditions(**partition_sym) - end.flatten - - # The guard's last_position must cover the full OR-context, not just - # the AND-filtered messages. Otherwise a message that passes the OR - # conditions but was excluded by AND filtering would look like a conflict. - last_pos = max_position_for(guard_conditions, after_position: after_position) - - guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) - ReadResult.new(messages: messages, guard: guard) - end - - # Conflict detection: returns messages matching conditions that appeared - # after the given position. Empty array means no conflicts. - # - # @param conditions [Array] conditions to check - # @param position [Integer] check for messages after this position - # @return [ReadResult] - def messages_since(conditions, position) - read(conditions, after_position: position) - end - - # Register a consumer group. Idempotent. - # When +partition_by+ is provided, offsets are created eagerly during {#append} - # instead of lazily via discovery in {#claim_next}. - # - # @param group_id [String] unique identifier for the consumer group - # @param partition_by [Array, nil] attribute names defining partitions - # @return [void] - def register_consumer_group(group_id, partition_by: nil) - partition_by_sorted = partition_by ? Array(partition_by).map(&:to_s).sort : nil - partition_by_json = partition_by_sorted ? JSON.dump(partition_by_sorted) : nil - now = Time.now.iso8601 - db.run(<<~SQL) - INSERT INTO #{@consumer_groups_table} (group_id, status, highest_position, partition_by, created_at, updated_at) - VALUES (#{db.literal(group_id)}, '#{ACTIVE}', 0, #{db.literal(partition_by_json)}, #{db.literal(now)}, #{db.literal(now)}) - ON CONFLICT(group_id) DO UPDATE SET partition_by = #{db.literal(partition_by_json)}, updated_at = #{db.literal(now)} - SQL - - # Cache for hot-path use in append - cg = db[@consumer_groups_table].where(group_id: group_id).first - @registered_groups[group_id] = { cg_id: cg[:id], partition_by: partition_by_sorted } - end - - # Whether the consumer group exists and is active. - # - # @param group_id [String, #group_id] identifier or object responding to +#group_id+ - # @return [Boolean] - def consumer_group_active?(group_id) - group_id = resolve_group_id(group_id) - row = db[@consumer_groups_table].where(group_id: group_id).select(:status).first - return false unless row - - row[:status] == ACTIVE - end - - # Stop a consumer group intentionally. Stopped groups are skipped by {#claim_next}. - # - # @param group_id [String, #group_id] identifier or object responding to +#group_id+ - # @param message [String, nil] optional operator-supplied reason - # @return [void] - def stop_consumer_group(group_id, message = nil) - group_id = resolve_group_id(group_id) - updating_consumer_group(group_id) do |group| - group.stop(message:) - end - end - - # Re-activate a stopped or failed consumer group, clearing retry state. - # - # @param group_id [String, #group_id] identifier or object responding to +#group_id+ - # @return [void] - def start_consumer_group(group_id) - group_id = resolve_group_id(group_id) - db[@consumer_groups_table] - .where(group_id: group_id) - .update(status: ACTIVE, retry_at: nil, error_context: nil, updated_at: Time.now.iso8601) - notifier.notify_reactor_resumed(group_id) - end - - # Load a consumer group row, yield a {GroupUpdater} for mutation, - # then persist the accumulated updates atomically. - # Mirrors SequelBackend#updating_consumer_group. - # - # @param group_id [String] - # @yieldparam group [CCC::GroupUpdater] - # @return [void] - def updating_consumer_group(group_id) - dataset = db[@consumer_groups_table].where(group_id: group_id) - row = dataset.first - raise ArgumentError, "Consumer group #{group_id} not found" unless row - - ctx = row[:error_context] ? JSON.parse(row[:error_context], symbolize_names: true) : {} - row[:error_context] = ctx - - group = CCC::GroupUpdater.new(group_id, row, logger) - yield group - - updates = group.updates.dup - updates[:error_context] = JSON.dump(updates[:error_context]) - dataset.update(updates) - end - - # Delete all offsets for a consumer group, resetting it to process from the beginning. - # - # @param group_id [String, #group_id] identifier or object responding to +#group_id+ - # @return [void] - def reset_consumer_group(group_id) - group_id = resolve_group_id(group_id) - cg = db[@consumer_groups_table].where(group_id: group_id).first - return unless cg - - db[@offsets_table].where(consumer_group_id: cg[:id]).delete - db[@consumer_groups_table].where(id: cg[:id]).update( - discovery_position: 0, - last_nil_types_max_pos: 0, - updated_at: Time.now.iso8601 - ) - end - - # Claim the next available partition for processing. - # - # Bootstraps partition offsets (discovering new partitions from messages with - # ALL +partition_by+ attributes), finds the unclaimed partition with the earliest - # pending message, claims it, and fetches messages using conditional AND semantics. - # - # Returns a {ConsistencyGuard} alongside the messages, built from each handled - # message class's declared payload attributes via {Message.to_conditions}. - # - # The +replaying+ flag indicates whether the returned messages have been - # processed by this consumer group before. A message is replaying when its - # position is at or before the consumer group's +highest_position+ — the - # furthest position ever successfully acked. After a reset, re-claimed - # messages are correctly flagged as replaying. - # - # @param group_id [String] consumer group identifier - # @param partition_by [String, Array] attribute name(s) defining partitions - # @param handled_types [Array] message type strings this consumer handles - # @param worker_id [String] identifier for the claiming worker - # @param batch_size [Integer, nil] max messages to fetch per claim (nil = unlimited) - # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, replaying:, guard: }+ or nil - def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: nil) - partition_by = Array(partition_by).sort - now = Time.now.iso8601 - cg = db[@consumer_groups_table] - .where(group_id: group_id, status: ACTIVE) - .where { Sequel.|({retry_at: nil}, Sequel.lit('retry_at <= ?', now)) } - .first - return nil unless cg - - # Short-circuit: no new messages since the last nil claim. - types_max_pos = db[@messages_table] - .where(message_type: handled_types) - .max(:position) || 0 - - return nil if types_max_pos <= cg[:last_nil_types_max_pos] - - claimed = nil - group_info = @registered_groups[group_id] - - if group_info&.fetch(:partition_by, nil) - # Eager path: offsets were created by append. Try fast claim first, - # fall back to discovery only for catch-up (new group against existing log). - claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) - unless claimed - discover_new_partitions(cg[:id], partition_by, handled_types) - claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) - end - else - # Legacy path: lazy discovery - has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? - if has_offsets && types_max_pos <= cg[:discovery_position] - claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) - end - unless claimed - discover_new_partitions(cg[:id], partition_by, handled_types) - claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) - end - end - - unless claimed - # Remember types_max_pos so next poll short-circuits instantly - db[@consumer_groups_table].where(id: cg[:id]) - .update(last_nil_types_max_pos: types_max_pos) - return nil - end - - key_pair_ids = db[@offset_key_pairs_table] - .where(offset_id: claimed[:offset_id]) - .select_map(:key_pair_id) - - messages = fetch_partition_messages(key_pair_ids, claimed[:last_position], handled_types, limit: batch_size) - - # If no messages pass the conditional AND filter, release and return nil - if messages.empty? - release(group_id, offset_id: claimed[:offset_id]) - return nil - end - - # Build partition_value hash from key_pairs - partition_value = {} - db[@key_pairs_table].where(id: key_pair_ids).each do |kp| - partition_value[kp[:name]] = kp[:value] - end - - # Build guard conditions from handled_types. - # Each class's to_conditions only generates conditions for attributes it actually has. - # We use handled_types (not just fetched messages) so the guard also covers - # message types that haven't appeared yet but would be conflicts. - partition_attrs = partition_value.transform_keys(&:to_sym) - guard_conditions = handled_types.filter_map do |type| - klass = Message.registry[type] - klass&.to_conditions(**partition_attrs) - end.flatten - - last_pos = messages.last.position - guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) - - # replaying: true when all messages are at or below the highest position - # ever acked by this consumer group (i.e. they've been processed before). - replaying = messages.last.position <= cg[:highest_position] - - ClaimResult.new( - offset_id: claimed[:offset_id], - key_pair_ids: key_pair_ids, - partition_key: claimed[:partition_key], - partition_value: partition_value, - messages: messages, - replaying: replaying, - guard: guard - ) - end - - # Acknowledge processing: advance the offset to +position+ and release the claim. - # Also advances the consumer group's +highest_position+ watermark (never decreases), - # which drives the {#claim_next} +replaying+ flag. - # - # @param group_id [String] consumer group identifier - # @param offset_id [Integer] offset ID from the claim result - # @param position [Integer] position of the last processed message - # @return [void] - def ack(group_id, offset_id:, position:) - cg = db[@consumer_groups_table].where(group_id: group_id).first - return unless cg - - db[@offsets_table].where(id: offset_id, consumer_group_id: cg[:id]).update( - last_position: position, - claimed: 0, - claimed_at: nil, - claimed_by: nil - ) - - # Advance the high watermark (never decrease) - if position > cg[:highest_position] - db[@consumer_groups_table].where(id: cg[:id]).update( - highest_position: position, - updated_at: Time.now.iso8601 - ) - end - end - - # Release a claim without advancing the offset. Use for error recovery - # so the partition can be re-claimed and retried. - # - # @param group_id [String] consumer group identifier - # @param offset_id [Integer] offset ID from the claim result - # @return [void] - def release(group_id, offset_id:) - cg = db[@consumer_groups_table].where(group_id: group_id).first - return unless cg - - db[@offsets_table].where(id: offset_id, consumer_group_id: cg[:id]).update( - claimed: 0, - claimed_at: nil, - claimed_by: nil - ) - end - - # Upsert heartbeat timestamps for active workers. - # - # @param worker_ids [Array] worker identifiers - # @param at [Time] timestamp to record (default Time.now) - # @return [Integer] number of workers heartbeated - def worker_heartbeat(worker_ids, at: Time.now) - ids = Array(worker_ids).uniq - return 0 if ids.empty? - - now = at.iso8601 - ids.each do |id| - db.run(<<~SQL) - INSERT INTO #{@workers_table} (id, last_seen) VALUES (#{db.literal(id)}, #{db.literal(now)}) - ON CONFLICT(id) DO UPDATE SET last_seen = #{db.literal(now)} - SQL - end - ids.size - end - - # Release claims held by workers that haven't heartbeated within ttl_seconds. - # - # @param ttl_seconds [Integer] age threshold - # @return [Integer] number of claims released - def release_stale_claims(ttl_seconds: 120) - cutoff = (Time.now - ttl_seconds).iso8601 - - stale_worker_ids = db[@workers_table] - .where(Sequel.lit('last_seen <= ?', cutoff)) - .select_map(:id) - - return 0 if stale_worker_ids.empty? - - db[@offsets_table] - .where(claimed: 1) - .where(claimed_by: stale_worker_ids) - .update(claimed: 0, claimed_at: nil, claimed_by: nil) - end - - # Advance a consumer group's offset for a specific partition to at least +position+. - # Bootstraps the offset row if it doesn't exist yet. - # Unlike {#ack}, this does not require a prior claim. - # - # @param group_id [String] consumer group identifier - # @param partition [Hash{String => String}] partition attribute names and values - # @param position [Integer] advance offset to at least this position - # @return [void] - def advance_offset(group_id, partition:, position:) - cg = db[@consumer_groups_table].where(group_id: group_id).first - return unless cg - - offset_id = ensure_offset_for_partition(cg[:id], partition) - return unless offset_id - - offset = db[@offsets_table].where(id: offset_id).first - return if offset[:last_position] >= position - - db[@offsets_table].where(id: offset_id).update(last_position: position) - - if position > cg[:highest_position] - db[@consumer_groups_table].where(id: cg[:id]).update( - highest_position: position, - updated_at: Time.now.iso8601 - ) - end - end - - # System-wide diagnostics for monitoring and debugging. - # - # @example - # stats = store.stats - # stats.max_position # => 42 - # stats.groups - # # => [ - # # { - # # group_id: "my_decider", - # # status: "active", - # # retry_at: nil, - # # error_context: {}, - # # oldest_processed: 10, - # # newest_processed: 42, - # # partition_count: 3 - # # }, - # # { - # # group_id: "failing_decider", - # # status: "failed", - # # retry_at: nil, - # # error_context: { exception_class: "RuntimeError", exception_message: "boom" }, - # # oldest_processed: 5, - # # newest_processed: 30, - # # partition_count: 2 - # # } - # # ] - # - # @return [CCC::Stats] max_position and per-group processing state - def stats - groups = db.fetch(<<~SQL).all - SELECT - cg.group_id, - cg.status, - cg.retry_at, - cg.error_context, - COALESCE(MIN(CASE WHEN o.last_position > 0 THEN o.last_position END), 0) AS oldest_processed, - COALESCE(MAX(o.last_position), 0) AS newest_processed, - COUNT(o.id) AS partition_count - FROM #{@consumer_groups_table} cg - LEFT JOIN #{@offsets_table} o ON o.consumer_group_id = cg.id - GROUP BY cg.id, cg.group_id, cg.status, cg.retry_at, cg.error_context - ORDER BY cg.group_id - SQL - - groups.each do |g| - g[:retry_at] = Time.parse(g[:retry_at]) if g[:retry_at] - g[:error_context] = g[:error_context] ? JSON.parse(g[:error_context], symbolize_names: true) : {} - end - - Stats.new(max_position: latest_position, groups: groups) - end - - # List offsets with optional group filtering and cursor-based pagination. - # - # @param group_id [String, nil] filter by consumer group (nil = all groups) - # @param limit [Integer] max offsets per page (default 50) - # @param from_id [Integer, nil] cursor — return offsets with id >= from_id (inclusive) - # @return [CCC::OffsetsResult] paginated offsets with total_count and fetcher for auto-pagination - def read_offsets(group_id: nil, limit: 50, from_id: nil) - dataset = db[@offsets_table].join(@consumer_groups_table, id: :consumer_group_id) - .select( - Sequel[@offsets_table][:id], - Sequel[@consumer_groups_table][:group_id].as(:group_name), - Sequel[@consumer_groups_table][:status].as(:group_status), - Sequel[@offsets_table][:partition_key], - Sequel[@offsets_table][:last_position], - Sequel[@offsets_table][:claimed], - Sequel[@offsets_table][:claimed_at], - Sequel[@offsets_table][:claimed_by] - ) - .order(Sequel[@offsets_table][:id]) - - count_dataset = db[@offsets_table].join(@consumer_groups_table, id: :consumer_group_id) - - if group_id - dataset = dataset.where(Sequel[@consumer_groups_table][:group_id] => group_id) - count_dataset = count_dataset.where(Sequel[@consumer_groups_table][:group_id] => group_id) - end - - if from_id - dataset = dataset.where(Sequel[@offsets_table][:id] >= from_id) - end - - total_count = count_dataset.count - offsets = dataset.limit(limit).all - - offsets.each do |o| - o[:claimed] = o[:claimed] == 1 - end - - fetcher = ->(next_from_id) { read_offsets(group_id: group_id, limit: limit, from_id: next_from_id) } - - OffsetsResult.new(offsets: offsets, total_count: total_count, fetcher: fetcher) - end - - # Fetch all messages sharing the same correlation_id as the given message. - # Useful for tracing causal chains (command -> events -> reactions). - # - # @param message_id [String] UUID of any message in the correlation chain - # @return [Array] correlated messages ordered by position, or [] if not found - def read_correlation_batch(message_id) - correlation_id = db[@messages_table] - .where(message_id: message_id) - .get(:correlation_id) - return [] unless correlation_id - - db[@messages_table] - .where(correlation_id: correlation_id) - .order(:position) - .map { |row| deserialize(row) } - end - - # Current max position in the message log. - # - # @return [Integer] max position, or 0 if the store is empty - def latest_position - db[@messages_table].max(:position) || 0 - end - - # Delete all data from all tables and reset autoincrement. For testing only. - # - # @return [void] - def clear! - db[@offset_key_pairs_table].delete - db[@offsets_table].delete - db[@consumer_groups_table].delete - db[@message_key_pairs_table].delete - db[@key_pairs_table].delete - db[@messages_table].delete - db[@scheduled_messages_table].delete - db[@workers_table].delete - db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) - end - - private - - # Resolve a group_id argument that is either a String - # or an object responding to +#group_id+. - # - # @param group_id [String, #group_id] - # @return [String] - def resolve_group_id(group_id) - group_id.respond_to?(:group_id) ? group_id.group_id : group_id - end - - # Create offsets eagerly for all registered consumer groups. - # Called inside the append transaction after messages and key_pairs are inserted. - # - # @param messages [Array] messages being appended - # @return [void] - def ensure_offsets_for_registered_groups(messages) - return if @registered_groups.empty? - - # Collect all partition attribute names across registered groups - attr_names = @registered_groups.each_value.flat_map { |gi| gi[:partition_by] || [] }.uniq - - # Pre-fetch relevant key_pair IDs in one query, keyed by "name:value" - kp_id_cache = {} - db[@key_pairs_table].where(name: attr_names).each do |row| - kp_id_cache["#{row[:name]}:#{row[:value]}"] = row[:id] - end - - @registered_groups.each_value do |group_info| - partition_by = group_info[:partition_by] - next unless partition_by - - cg_id = group_info[:cg_id] - seen = Set.new - - messages.each do |msg| - keys = msg.extracted_keys.to_h # {"device_id"=>"dev-1", "name"=>"A"} - next unless partition_by.all? { |attr| keys.key?(attr) } - - values = partition_by.to_h { |attr| [attr, keys[attr]] } - pk = build_partition_key(partition_by, values) - next if seen.include?(pk) - seen << pk - - kp_ids = partition_by.map { |attr| kp_id_cache["#{attr}:#{values[attr]}"] } - create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) - end - end - end - - # Build canonical partition key string from attribute names and values. - # Sorted by attribute name for deterministic uniqueness. - # - # @param partition_by [Array] attribute names - # @param values [Hash{String => String}] attribute values keyed by name - # @return [String] e.g. "course_name:Algebra|user_id:joe" - def build_partition_key(partition_by, values) - partition_by.sort.map { |attr| "#{attr}:#{values[attr]}" }.join('|') - end - - # Scan a bounded window of messages forward from the consumer group's - # discovery_position watermark, find new partition tuples, create offsets - # for them, then advance the watermark. - # - # @param cg_id [Integer] consumer group internal ID - # @param partition_by [Array] sorted attribute names - # @param handled_types [Array] message type strings - # @return [void] - def discover_new_partitions(cg_id, partition_by, handled_types) - cg = db[@consumer_groups_table].where(id: cg_id).first - discovery_pos = cg[:discovery_position] - - types_list = handled_types.map { |t| db.literal(t) }.join(', ') - - # CTE per partition attribute: pre-joins message_key_pairs with key_pairs - # so the main query only self-joins on the CTEs (N joins instead of 2N). - ctes = [] - selects = [] - joins = [] - partition_by.each_with_index do |attr, i| - ctes << <<~CTE - attr#{i} AS ( - SELECT mkp.message_position, kp.id AS kp_id, kp.value AS val - FROM #{@message_key_pairs_table} mkp - JOIN #{@key_pairs_table} kp ON mkp.key_pair_id = kp.id AND kp.name = #{db.literal(attr)} - ) - CTE - selects << "a#{i}.kp_id AS kp_id_#{i}, a#{i}.val AS val_#{i}" - joins << "JOIN attr#{i} a#{i} ON m.position = a#{i}.message_position" - end - - group_by = partition_by.each_index.map { |i| "a#{i}.kp_id" }.join(', ') - - # Discover all partition tuples in the window (no NOT EXISTS — fast). - # Duplicates are filtered in Ruby against known partition_keys, and - # INSERT OR IGNORE handles any remaining races. - sql = <<~SQL - WITH #{ctes.join(",\n")} - SELECT #{selects.join(', ')}, - MIN(m.position) AS min_pos, - MAX(m.position) AS max_pos - FROM #{@messages_table} m - #{joins.join("\n")} - WHERE m.message_type IN (#{types_list}) - AND m.position > #{db.literal(discovery_pos)} - GROUP BY #{group_by} - ORDER BY min_pos ASC - LIMIT #{DISCOVERY_BATCH_SIZE} - SQL - - rows = db.fetch(sql).all - - max_discovered_pos = 0 - - if rows.any? - db.transaction do - rows.each do |row| - values = {} - kp_ids = [] - partition_by.each_with_index do |attr, i| - values[attr] = row[:"val_#{i}"] - kp_ids << row[:"kp_id_#{i}"] - end - - # INSERT OR IGNORE handles duplicates — no need to pre-filter. - # At most DISCOVERY_BATCH_SIZE (100) tuples per call, so the - # cost of re-attempting known offsets is bounded. - create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) - max_discovered_pos = row[:max_pos] if row[:max_pos] > max_discovered_pos - end - end - end - - # Advance watermark to the max position seen in the window (whether new or known). - # This ensures we don't re-scan these messages on the next discovery call. - max_window_pos = rows.any? ? rows.map { |r| r[:max_pos] }.max : 0 - new_watermark = [max_discovered_pos, max_window_pos, latest_position].select { |p| p > 0 }.min || discovery_pos - # If we found a full batch, advance only to the batch's max (more may follow). - # If we found fewer than a batch, we've scanned to the end — advance to latest. - new_watermark = latest_position if rows.size < DISCOVERY_BATCH_SIZE - - if new_watermark > discovery_pos - db[@consumer_groups_table].where(id: cg_id).update( - discovery_position: new_watermark, - updated_at: Time.now.iso8601 - ) - end - end - - # Ensure an offset row exists for a specific partition (by attribute values). - # Resolves key_pair IDs from the key_pairs table; returns nil if any - # partition attribute has no corresponding key_pair (meaning no messages - # with that attribute value exist yet). - # - # @param cg_id [Integer] consumer group internal ID - # @param partition [Hash{String => String}] attribute names and values - # @return [Integer, nil] offset ID, or nil if key_pairs not found - def ensure_offset_for_partition(cg_id, partition) - partition_by = partition.keys.sort - kp_ids = [] - partition_by.each do |attr| - kp = db[@key_pairs_table].where(name: attr, value: partition[attr].to_s).first - return nil unless kp - - kp_ids << kp[:id] - end - - create_offset_with_key_pairs(cg_id, partition_by, partition, kp_ids) - end - - # Create an offset row and its key_pair associations. Idempotent via INSERT OR IGNORE. - # - # @param cg_id [Integer] consumer group internal ID - # @param partition_by [Array] sorted attribute names - # @param values [Hash{String => String}] attribute values keyed by name - # @param kp_ids [Array] key_pair IDs - # @return [Integer] offset ID - def create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) - partition_key = build_partition_key(partition_by, values) - - db.run(<<~SQL) - INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) - VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) - SQL - - offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) - - kp_ids.each do |kp_id| - db.run(<<~SQL) - INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) - VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) - SQL - end - - offset_id - end - - # Find the next unclaimed partition with pending messages and claim it. - # Uses OR joins for candidate detection (any matching key_pair), then - # a NOT EXISTS clause to exclude messages that conflict on a shared - # attribute name (same name, different value). This gives AND semantics: - # a message only counts as pending for a partition when every attribute - # it shares with the partition has the same value. - # - # @param cg_id [Integer] consumer group internal ID - # @param handled_types [Array] message type strings - # @param worker_id [String] claiming worker identifier - # @return [Hash, nil] +{ offset_id:, partition_key:, last_position: }+ or nil - def find_and_claim_partition(cg_id, handled_types, worker_id) - types_list = handled_types.map { |t| db.literal(t) }.join(', ') - - row = nil - db[@offsets_table] - .where(consumer_group_id: cg_id, claimed: 0) - .select(:id, :partition_key, :last_position) - .order(:last_position) - .each do |offset| - - pending = db.fetch(<<~SQL).first - SELECT 1 - FROM #{@offset_key_pairs_table} okp - JOIN #{@message_key_pairs_table} mkp ON okp.key_pair_id = mkp.key_pair_id - JOIN #{@messages_table} m ON mkp.message_position = m.position - WHERE okp.offset_id = #{db.literal(offset[:id])} - AND m.position > #{db.literal(offset[:last_position])} - AND m.message_type IN (#{types_list}) - GROUP BY m.position - HAVING COUNT(*) = ( - SELECT COUNT(*) FROM #{@offset_key_pairs_table} WHERE offset_id = #{db.literal(offset[:id])} - ) - LIMIT 1 - SQL - - if pending - row = { offset_id: offset[:id], partition_key: offset[:partition_key], last_position: offset[:last_position] } - break - end - end - - return nil unless row - - now = Time.now.iso8601 - updated = db[@offsets_table] - .where(id: row[:offset_id], claimed: 0) - .update(claimed: 1, claimed_at: now, claimed_by: worker_id) - - return nil if updated == 0 - - row - end - - # Fetch messages for a partition using conditional AND semantics. - # For each candidate message, it must match ALL of the partition's attributes - # that the message itself has. Messages with a single partition attribute match - # on that one; messages with multiple must match all of them. - # - # @param key_pair_ids [Array] partition key_pair IDs - # @param last_position [Integer] fetch messages after this position - # @param handled_types [Array] message type strings - # @param limit [Integer, nil] max messages to return (nil = unlimited) - # @return [Array] - def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: nil) - return [] if key_pair_ids.empty? - - kp_ids_list = key_pair_ids.map { |id| db.literal(id) }.join(', ') - types_list = handled_types.map { |t| db.literal(t) }.join(', ') - - # CTE pre-resolves which attribute names the partition's key_pairs cover. - # The main query uses this to count-match: a message qualifies when it - # matches as many partition key_pairs as it has attributes in common with - # the partition (AND semantics for shared attributes). - sql = <<~SQL - WITH partition_attr_names AS ( - SELECT DISTINCT name FROM #{@key_pairs_table} WHERE id IN (#{kp_ids_list}) - ) - SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at - FROM #{@messages_table} m - WHERE m.position > #{db.literal(last_position)} - AND m.message_type IN (#{types_list}) - AND EXISTS ( - SELECT 1 FROM #{@message_key_pairs_table} mkp - WHERE mkp.message_position = m.position - AND mkp.key_pair_id IN (#{kp_ids_list}) - ) - AND ( - SELECT COUNT(*) FROM #{@message_key_pairs_table} mkp - WHERE mkp.message_position = m.position - AND mkp.key_pair_id IN (#{kp_ids_list}) - ) = ( - SELECT COUNT(DISTINCT kp_msg.name) - FROM #{@message_key_pairs_table} mkp2 - JOIN #{@key_pairs_table} kp_msg ON mkp2.key_pair_id = kp_msg.id - WHERE mkp2.message_position = m.position - AND kp_msg.name IN (SELECT name FROM partition_attr_names) - ) - ORDER BY m.position ASC - SQL - sql += " LIMIT #{db.literal(limit)}" if limit - - db.fetch(sql).map { |row| deserialize(row) } - end - - # Core query logic shared by {#read}, {#read_all}, and {#check_conflicts}. - # Resolves key_pair IDs from conditions, then queries messages. - # Attributes within each condition are AND'd; conditions are OR'd. - # - # @param conditions [Array] - # @param after_position [Integer, nil] only include messages after this position (exclusive) - # @param limit [Integer, nil] - # @param order [:asc, :desc] position order (default :asc) - # @return [Array] - def query_messages(conditions, after_position: nil, limit: nil, order: :asc) - subqueries = condition_position_subqueries(conditions, after_position: after_position) - return [] if subqueries.empty? - - union = subqueries.join(" UNION ") - direction = order == :desc ? "DESC" : "ASC" - - sql = <<~SQL - SELECT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at - FROM #{@messages_table} m - WHERE m.position IN (#{union}) - ORDER BY m.position #{direction} - SQL - sql += " LIMIT #{db.literal(limit)}" if limit - - db.fetch(sql).map { |row| deserialize(row) } - end - - # Check for conflicting messages after a given position. - # - # @param conditions [Array] - # @param after_position [Integer] - # @return [Array] - def check_conflicts(conditions, after_position) - return [] if conditions.empty? - - query_messages(conditions, after_position:) - end - - # Max position among messages matching the given conditions. - # Attributes within each condition are AND'd; conditions are OR'd. - # Returns after_position (or latest_position) if no matches. - # - # @param conditions [Array] - # @param after_position [Integer, nil] only consider messages after this position (exclusive) - # @return [Integer] - def max_position_for(conditions, after_position: nil) - return after_position || latest_position if conditions.empty? - - subqueries = condition_position_subqueries(conditions, after_position: after_position) - return after_position || latest_position if subqueries.empty? - - union = subqueries.join(" UNION ") - row = db.fetch("SELECT MAX(position) AS max_pos FROM (#{union})").first - row[:max_pos] || after_position || latest_position - end - - # Build per-condition position subqueries with AND-within/OR-across semantics. - # Resolves key_pair IDs, then builds one SQL subquery per condition. - # Each subquery selects positions where the message matches ALL attrs in the condition. - # - # @param conditions [Array] - # @param after_position [Integer, nil] only include positions after this (exclusive) - # @return [Array] SQL subquery strings (empty if no conditions can match) - def condition_position_subqueries(conditions, after_position: nil) - all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq - return [] if all_lookups.empty? - - or_clauses = all_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } - key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all - - key_pair_index = {} - key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } - - position_filter = after_position ? "AND m.position > #{db.literal(after_position)}" : "" - - conditions.filter_map do |c| - kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } - next if kp_ids.size < c.attrs.size - - kp_ids_list = kp_ids.map { |id| db.literal(id) }.join(', ') - - <<~SQL - SELECT m.position - FROM #{@messages_table} m - JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position - WHERE m.message_type = #{db.literal(c.message_type)} - AND mkp.key_pair_id IN (#{kp_ids_list}) - #{position_filter} - GROUP BY m.position - HAVING COUNT(DISTINCT mkp.key_pair_id) = #{kp_ids.size} - SQL - end - end - - # Deserialize a database row into a {PositionedMessage}. - # Looks up the message class from the registry; falls back to base {Message}. - # - # @param row [Hash] database row with :position, :message_id, :message_type, :causation_id, :correlation_id, :payload, :metadata, :created_at - # @return [PositionedMessage] - def deserialize(row) - payload = JSON.parse(row[:payload], symbolize_names: true) - metadata = row[:metadata] ? JSON.parse(row[:metadata], symbolize_names: true) : {} - - klass = Message.registry[row[:message_type]] - attrs = { - id: row[:message_id], - type: row[:message_type], - causation_id: row[:causation_id], - correlation_id: row[:correlation_id], - created_at: row[:created_at], - metadata: metadata, - payload: payload - } - - msg = if klass - klass.new(attrs) - else - Message.new(attrs) - end - - PositionedMessage.new(msg, row[:position]) - end - end - end -end diff --git a/lib/sourced/ccc/supervisor.rb b/lib/sourced/ccc/supervisor.rb deleted file mode 100644 index cc98b028..00000000 --- a/lib/sourced/ccc/supervisor.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/ccc/dispatcher' - -module Sourced - module CCC - # Top-level process entry point for CCC background workers. - # Creates a {Dispatcher} (which embeds Workers, CatchUpPoller, notifier, - # and StaleClaimReaper) and spawns it into an executor. - # - # Mirrors {Sourced::Supervisor} but simpler: no separate HouseKeepers, - # since housekeeping (heartbeat + stale claim reaping) is embedded in - # the CCC Dispatcher's StaleClaimReaper. - # - # @example Start with defaults - # CCC::Supervisor.start(router: my_ccc_router) - # - # @example Create and start manually - # supervisor = CCC::Supervisor.new(router: my_ccc_router, count: 4) - # supervisor.start - class Supervisor - # Start a new supervisor instance with the given options. - # - # @param args [Hash] Arguments passed to {#initialize} - # @return [void] This method blocks until the supervisor is stopped - def self.start(...) - new(...).start - end - - # @param router [CCC::Router] the CCC router providing reactors and store - # @param logger [Object] Logger instance for supervisor output - # @param count [Integer] Number of worker fibers to spawn - # @param batch_size [Integer] Messages per backend fetch - # @param max_drain_rounds [Integer] Max drain iterations per reactor pickup - # @param catchup_interval [Numeric] Seconds between catch-up polls - # @param housekeeping_interval [Numeric] Seconds between heartbeat/reap cycles - # @param claim_ttl_seconds [Integer] Stale claim age threshold in seconds - # @param executor [Object] Executor instance for running concurrent workers - def initialize( - router: CCC.router, - logger: CCC.config.logger, - count: CCC.config.worker_count, - batch_size: CCC.config.batch_size, - max_drain_rounds: CCC.config.max_drain_rounds, - catchup_interval: CCC.config.catchup_interval, - housekeeping_interval: CCC.config.housekeeping_interval, - claim_ttl_seconds: CCC.config.claim_ttl_seconds, - executor: Sourced.config.executor - ) - @router = router - @logger = logger - @count = count - @batch_size = batch_size - @max_drain_rounds = max_drain_rounds - @catchup_interval = catchup_interval - @housekeeping_interval = housekeeping_interval - @claim_ttl_seconds = claim_ttl_seconds - @executor = executor - end - - # Start the supervisor and dispatcher. - # This method blocks until the supervisor receives a shutdown signal. - def start - logger.info("CCC::Supervisor: starting with #{@count} workers and #{@executor} executor") - set_signal_handlers - - @dispatcher = Dispatcher.new( - router: @router, - worker_count: @count, - batch_size: @batch_size, - max_drain_rounds: @max_drain_rounds, - catchup_interval: @catchup_interval, - housekeeping_interval: @housekeeping_interval, - claim_ttl_seconds: @claim_ttl_seconds, - logger: logger - ) - - @executor.start do |task| - @dispatcher.spawn_into(task) - end - end - - # Stop all components gracefully. - def stop - logger.info('CCC::Supervisor: stopping dispatcher') - @dispatcher&.stop - logger.info('CCC::Supervisor: all workers stopped') - end - - # Set up signal handlers for graceful shutdown. - def set_signal_handlers - Signal.trap('INT') { stop } - Signal.trap('TERM') { stop } - end - - private - - attr_reader :logger - end - end -end diff --git a/lib/sourced/ccc/sync.rb b/lib/sourced/ccc/sync.rb deleted file mode 100644 index bbc95ecf..00000000 --- a/lib/sourced/ccc/sync.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Sync mixin for CCC reactors. - # Registers blocks that run within the store transaction (+sync+) - # or after the transaction commits (+after_sync+). - module Sync - def self.included(base) - super - base.extend ClassMethods - end - - # Build {Actions::Sync} wrappers for all registered +sync+ blocks. - # - # @param args [Hash] keyword arguments forwarded to each block - # @return [Array] - def sync_actions(**args) - self.class.sync_blocks.map do |block| - Actions::Sync.new(proc { instance_exec(**args, &block) }) - end - end - - # Build {Actions::AfterSync} wrappers for all registered +after_sync+ blocks. - # - # @param args [Hash] keyword arguments forwarded to each block - # @return [Array] - def after_sync_actions(**args) - self.class.after_sync_blocks.map do |block| - Actions::AfterSync.new(proc { instance_exec(**args, &block) }) - end - end - - # Build all sync and after_sync actions together. - # - # @param args [Hash] keyword arguments forwarded to each block - # @return [Array] - def collect_actions(**args) - sync_actions(**args) + after_sync_actions(**args) - end - - module ClassMethods - # @api private - def inherited(subclass) - super - sync_blocks.each do |blk| - subclass.sync_blocks << blk - end - after_sync_blocks.each do |blk| - subclass.after_sync_blocks << blk - end - end - - # @return [Array] registered sync blocks - def sync_blocks - @sync_blocks ||= [] - end - - # Register a block to run inside the store transaction. - # - # The block receives the same keyword arguments as the reactor's - # action-building step (e.g. +state:+, +messages:+, +events:+ - # for deciders, or +state:+, +messages:+, +replaying:+ for projectors). - # - # @yield [**args] side-effect block executed within the transaction - # @return [void] - def sync(&block) - sync_blocks << block - end - - # @return [Array] registered after_sync blocks - def after_sync_blocks - @after_sync_blocks ||= [] - end - - # Register a block to run after the store transaction commits. - # - # Use this for side effects that should only happen on successful - # commit (e.g. sending emails, HTTP calls, pushing to external queues). - # - # The block receives the same keyword arguments as +sync+. - # - # @yield [**args] side-effect block executed after transaction commit - # @return [void] - def after_sync(&block) - after_sync_blocks << block - end - end - end - end -end diff --git a/lib/sourced/ccc/testing/rspec.rb b/lib/sourced/ccc/testing/rspec.rb deleted file mode 100644 index 27f350d5..00000000 --- a/lib/sourced/ccc/testing/rspec.rb +++ /dev/null @@ -1,284 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/ccc' -require 'sourced/ccc/store' - -module Sourced - module CCC - module Testing - module RSpec - NONE = [].freeze - - # Entry point for CCC reactor GWT tests. - # - # Works with any reactor that responds to the standard - # handle_batch(partition_values, new_messages, history:, replaying:) - # contract (Deciders, Projectors, DurableWorkflows). - # - # @param reactor_class [Class] a CCC reactor class - # @param partition_attrs [Hash] partition key-value pairs (e.g. device_id: 'd1') - # @return [GWT] - # - # @example Decider - # with_reactor(MyDecider, device_id: 'd1') - # .given(DeviceRegistered, device_id: 'd1', name: 'Sensor') - # .when(BindDevice, device_id: 'd1', asset_id: 'a1') - # .then(DeviceBound, device_id: 'd1', asset_id: 'a1') - # - # @example Projector — assert state via block - # with_reactor(MyProjector, list_id: 'L1') - # .given(ItemAdded, list_id: 'L1', name: 'Apple') - # .then { |r| expect(r.state[:items]).to eq(['Apple']) } - def with_reactor(reactor_class, **partition_attrs) - GWT.new(reactor_class, **partition_attrs) - end - - # Uniform result yielded to +.then+ / +.then!+ block callbacks. - # Gives access to both the reactor's produced action pairs / messages - # (what deciders typically assert on) and the evolved state (what - # projectors typically assert on). - RunResult = Data.define(:pairs, :messages, :state) - - class MessageMatcher - def initialize(expected_messages) - @expected_messages = Array(expected_messages) - @errors = [] - @mismatching = Hash.new { |h, k| h[k] = [] } - end - - def matches?(actual_messages) - if @expected_messages.size != actual_messages.size - @errors << "Expected #{@expected_messages.size} messages, but got #{actual_messages.size}" - @errors << actual_messages.inspect - return false - end - - @expected_messages.each.with_index do |expected, idx| - actual = actual_messages[idx] - @mismatching[idx] << "expected a #{expected.class}, got #{actual.class}" unless actual.class == expected.class - @mismatching[idx] << "expected payload #{expected.payload.to_h.inspect}, got #{actual.payload.to_h.inspect}" unless expected.payload == actual.payload - end - - return false if @mismatching.any? - - true - end - - def failure_message - err = +@errors.join("\n") - @mismatching.each do |idx, errors| - err << "Message #{idx}: \n" - errors.each do |e| - err << "- #{e}\n" - end - err << "\n" - end - err - end - end - - class GWT - def initialize(reactor_class, **partition_attrs) - @reactor_class = reactor_class - @partition_values = partition_attrs - @given_messages = [] - @when_messages = [] - @asserted = false - end - - # Accumulate history / context messages. These become +history.messages+ - # passed to the reactor's +handle_batch+. - # - # @param klass_or_instance [Class, CCC::Message] - # @param payload [Hash] - # @return [self] - def given(klass_or_instance = nil, **payload) - raise 'test case already asserted' if @asserted - - @given_messages << build_message(klass_or_instance, **payload) - self - end - - alias_method :and, :given - - # The batch of new messages to process via +handle_batch+. Can be - # called multiple times to supply several messages. - # - # @param klass_or_instance [Class, CCC::Message] - # @param payload [Hash] - # @return [self] - def when(klass_or_instance = nil, **payload) - raise 'test case already asserted' if @asserted - - @when_messages << build_message(klass_or_instance, **payload) - self - end - - # Assert expected outcomes. - # - # - Pass message class + payload pairs (or instances) to assert - # produced messages. - # - Pass +[]+ or +NONE+ to assert no messages. - # - Pass an Exception class (+ optional message) to assert the - # reactor raised. - # - Pass a block to receive a {RunResult} for custom assertions. - # - # @return [self] - def then(*expected, **payload, &block) - run_then(false, *expected, **payload, &block) - end - - # Like #then, but runs Sync / AfterSync actions before computing - # the result yielded to the block (or before extracting messages). - def then!(*expected, **payload, &block) - run_then(true, *expected, **payload, &block) - end - - private - - def build_message(klass_or_instance, **payload) - if klass_or_instance.is_a?(CCC::Message) - klass_or_instance - else - klass_or_instance.new(payload: payload) - end - end - - def run_then(sync, *expected, **payload, &block) - @asserted = true - - # Shorthand: .then(Class, key: val) → build message from class + payload - if expected.size == 1 && expected[0].is_a?(Class) && !(expected[0] < ::Exception) && payload.any? - expected = [expected[0].new(payload: payload)] - end - - # Exception expectation - if expected.size >= 1 && exception_expectation?(expected[0]) - expect_exception(expected[0], expected[1]) - return self - end - - pairs = run_handle_batch - - if sync - pairs.each do |actions, _| - Array(actions).select { |a| - a.is_a?(CCC::Actions::Sync) || a.is_a?(CCC::Actions::AfterSync) - }.each(&:call) - end - end - - if block_given? - block.call(RunResult.new(pairs: pairs, messages: extract_messages(pairs), state: compute_state(sync: sync))) - return self - end - - actual_messages = extract_messages(pairs) - expected_msgs = build_expected(*expected) - - if expected_msgs.empty? - unless actual_messages.empty? - ::RSpec::Expectations.fail_with( - "Expected no messages, but got #{actual_messages.size}: #{actual_messages.inspect}" - ) - end - return self - end - - matcher = MessageMatcher.new(expected_msgs) - unless matcher.matches?(actual_messages) - ::RSpec::Expectations.fail_with(matcher.failure_message) - end - - self - end - - def run_handle_batch - guard = CCC::ConsistencyGuard.new(conditions: [], last_position: 0) - history = CCC::ReadResult.new(messages: @given_messages, guard: guard) - @reactor_class.handle_batch( - @partition_values, - @when_messages, - history: history, - replaying: false - ) - end - - # Build an instance and evolve it with all known messages so the - # caller can assert on state regardless of reactor type. For reactors - # whose +handle_batch+ evolves its own instance (Decider, Projector, - # DurableWorkflow), this is an independent, predictable computation. - # When +sync+ is true, also runs the reactor's Sync / AfterSync - # blocks against this instance so their state mutations are visible. - def compute_state(sync: false) - instance = @reactor_class.new(@partition_values) - return nil unless instance.respond_to?(:evolve) - - messages = @given_messages + @when_messages - instance.evolve(messages) - run_sync_on(instance, messages) if sync - instance.state - end - - # Invoke Sync / AfterSync blocks against +instance+. Per-block - # kwarg signatures vary by reactor type (deciders expect +events:+, - # projectors expect +replaying:+); we inspect each block's - # parameters and pass only what it declares. - def run_sync_on(instance, messages) - all_args = { state: instance.state, messages: messages, events: [], replaying: false } - klass = instance.class - blocks = [] - blocks.concat(klass.sync_blocks) if klass.respond_to?(:sync_blocks) - blocks.concat(klass.after_sync_blocks) if klass.respond_to?(:after_sync_blocks) - blocks.each do |block| - wanted = block.parameters.select { |type, _| type == :keyreq || type == :key }.map(&:last) - instance.instance_exec(**all_args.slice(*wanted), &block) - end - end - - def extract_messages(pairs) - pairs.flat_map { |actions, _| - Array(actions) - .select { |a| a.respond_to?(:messages) } - .flat_map(&:messages) - } - end - - def build_expected(*args) - return [] if args == [[]] || args == [NONE] - return [] if args.empty? - - args.map do |arg| - case arg - when CCC::Message - arg - else - raise ArgumentError, "unsupported expected message: #{arg.inspect}" - end - end - end - - def exception_expectation?(arg) - arg.is_a?(Class) && arg < ::Exception - end - - def expect_exception(exception_class, message = nil) - begin - run_handle_batch - rescue exception_class => e - if message && e.message != message - ::RSpec::Expectations.fail_with( - "expected #{exception_class} with message #{message.inspect}, " \ - "but got #{e.message.inspect}" - ) - end - return - end - - ::RSpec::Expectations.fail_with("expected #{exception_class} to be raised, but nothing was raised") - end - end - end - end - end -end diff --git a/lib/sourced/ccc/topology.rb b/lib/sourced/ccc/topology.rb deleted file mode 100644 index ee7fab63..00000000 --- a/lib/sourced/ccc/topology.rb +++ /dev/null @@ -1,437 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/topology' - -module Sourced - module CCC - # Analyzes registered CCC reactors (Deciders and Projectors) and builds a - # flat array of node structs describing the message flow graph. This enables - # visualization, introspection, and Event Modeling diagram generation for - # CCC-based systems. - # - # Works like {Sourced::Topology} but adapted for CCC's stream-less messages, - # CCC-specific handler prefixes (+ccc_decide+, +ccc_reaction+), and - # {CCC::Message} registry. - # - # @example Build topology from registered reactors - # nodes = Sourced::CCC::Topology.build([MyDecider, MyProjector]) - # nodes.each do |node| - # puts "#{node.type}: #{node.name} (#{node.id})" - # end - # - # @example Access topology via the CCC module (uses global router) - # Sourced::CCC.register(MyDecider) - # Sourced::CCC.register(MyProjector) - # Sourced::CCC.topology.each { |node| puts node.id } - # - # @example Filter by node type - # commands = Sourced::CCC::Topology.build(reactors).select { |n| n.type == 'command' } - # commands.each { |cmd| puts "#{cmd.name} => #{cmd.produces.inspect}" } - # - # @see Sourced::Topology - module Topology - # @!parse - # # A command node in the topology graph. - # # @attr type [String] always +'command'+ - # # @attr id [String] message type string (e.g. +'orders.create_order'+) - # # @attr name [String] Ruby class name (e.g. +'Orders::CreateOrder'+) - # # @attr group_id [String] reactor group that handles this command - # # @attr produces [Array] event type strings produced by this command handler - # # @attr schema [Hash] JSON Schema of the command payload, or +{}+ - # CommandNode = Struct.new(:type, :id, :name, :group_id, :produces, :schema) - # - # # An event node in the topology graph. - # # @attr type [String] always +'event'+ - # # @attr id [String] message type string - # # @attr name [String] Ruby class name - # # @attr group_id [String] reactor group where this event was first seen - # # @attr produces [Array] always +[]+ - # # @attr schema [Hash] JSON Schema of the event payload, or +{}+ - # EventNode = Struct.new(:type, :id, :name, :group_id, :produces, :schema) - # - # # An automation (reaction) node in the topology graph. - # # @attr type [String] always +'automation'+ - # # @attr id [String] composite ID (e.g. +'evt.type-GroupId-aut'+) - # # @attr name [String] human-readable label (e.g. +'reaction(WidgetCreated)'+) - # # @attr group_id [String] reactor group that owns this reaction - # # @attr consumes [Array] event type strings or readmodel IDs consumed - # # @attr produces [Array] command type strings dispatched - # AutomationNode = Struct.new(:type, :id, :name, :group_id, :consumes, :produces) - # - # # A read model node in the topology graph (projectors only). - # # @attr type [String] always +'readmodel'+ - # # @attr id [String] composite ID (e.g. +'my_app.widget_projector-rm'+) - # # @attr name [String] projector group_id - # # @attr group_id [String] projector group_id - # # @attr consumes [Array] event type strings that feed this read model - # # @attr produces [Array] automation node IDs triggered by this read model - # # @attr schema [Hash] always +{}+ - # ReadModelNode = Struct.new(:type, :id, :name, :group_id, :consumes, :produces, :schema) - - CommandNode = Sourced::Topology::CommandNode - EventNode = Sourced::Topology::EventNode - AutomationNode = Sourced::Topology::AutomationNode - ReadModelNode = Sourced::Topology::ReadModelNode - - # Prism-based source analyzer adapted for CCC handler prefixes. - # - # Overrides {Sourced::Topology::SourceAnalyzer#events_produced_by} and - # {Sourced::Topology::SourceAnalyzer#commands_dispatched_by} to use the - # +ccc_decide+ and +ccc_reaction+ method name prefixes instead of the - # main Sourced +decide+ and +reaction+ prefixes. - # - # @example - # analyzer = CCC::Topology::Analyzer.new - # refs = analyzer.events_produced_by(MyDecider, CreateWidget) - # # => [[:const, "WidgetCreated"]] - # - # @see Sourced::Topology::SourceAnalyzer - class Analyzer < Sourced::Topology::SourceAnalyzer - # Extract event references from a CCC command handler block. - # - # Looks up the handler method generated by +Decider.command(CmdClass)+ - # and uses Prism AST analysis to find +event(...)+ calls within it. - # - # @param reactor [Class] a {CCC::Decider} subclass - # @param cmd_class [Class] a {CCC::Command} subclass handled by the reactor - # @return [Array] AST references, e.g. +[[:const, "WidgetCreated"]]+ - # - # @example - # analyzer.events_produced_by(WidgetDecider, CreateWidget) - # # => [[:const, "WidgetCreated"]] - def events_produced_by(reactor, cmd_class) - return [] unless @prism_available - - method_name = Sourced.message_method_name('ccc_decide', cmd_class.name) - extract_calls_from_handler(reactor, method_name, :event) - end - - # Extract dispatch references from a CCC reaction handler block. - # - # Looks up the handler method generated by +reaction(EvtClass)+ - # and uses Prism AST analysis to find +dispatch(...)+ calls within it. - # Follows chained calls like +dispatch(Cmd).at(time)+. - # - # @param reactor [Class] a CCC reactor class (Decider or Projector subclass) - # @param evt_class [Class] the event class whose reaction handler to analyze - # @return [Array] AST references, e.g. +[[:const, "NotifyWidget"]]+ - # - # @example - # analyzer.commands_dispatched_by(WidgetDecider, WidgetCreated) - # # => [[:const, "NotifyWidget"]] - def commands_dispatched_by(reactor, evt_class) - return [] unless @prism_available - - method_name = Sourced.message_method_name(CCC::React::PREFIX, evt_class.name) - extract_calls_from_handler(reactor, method_name, :dispatch) - end - end - - # Analyze registered CCC reactors and build the topology graph. - # - # Iterates each reactor class and extracts: - # - {CommandNode}s from Decider +handled_commands+, with Prism-extracted +produces+ - # - {EventNode}s from command produces and evolve handlers, deduplicated by type string - # - {AutomationNode}s from specific and catch-all reactions, with Prism-extracted +produces+ - # - {ReadModelNode}s for Projector subclasses, linking consumed events to automation outputs - # - # Nodes are deduplicated across reactors: the first reactor to declare a command or - # event "owns" it (gets its +group_id+). - # - # @param reactors [Enumerable] reactor classes ({CCC::Decider} and/or - # {CCC::Projector} subclasses) - # @return [Array] - # flat array of topology nodes - # - # @example Build from explicit reactor list - # nodes = CCC::Topology.build([OrderDecider, OrderProjector]) - # - # nodes.select { |n| n.type == 'command' }.each do |cmd| - # puts "#{cmd.name} produces: #{cmd.produces}" - # end - # # OrderDecider::PlaceOrder produces: ["orders.order_placed"] - # - # @example Inspect automation (reaction) wiring - # nodes = CCC::Topology.build([OrderDecider, NotificationDecider]) - # - # nodes.select { |n| n.type == 'automation' }.each do |aut| - # puts "#{aut.name}: consumes #{aut.consumes} => dispatches #{aut.produces}" - # end - # # reaction(OrderPlaced): consumes ["orders.order_placed"] => dispatches ["notifications.send_receipt"] - # - # @example Projector readmodel wiring - # nodes = CCC::Topology.build([OrderSummaryProjector]) - # - # rm = nodes.find { |n| n.type == 'readmodel' } - # puts "#{rm.name} consumes #{rm.consumes}, triggers #{rm.produces}" - # # OrderSummaryProjector consumes ["orders.order_placed"], triggers ["orders.order_placed-OrderSummaryProjector-aut"] - def self.build(reactors) - analyzer = Analyzer.new - nodes = [] - command_ids = {} - event_nodes = {} - - reactors.each do |reactor| - group_id = reactor.group_id - - # Command nodes (deciders only) - if reactor.respond_to?(:handled_commands) - reactor.handled_commands.each do |cmd_class| - next if command_ids.key?(cmd_class.type) - - produced_refs = analyzer.events_produced_by(reactor, cmd_class) - produced_types = resolve_refs(produced_refs, reactor) - - schema = extract_schema(cmd_class) - cmd_node = CommandNode.new( - type: 'command', - id: cmd_class.type, - name: cmd_class.name, - group_id: group_id, - produces: produced_types, - schema: schema - ) - nodes << cmd_node - command_ids[cmd_class.type] = cmd_node - - # Register event nodes discovered via Prism from this command's handler - produced_types.each do |evt_type| - next if event_nodes.key?(evt_type) - next if command_ids.key?(evt_type) - - evt_class = find_event_class(evt_type) - next unless evt_class - - event_nodes[evt_type] = EventNode.new( - type: 'event', - id: evt_type, - name: evt_class.name, - group_id: group_id, - produces: [], - schema: extract_schema(evt_class) - ) - end - end - end - - # Event nodes from evolve handlers (covers projectors and events not yet seen) - if reactor.respond_to?(:handled_messages_for_evolve) - reactor.handled_messages_for_evolve.each do |evt_class| - # Skip command classes that ended up in evolve handlers - next if evt_class < CCC::Command - - evt_type = evt_class.type - next if event_nodes.key?(evt_type) - next if command_ids.key?(evt_type) - - event_nodes[evt_type] = EventNode.new( - type: 'event', - id: evt_type, - name: evt_class.name, - group_id: group_id, - produces: [], - schema: extract_schema(evt_class) - ) - end - end - - is_projector = reactor < CCC::Projector - rm_id = is_projector ? "#{Sourced::Types::ModuleToMessageType.parse(group_id)}-rm" : nil - projector_aut_ids = [] - - # Automation nodes from reactions - if reactor.respond_to?(:handled_messages_for_react) - catch_all_events = reactor.respond_to?(:catch_all_react_events) ? reactor.catch_all_react_events : Set.new - specific_events = reactor.handled_messages_for_react.reject { |e| catch_all_events.include?(e) } - - # Specific reactions: one automation node per reacted event - specific_events.each do |evt_class| - produced_refs = analyzer.commands_dispatched_by(reactor, evt_class) - produced_types = resolve_refs(produced_refs, reactor) - - aut_id = "#{evt_class.type}-#{group_id}-aut" - projector_aut_ids << aut_id if is_projector - - nodes << AutomationNode.new( - type: 'automation', - id: aut_id, - name: "reaction(#{evt_class.name})", - group_id: group_id, - consumes: is_projector ? [rm_id] : [evt_class.type], - produces: produced_types - ) - end - - # Catch-all reaction: single automation node for all catch-all events - if catch_all_events.any? - produced_refs = analyzer.commands_dispatched_by(reactor, catch_all_events.first) - produced_types = resolve_refs(produced_refs, reactor) - - group_type_id = Sourced::Types::ModuleToMessageType.parse(group_id) - aut_id = "#{group_type_id}-aut" - projector_aut_ids << aut_id if is_projector - - consumes = if is_projector - [rm_id] - else - catch_all_events.map(&:type) - end - - nodes << AutomationNode.new( - type: 'automation', - id: aut_id, - name: "reaction(#{group_id})", - group_id: group_id, - consumes: consumes, - produces: produced_types - ) - end - end - - # ReadModel nodes (projectors only) - if is_projector - consumes = reactor.handled_messages_for_evolve - .reject { |c| c < CCC::Command } - .map(&:type) - - nodes << ReadModelNode.new( - type: 'readmodel', - id: rm_id, - name: group_id, - group_id: group_id, - consumes: consumes, - produces: projector_aut_ids, - schema: {} - ) - end - end - - nodes + event_nodes.values - end - - # Resolve AST references to CCC message type strings. - # - # Each reference is a two-element array from the Prism analyzer: - # - +[:const, "WidgetCreated"]+ — constant reference - # - +[:symbol, "widget_created"]+ — symbol reference resolved via +reactor[]+ - # - +[:const_path, "MyApp::WidgetCreated"]+ — fully qualified constant - # - +[:const_index, {receiver: "Reactor", index: "cmd"}]+ — bracket accessor - # - # @param refs [Array] AST reference pairs - # @param reactor [Class] reactor class for namespace resolution - # @return [Array] resolved, unique message type strings - def self.resolve_refs(refs, reactor) - refs.filter_map do |ref_type, ref_value| - resolve_ref(ref_type, ref_value, reactor) - end.uniq - end - - # Resolve a single AST reference to a message type string. - # - # @param ref_type [Symbol] one of +:symbol+, +:const+, +:const_path+, +:const_index+ - # @param ref_value [String, Hash] the reference value from Prism analysis - # @param reactor [Class] reactor class for namespace resolution - # @return [String, nil] resolved type string, or +nil+ if unresolvable - def self.resolve_ref(ref_type, ref_value, reactor) - case ref_type - when :symbol - resolve_symbol_ref(ref_value, reactor) - when :const - resolve_const_ref(ref_value, reactor) - when :const_path - resolve_const_path_ref(ref_value) - when :const_index - resolve_const_index_ref(ref_value, reactor) - end - end - - # Resolve a symbol reference (e.g. +:widget_created+) via the reactor's - # bracket accessor (+reactor[:widget_created]+). - # - # @param symbol_name [String] symbol name without colon - # @param reactor [Class] reactor class responding to +[]+ - # @return [String, nil] type string or +nil+ - def self.resolve_symbol_ref(symbol_name, reactor) - sym = symbol_name.to_sym - if reactor.respond_to?(:[]) - begin - reactor[sym]&.type - rescue StandardError - nil - end - end - end - - # Resolve an unqualified constant name by searching the reactor's module - # hierarchy, then top-level. - # - # @param const_name [String] unqualified constant name (e.g. +"WidgetCreated"+) - # @param reactor [Class] reactor class for namespace context - # @return [String, nil] type string or +nil+ - def self.resolve_const_ref(const_name, reactor) - klass = Sourced::Topology.send(:resolve_constant_in_context, const_name, reactor) - klass&.respond_to?(:type) ? klass.type : nil - end - - # Resolve a fully qualified constant path (e.g. +"MyApp::WidgetCreated"+). - # - # @param path [String] fully qualified constant path - # @return [String, nil] type string or +nil+ - def self.resolve_const_path_ref(path) - klass = Object.const_get(path) - klass.type - rescue NameError - nil - end - - # Resolve a +Reactor[:symbol]+ bracket-accessor reference. - # - # @param ref [Hash] with +:receiver+ (constant name) and +:index+ (symbol name) - # @param reactor [Class] reactor class for namespace resolution - # @return [String, nil] type string or +nil+ - def self.resolve_const_index_ref(ref, reactor) - receiver_name = ref[:receiver] - klass = if receiver_name.include?('::') - Object.const_get(receiver_name) - else - Sourced::Topology.send(:resolve_constant_in_context, receiver_name, reactor) - end - return nil unless klass && klass.respond_to?(:[]) - - klass[ref[:index].to_sym].type - rescue StandardError - nil - end - - # Look up a CCC event class by type string from {CCC::Message.registry}. - # - # @param type_string [String] message type (e.g. +"orders.order_placed"+) - # @return [Class, nil] the event class or +nil+ - def self.find_event_class(type_string) - CCC::Message.registry[type_string] - end - - # Extract JSON Schema from a message class's +Payload+ constant. - # - # @param msg_class [Class] a {CCC::Message} subclass - # @return [Hash] JSON Schema hash, or +{}+ if no schema is available - def self.extract_schema(msg_class) - return {} unless msg_class.const_defined?(:Payload, false) - - payload_class = msg_class::Payload - if payload_class.respond_to?(:to_json_schema) - payload_class.to_json_schema - else - {} - end - rescue StandardError - {} - end - - private_class_method :resolve_refs, :resolve_ref, :resolve_symbol_ref, - :resolve_const_ref, :resolve_const_path_ref, - :resolve_const_index_ref, - :find_event_class, :extract_schema - end - end -end diff --git a/lib/sourced/ccc/worker.rb b/lib/sourced/ccc/worker.rb deleted file mode 100644 index 3b4754a4..00000000 --- a/lib/sourced/ccc/worker.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module CCC - # Processes CCC reactors from a {WorkQueue} in a signal-driven drain loop. - # - # Mirrors {Sourced::Worker} but calls {CCC::Router#handle_next_for} instead - # of the Sourced-specific +handle_next_event_for_reactor+. - # - # @example Signal-driven mode (production) - # queue = Sourced::WorkQueue.new(max_per_reactor: 4) - # worker = CCC::Worker.new(work_queue: queue, router: router, name: 'worker-0') - # worker.run # blocks, processing reactors popped from queue - # - # @example Single-tick for testing - # worker = CCC::Worker.new(work_queue: queue, router: router, name: 'test') - # worker.tick(MyReactor) # => true if messages were processed - class Worker - # @return [String] unique identifier for this worker instance - attr_reader :name - - # @param work_queue [WorkQueue] queue to receive reactor signals from - # @param router [CCC::Router] CCC router for dispatching messages - # @param name [String] unique name for this worker - # @param batch_size [Integer] max messages per claim - # @param max_drain_rounds [Integer] max consecutive drain iterations per reactor pickup - # @param logger [Object] logger instance - def initialize( - work_queue:, - router:, - name: SecureRandom.hex(4), - batch_size: 50, - max_drain_rounds: 10, - logger: CCC.config.logger - ) - @work_queue = work_queue - @router = router - @name = [Process.pid, name].join('-') - @batch_size = batch_size - @max_drain_rounds = max_drain_rounds - @logger = logger - @running = false - end - - # Signal the worker to stop after the current drain completes. - # @return [void] - def stop - @running = false - end - - # Main run loop. Blocks on the {WorkQueue} waiting for reactor signals. - # @return [void] - def run - @running = true - - while @running - reactor = @work_queue.pop - break if reactor.nil? # shutdown sentinel - - drain(reactor) - end - - @logger.info "CCC::Worker #{name}: stopped" - end - - # Drain available messages for a reactor in a bounded loop. - # - # Processes up to +max_drain_rounds+ batches. If all rounds are consumed, - # re-enqueues the reactor for fair scheduling across all reactors. - # - # @param reactor [Class] reactor class to drain messages for - # @return [void] - def drain(reactor) - rounds = 0 - while @running && rounds < @max_drain_rounds - found = @router.handle_next_for(reactor, worker_id: name, batch_size: @batch_size) - break unless found - - rounds += 1 - end - # More work likely — re-enqueue so another worker (or this one) continues - @work_queue.push(reactor) if @running && rounds >= @max_drain_rounds - end - - # Process one tick of work for a specific reactor. Convenience for testing. - # - # @param reactor [Class] reactor class to process - # @return [Boolean] true if messages were processed, false otherwise - def tick(reactor) - @router.handle_next_for(reactor, worker_id: name, batch_size: @batch_size) - end - end - end -end diff --git a/lib/sourced/command_context.rb b/lib/sourced/command_context.rb index 003d40f9..b831413d 100644 --- a/lib/sourced/command_context.rb +++ b/lib/sourced/command_context.rb @@ -3,65 +3,112 @@ require 'sourced/types' module Sourced - # A command factory to instantiate commands from Hash attributes - # including extra metadata. - # @example + # Builds commands instances with shared default metadata. # - # ctx = Sourced::CommandContext.new( - # stream_id: params[:stream_id], - # metadata: { - # user_id: session[:user_id] - # } - # ) + # @example Build a command from a type string + # ctx = CommandContext.new(metadata: { user_id: 10 }) + # cmd = ctx.build(type: 'orders.place', payload: { item: 'hat' }) + # cmd.metadata[:user_id] # => 10 # - # # params[:command] should be a Hash with { type: String, payload: Hash | nil } - # - # cmd = ctx.build(params[:command]) - # cmd.stream_id # String - # cmd.metadata[:user_id] # == session[:user_id] - # - # Passing a command subclass will scope command lookup to subclasses of that class. - # Useful for restricting clients to a specific set of commands. - # - # @example - # - # ctx = Sourced::CommandContext.new(scope: PublicCommand) - # - # cmd = ctx.build(type: 'do_something', payload: { foo: 'bar' }) - # - # # Or with class and attrs - # cmd = ctx.build(SomeCommand, stream_id: '111', payload: { foo: 'bar' }) - # - # Attempting to build a command not in the scope will raise an error. + # @example Scope to a custom command subclass + # scope = Class.new(Sourced::Command) + # MyCmd = scope.define('my.cmd') { attribute :name, String } + # ctx = CommandContext.new(scope: scope) + # cmd = ctx.build(type: 'my.cmd', payload: { name: 'hello' }) class CommandContext - # @option stream_id [String] - # @option metadata [Hash] metadata to add to commands built by this context - # @option scope [Sourced::Message] Message class to use as command registry - def initialize(stream_id: nil, metadata: Plumb::BLANK_HASH, scope: Sourced::Command) - @defaults = { - stream_id:, - metadata: - }.freeze + class << self + # Register a block to run when building specific command type(s). + # The block receives the app scope and the command, and must return the (possibly modified) command. + # + # @param message_classes [Class] one or more command classes to match + # @yield [app, cmd] transformation block + # @return [void] + def on(*message_classes, &block) + message_classes.each { |klass| (message_blocks[klass] ||= []) << block } + end + + # Register a block to run for all commands. + # The block receives the app scope and the command, and must return the (possibly modified) command. + # + # @yield [app, cmd] transformation block + # @return [void] + def any(&block) + any_blocks << block + end + + # @api private + def message_blocks + @message_blocks ||= {} + end + + # @api private + def any_blocks + @any_blocks ||= [] + end + + # @api private + def inherited(subclass) + super + message_blocks.each { |k, v| subclass.message_blocks[k] = v.dup } + any_blocks.each { |blk| subclass.any_blocks << blk } + end + end + + # @param metadata [Hash] default metadata merged into every command built by this context + # @param scope [Class] message class whose registry is used to resolve type strings (default: {Sourced::Command}) + # @param app [Object, nil] request-scoped object passed to callback blocks + # + # @example + # ctx = CommandContext.new(metadata: { user_id: 42 }, app: rack_app) + def initialize(metadata: Plumb::BLANK_HASH, scope: Sourced::Command, app: nil) + @defaults = { metadata: }.freeze @scope = scope + @app = app end - # @param attrs [Hash] attributes to lookup and buils a scope from. - # @return [Sourced::Message] + # Build a command instance, merging in default metadata. + # + # @overload build(attrs) + # Resolve the command class from the +type+ key in +attrs+ via the scope's registry. + # @param attrs [Hash] must include +:type+ and +:payload+ keys + # @return [Sourced::Message] + # @example + # ctx.build(type: 'orders.place', payload: { item: 'hat' }) + # + # @overload build(klass, attrs) + # Use the given command class directly. + # @param klass [Class] a Sourced::Command subclass + # @param attrs [Hash] must include +:payload+ key + # @return [Sourced::Message] + # @example + # ctx.build(PlaceOrder, payload: { item: 'hat' }) + # + # @raise [ArgumentError] if arguments don't match either form + # @raise [Sourced::UnknownMessageError] if the type string is not registered in the scope def build(*args) - case args - in [Class => klass, Hash => attrs] - attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) - klass.parse(attrs) - in [Hash => attrs] - attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) - scope.from(attrs) - else - raise ArgumentError, "Invalid arguments: #{args.inspect}" - end + cmd = case args + in [Class => klass, Hash => attrs] + attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) + klass.parse(attrs) + in [Hash => attrs] + attrs = defaults.merge(Types::SymbolizedHash.parse(attrs)) + scope.from(attrs) + else + raise ArgumentError, "Invalid arguments: #{args.inspect}" + end + run_pipeline(cmd) end private - attr_reader :defaults, :scope + EMPTY_ARRAY = [].freeze + + attr_reader :defaults, :scope, :app + + def run_pipeline(cmd) + blocks = self.class.message_blocks[cmd.class] || EMPTY_ARRAY + steps = blocks + self.class.any_blocks + steps.reduce(cmd) { |c, st| instance_exec(app, c, &st) } + end end end diff --git a/lib/sourced/command_methods.rb b/lib/sourced/command_methods.rb deleted file mode 100644 index a5286b44..00000000 --- a/lib/sourced/command_methods.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -module Sourced - # Provides command invocation methods for actors. - # - # CommandMethods automatically generates instance methods from command definitions, - # allowing you to invoke commands in two ways: - # - # 1. **In-memory version** (e.g., `actor.start(name: 'Joe')`) - # - Validates the command and executes the decision handler - # - Returns a tuple of [cmd, new_events] - # - Does NOT persist events to backend - # - # 2. **Durable version** (e.g., `actor.start!(name: 'Joe')`) - # - Same as in-memory, but also appends events to backend - # - Raises {FailedToAppendMessagesError} if backend fails - # - # ## Usage - # - # Include the module in an Actor and define commands normally: - # - # class MyActor < Sourced::Actor - # include Sourced::CommandMethods - # - # command :create_item, name: String do |state, cmd| - # event :item_created, cmd.payload - # end - # end - # - # actor = MyActor.new(id: 'actor-123') - # cmd, events = actor.create_item(name: 'Widget') # In-memory - # cmd, events = actor.create_item!(name: 'Widget') # Persists to backend - # - # ## Method Naming - # - # Methods are created based on your command name: - # - Symbol commands like `:create_item` → `create_item` and `create_item!` methods - # - Class commands like `CreateItem` → `create_item` and `create_item!` methods - # - module CommandMethods - # Raised when the durable command (bang method) fails to append events to the backend. - # - # @see #__issue_command - class FailedToAppendMessagesError < StandardError - def initialize(cmd, events) - super <<~MSG - Failed to append events to backend. - Command is #{cmd.inspect} - Events are #{events.inspect} - MSG - end - end - - def self.included(base) - base.send :extend, ClassMethods - end - - # Issues a command without persisting to the backend. - # - # This is the core logic used by the generated command methods. It validates - # the command and executes the decision handler if valid. - # - # @private - # @param cmd_class [Class] A subclass of {Sourced::Message} representing the command - # @param payload [Hash] The command payload data - # @return [Array<(Sourced::Message, Array)>] A tuple of [command, events] - # where command is the validated command object and events are the produced events - # (empty array if command was invalid) - # - # @example - # cmd, events = __issue_command(MyActor::CreateItem, name: 'Widget') - # puts cmd.valid? # => true - # puts events.length # => 1 - private def __issue_command(cmd_class, payload = {}) - cmd = cmd_class.new(stream_id: id, payload:) - return [cmd, React::EMPTY_ARRAY] unless cmd.valid? - - [cmd, decide(cmd)] - end - - # @private - # Class methods that hook into the command definition and create instance methods. - module ClassMethods - # Hooks into the command definition to automatically generate instance methods. - # - # This method is called automatically when you define a command in your actor. - # It creates both in-memory and durable (bang) versions of the command method. - # - # @see Sourced::Actor.command - def command(*args, &block) - super.tap do |_| - case args - in [Symbol => cmd_name, *_] - klass_name = cmd_name.to_s.split('_').map(&:capitalize).join - cmd_class = const_get(klass_name) - __command_methods_define(cmd_name, cmd_class) - in [Class => cmd_class] if cmd_class < Sourced::Message - cmd_name = Sourced::Types::ModuleToMethodName.parse(cmd_class.name.split('::').last) - __command_methods_define(cmd_name, cmd_class) - end - end - end - - # Creates in-memory and durable command invocation methods. - # - # Defines two methods on the actor instance: - # - `method_name(**payload)` - In-memory version, validates and decides without persisting - # - `method_name!(**payload)` - Durable version, also appends events to backend - # - # @private - # @param cmd_name [Symbol] The command method name (e.g., :create_item) - # @param cmd_class [Class] The command message class - # - # @example Generated methods - # class MyActor < Sourced::Actor - # include Sourced::CommandMethods - # command :create_item, name: String do |state, cmd| - # event :item_created, cmd.payload - # end - # end - # - # actor = MyActor.new(id: 'a1') - # - # # In-memory version - # cmd, events = actor.create_item(name: 'Widget') - # # => Returns [cmd, events] without touching backend - # # => If invalid: [invalid_cmd, []] - # - # # Durable version - # cmd, events = actor.create_item!(name: 'Widget') - # # => Returns [cmd, events] if valid and appended successfully - # # => Raises FailedToAppendMessagesError if backend fails - # # => If invalid: [invalid_cmd, []] - private def __command_methods_define(cmd_name, cmd_class) - define_method(cmd_name) do |**payload| - __issue_command(cmd_class, payload) - end - - define_method("#{cmd_name}!") do |**payload| - cmd_events = __issue_command(cmd_class, payload) - return cmd_events unless cmd_events.first.valid? - - success = Sourced.config.backend.append_to_stream(id, cmd_events.last) - raise FailedToAppendMessagesError.new(*cmd_events) unless success - - cmd_events - end - end - end - end -end diff --git a/lib/sourced/configuration.rb b/lib/sourced/configuration.rb index 855f0428..1a19bae4 100644 --- a/lib/sourced/configuration.rb +++ b/lib/sourced/configuration.rb @@ -1,184 +1,77 @@ # frozen_string_literal: true -require 'console' #  comes with async gem -require 'sourced/types' -require 'sourced/backends/test_backend' -require 'sourced/pubsub/test' +require 'logger' require 'sourced/error_strategy' require 'sourced/async_executor' module Sourced - # Configure a Sourced app. - # @example - # - # Sourced.configure do |config| - # config.backend = Sequel.Postgres('postgres://localhost/mydb') - # config.logger = Logger.new(STDOUT) - # end - # class Configuration - #  Backends must expose these methods - BackendInterface = Types::Interface[ + StoreInterface = Types::Interface[ :installed?, - :reserve_next_for_reactor, - :append_to_stream, - :read_correlation_batch, - :read_stream, - :transaction, - :stats, - :updating_consumer_group, + :install!, + :append, + :read, + :read_partition, + :claim_next, + :ack, + :release, :register_consumer_group, - :start_consumer_group, - :stop_consumer_group, - :reset_consumer_group + :worker_heartbeat, + :release_stale_claims, + :notifier ] - PubSubInterface = Types::Interface[:publish, :subscribe] + attr_accessor :logger, :worker_count, :batch_size, + :catchup_interval, :max_drain_rounds, + :claim_ttl_seconds, :housekeeping_interval, + :executor - # Interface that all executors must implement - # @see AsyncExecutor - # @see ThreadExecutor - ExecutorInterface = Types::Interface[ - :start - ] - - attr_accessor :logger - # House-keeping configuration - # interval: main loop tick for housekeeping (seconds) - # heartbeat interval: how often to record heartbeats (seconds) - # claim_ttl_seconds: how long before a worker is considered dead for claim release - attr_accessor( - :worker_count, - :worker_batch_size, - :housekeeping_count, - :housekeeping_interval, - :housekeeping_heartbeat_interval, - :housekeeping_claim_ttl_seconds, - :catchup_interval, - :max_drain_rounds - ) - - attr_reader :backend, :executor, :pubsub + attr_reader :store, :router def initialize - @logger = Console - @backend = Backends::TestBackend.new - @pubsub = PubSub::Test.new - @error_strategy = ErrorStrategy.new - @executor = AsyncExecutor.new - @setup = false - # Worker and house-keeping defaults + @logger = Logger.new($stdout) @worker_count = 2 - @worker_batch_size = 50 - @housekeeping_count = 1 - @housekeeping_interval = 3 - @housekeeping_heartbeat_interval = 5 - @housekeeping_claim_ttl_seconds = 120 + @batch_size = 50 @catchup_interval = 5 @max_drain_rounds = 10 - @subscribers = [] - end - - def subscribe(callable = nil, &block) - callable ||= block - @subscribers << callable - self - end - - def setup! - return if @setup - - @backend.setup!(self) if @backend.respond_to?(:setup!) - @subscribers.each { |s| s.call(self) } - @setup = true + @claim_ttl_seconds = 120 + @housekeeping_interval = 30 + @executor = AsyncExecutor.new + @store = nil + @router = nil + @error_strategy = ErrorStrategy.new + @setup = false end - # Configure the backend for the app. - # Defaults to in-memory TestBackend - # Also auto-sets pubsub when backend is a PG database. - # @param bnd [BackendInterface] - def backend=(bnd) - @backend = case bnd.class.name - when 'Sequel::Postgres::Database' - require 'sourced/backends/pg_backend' - require 'sourced/pubsub/pg' - @pubsub = PubSub::PG.new(db: bnd, logger: @logger) - Sourced::Backends::PGBackend.new(bnd) + # Accepts a Sourced::Store instance, a Sequel::SQLite::Database connection + # (auto-wrapped in Sourced::Store.new(db)), or any object implementing StoreInterface. + def store=(s) + @store = case s.class.name when 'Sequel::SQLite::Database' - require 'sourced/backends/sqlite_backend' - Sourced::Backends::SQLiteBackend.new(bnd) - else - BackendInterface.parse(bnd) + require 'sourced/store' + Store.new(s) + else StoreInterface.parse(s) end end - # Configure the pubsub implementation for the app. - # Defaults to in-memory PubSub::Test. - # Automatically set when backend is a PG database. - # Can be overridden with any object implementing `subscribe` and `publish`. - # @param ps [#subscribe, #publish] - def pubsub=(ps) - @pubsub = PubSubInterface.parse(ps) - end - - # Configure the executor for the app. - # Supports both symbol shortcuts and executor instances. - # Defaults to AsyncExecutor for fiber-based concurrency. - # - # @param ex [Symbol, Object] The executor to use - # @option ex [Symbol] :async Use AsyncExecutor (fiber-based, default) - # @option ex [Symbol] :thread Use ThreadExecutor (thread-based) - # @option ex [ExecutorInterface] Custom executor instance - # @raise [ArgumentError] if executor doesn't implement ExecutorInterface - # - # @example Using symbol shortcuts - # config.executor = :async # Default fiber-based - # config.executor = :thread # Thread-based for CPU-intensive work - # - # @example Using custom executor - # config.executor = MyCustomExecutor.new - def executor=(ex) - @executor = case ex - when :async - AsyncExecutor.new - when :thread - require 'sourced/thread_executor' - ThreadExecutor.new - when ExecutorInterface - ex - else - raise ArgumentError, "executor=(e) must support interface #{ExecutorInterface.inspect}" - end - end - - # Assign an error strategy - # @param strategy [ErrorStrategy, #call(Exception, Sourced::Message, Group)] - # @raise [ArgumentError] if strategy does not respond to #call def error_strategy=(strategy) - raise ArgumentError, 'Must respond to #call(Exception, Sourced::Message, Group)' unless strategy.respond_to?(:call) + raise ArgumentError, 'Must respond to #call' unless strategy.respond_to?(:call) @error_strategy = strategy end - # Configure a built-in Sourced::ErrorStrategy - # @example - # config.error_strategy do |s| - # s.retry(times: 30, after: 50, backoff: ->(retry_after, retry_count) { retry_after * retry_count }) - # - # s.on_retry do |n, exception, message, later| - # puts "Retrying #{n} times" } - # end - # - # s.on_fail do |exception, message| - # Sentry.capture_exception(exception) - # end - # end - # - # @yieldparam s [ErrorStrategy] - def error_strategy(&blk) - return @error_strategy unless block_given? + attr_reader :error_strategy + + def setup! + return if @setup - self.error_strategy = ErrorStrategy.new(&blk) + unless @store + require 'sourced/store' + @store = Store.new(Sequel.sqlite) + end + @store.install! + @router ||= Router.new(store: @store) + @setup = true end end end diff --git a/lib/sourced/consumer.rb b/lib/sourced/consumer.rb index 03798da4..d6d3caf9 100644 --- a/lib/sourced/consumer.rb +++ b/lib/sourced/consumer.rb @@ -1,122 +1,164 @@ # frozen_string_literal: true module Sourced - # This mixin provides consumer info configuration - # and a .consumer_info method to access it. - # @example - # - # class MyConsumer - # extend Sourced::Consumer - # - # consumer do |c| - # # consumer group - # c.group_id = 'my-group' - # - # # Start consuming events from the beginning of history - # c.start_from = :beginning - # end - # end - # - # MyConsumer.consumer_info.group_id # => 'my-group' - # + # Accumulates mutations to a consumer group row for atomic persistence. + # Accumulates consumer group mutations for atomic persistence. + # Used by {Store#updating_consumer_group}. + class GroupUpdater + attr_reader :group_id, :updates, :error_context + + def initialize(group_id, row, logger) + @group_id = group_id + @logger = logger + @error_context = row[:error_context] + @updates = { error_context: @error_context.dup } + end + + def stop(message: nil) + @logger.error "Sourced: stopping consumer group #{group_id}" + @updates[:status] = Store::STOPPED + @updates[:retry_at] = nil + @updates[:updated_at] = Time.now.iso8601 + @updates[:error_context][:message] = message if message + end + + def fail(exception: nil) + @logger.error "Sourced: failing consumer group #{group_id}. #{exception&.class}: #{exception&.message}" + @updates[:status] = Store::FAILED + @updates[:retry_at] = nil + @updates[:updated_at] = Time.now.iso8601 + if exception + @updates[:error_context][:exception_class] = exception.class.to_s + @updates[:error_context][:exception_message] = exception.message + end + end + + def retry(time, **ctx) + @logger.warn "Sourced: retrying consumer group #{group_id} at #{time}" + @updates[:updated_at] = Time.now.iso8601 + @updates[:retry_at] = time.iso8601 + @updates[:error_context].merge!(ctx) + end + end + + # Shared consumer configuration for reactors. + # Extended (not included) onto reactor classes. module Consumer - class ConsumerInfo < Types::Data - ToBlock = Types::Any.transform(Proc) { |v| -> { v } } - StartFromBeginning = Types::Value[:beginning] >> Types::Static[nil] >> ToBlock - StartFromNow = Types::Value[:now] >> Types::Static[-> { Time.now - 5 }.freeze] - StartFromTime = Types::Interface[:call].check('must return a Time') { |v| v.call.is_a?(Time) } + def self.extended(base) + super + base.extend ClassMethods + end - StartFrom = ( - StartFromBeginning | StartFromNow | StartFromTime - ).default { -> { nil } } + def partition_keys + @partition_keys ||= [] + end - attribute :group_id, Types::String.present, writer: true - attribute :start_from, StartFrom, writer: true - attribute :batch_size, Types::Integer.nullable.default(nil), writer: true + def partition_by(*keys) + @partition_keys = keys.flatten.map(&:to_sym) end - def consumer_info - @consumer_info ||= ConsumerInfo.new(group_id: name, start_from: :beginning) + def group_id + @group_id ||= name end - def consumer(&) - return consumer_info unless block_given? + def consumer_group(id) + @group_id = id + end - info = ConsumerInfo.new(group_id: name) - yield info - raise Plumb::ParseError, info.errors unless info.valid? + # Message types this consumer evolves from. Used by {#context_for} + # to build query conditions for history reads. + # Defaults to empty; overridden by Sourced::Evolve mixin. + def handled_messages_for_evolve + @handled_messages_for_evolve ||= [] + end - @consumer_info = info + # Build query conditions from partition attributes and handled evolve types. + # Override in reactor for custom per-command conditions. + def context_for(partition_attrs) + handled_messages_for_evolve.flat_map { |klass| + klass.to_conditions(**partition_attrs) + } end - # Implement this in your reactors - # to manage exception handling in eventually-consistent workflows - # @example retry with exponential back off - # - # def self.on_exception(exception, _message, group) - # retry_count = group.error_context[:retry_count] || 0 - # if retry_count < 3 - # later = 5 + 5 * retry_count - # group.retry(later, retry_count: retry_count + 1) - # else - # group.fail(exception:) - # end - # end - # - # @param exception [Exception] the exception raised - # @param message [Sourced::Message] the event or command being handled - # @param group [#stop, #fail, #retry] consumer group object to update state, ie. for retries def on_exception(exception, message, group) Sourced.config.error_strategy.call(exception, message, group) end - # Default handle_batch implementation that wraps per-message .handle calls. - # Returns array of [actions, source_message] pairs. - # Reactors with optimized batch processing (Projector, Actor) override this. + # Called by {Router#stop_consumer_group} after the group is marked as stopped. + # Override in reactor classes to run cleanup logic on stop. # - # @param batch [Array<[Message, Boolean]>] array of [message, replaying] pairs - # @return [Array<[actions, source_message]>] action pairs - def handle_batch(batch) - each_with_partial_ack(batch) do |message, replaying| - kargs = {} - kargs[:replaying] = replaying if handle_kargs.include?(:replaying) - kargs[:logger] = Sourced.config.logger if handle_kargs.include?(:logger) - actions = handle(message, **kargs) - [actions, message] - end + # @param message [String, nil] optional reason for stopping + # @return [void] + def on_stop(message = nil) + # no-op by default end - private - - # Iterate batch with per-message error handling. - # Collects [actions, message] pairs returned by the block. - # On mid-batch failure, raises PartialBatchError with pairs collected so far, - # allowing the backend to ACK up to the last successful message. - # If the first message fails, raises the original error (no partial ACK possible). + # Called by {Router#reset_consumer_group} after the group's offsets are cleared. + # Override in reactor classes to run cleanup logic on reset + # (e.g. clearing caches or projections). # - # Used by Consumer (default handle_batch), Actor, and Projector (reaction loop) - # to provide partial ACK across all reactor types. + # @return [void] + def on_reset + # no-op by default + end + + # Called by {Router#start_consumer_group} after the group is marked as active. + # Override in reactor classes to run setup logic on start. # - # @param batch [Array<[Message, Boolean]>] array of [message, replaying] pairs - # @yield [message, replaying] called for each message in the batch - # @yieldreturn [Array(actions, Message), nil] action pair, or nil to skip - # @return [Array<[actions, source_message]>] action pairs - def each_with_partial_ack(batch) + # @return [void] + def on_start + # no-op by default + end + + # Iterate messages collecting [actions, message] pairs. + # On mid-batch failure, raises PartialBatchError with pairs collected so far. + # If the first message fails, re-raises the original error. + def each_with_partial_ack(messages) results = [] - batch.each do |message, replaying| - pair = yield(message, replaying) + messages.each do |msg| + pair = yield(msg) results << pair if pair rescue StandardError => e raise e if results.empty? - raise PartialBatchError.new(results, message, e) + raise Sourced::PartialBatchError.new(results, msg, e) end results end - # Lazily resolved keyword argument names for the reactor's .handle method. - # Cached as a class instance variable. - def handle_kargs - @handle_kargs ||= Injector.resolve_args(self, :handle) + module ClassMethods + # Resolve a messages class from a symbol or type-like string. + # + # Symbols are normalized by replacing dots with underscores before + # matching against registered message types. For example, + # +:course_created+ matches "course.created" and + # "course_created". + # + # @param message_symbol [Symbol, String] symbolic message identifier + # @return [Class, nil] matching messages class, or +nil+ if none found + # + # @example + # CourseDecider[:courses_created] + # # => CourseCreated + def [](message_symbol) + normalized = message_symbol.to_s.tr('.', '_') + find_registered_message_class(normalized) + end + + private + + def find_registered_message_class(normalized_name, base = Sourced::Message) + base.registry.keys.each do |type| + klass = base.registry[type] + return klass if type.tr('.', '_') == normalized_name + end + + base.subclasses.each do |subclass| + klass = find_registered_message_class(normalized_name, subclass) + return klass if klass + end + + nil + end end end end diff --git a/lib/sourced/decider.rb b/lib/sourced/decider.rb new file mode 100644 index 00000000..e389f973 --- /dev/null +++ b/lib/sourced/decider.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Sourced + # Reactor base class for command-handling workflows in Sourced. + class Decider + include Sourced::Evolve + include Sourced::React + include Sourced::Sync + extend Sourced::Consumer + + class << self + # @return [Array] command message classes handled by this decider + def handled_commands + @handled_commands ||= [] + end + + # Messages to claim: commands to decide on + events to react to. + # Evolve types are NOT included — they are only for context_for(). + # + # @return [Array] command and reaction message classes + def handled_messages + handled_commands + handled_messages_for_react + end + + # Register a command handler. + # + # @param message_class [Class] commands class to handle + # @yield [state, message] command handler block + # @return [void] + def command(message_class, &block) + handled_commands << message_class + define_method(Sourced.message_method_name('sourced_decide', message_class.to_s), &block) + end + + def handle_batch(partition_values, new_messages, history:, replaying: false) + instance = new(partition_values) + instance.evolve(history.messages) + + each_with_partial_ack(new_messages) do |msg| + if handled_commands.include?(msg.class) + raw_events = instance.decide(msg) + correlated_events = raw_events.map { |e| msg.correlate(e) } + actions = [] + actions.concat( + Actions.build_for(correlated_events, guard: history.guard, correlated: true) + ) + + correlated_events.each do |evt| + next unless instance.reacts_to?(evt) + reaction_msgs = Array(instance.react(evt)) + actions.concat(Actions.build_for(reaction_msgs, source: evt)) + end + + actions += instance.collect_actions( + state: instance.state, messages: [msg], events: raw_events + ) + + [actions, msg] + else + [Actions::OK, msg] + end + end + end + + # Build executable actions for a claimed batch. + # + # @param claim [ClaimResult] claimed partition batch + # @param history [ReadResult] event history for the partition + # @return [Array, PositionedMessage)>] action/source pairs + def handle_claim(claim, history:) + values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } + handle_batch(values, claim.messages, history:) + end + + # Copy registered command handlers into subclasses. + # + # @param subclass [Class] subclass being created + # @return [void] + def inherited(subclass) + super + handled_commands.each do |cmd_class| + subclass.handled_commands << cmd_class + end + end + end + + attr_reader :partition_values + + # @param partition_values [Hash{Symbol => String}] partition key-value pairs + def initialize(partition_values = {}) + @partition_values = partition_values + @uncommitted_events = [] + end + + # Decide a command against the decider's current in-memory state. + # + # @param command [Sourced::Message] command to handle + # @return [Array] newly produced events + def decide(command) + @uncommitted_events = [] + method_name = Sourced.message_method_name('sourced_decide', command.class.to_s) + send(method_name, state, command) if respond_to?(method_name) + @uncommitted_events.dup + end + + # Produce a new event from within a command handler and apply it + # to the decider's in-memory state immediately. + # + # Accepts either a messages class or a symbol resolved via .[]. + # + # @param event_class [Class, Symbol] event class or symbolic event name + # @param payload [Hash] payload attributes for the event + # @return [Sourced::Message] the newly built event + # + # @example Produce by class + # command RegisterDevice do |_state, cmd| + # event DeviceRegistered, device_id: cmd.payload.device_id + # end + # + # @example Produce by symbol + # command RegisterDevice do |_state, cmd| + # event :device_registered, device_id: cmd.payload.device_id + # end + def event(event_class, payload = {}) + event_class = self.class[event_class] if event_class.is_a?(Symbol) + evt = event_class.new(payload: payload) + @uncommitted_events << evt + evolve([evt]) + evt + end + end +end diff --git a/lib/sourced/dispatcher.rb b/lib/sourced/dispatcher.rb index bdd4d979..651b6618 100644 --- a/lib/sourced/dispatcher.rb +++ b/lib/sourced/dispatcher.rb @@ -1,41 +1,37 @@ # frozen_string_literal: true require 'sourced/work_queue' -require 'sourced/worker' require 'sourced/catchup_poller' +require 'sourced/worker' +require 'sourced/scheduled_message_poller' +require 'sourced/stale_claim_reaper' module Sourced # Orchestrator that wires together the signal-driven dispatch pipeline: - # {WorkQueue}, {NotificationQueuer}, {CatchUpPoller}, backend notifier, and {Worker}s. + # {WorkQueue}, {NotificationQueuer}, {CatchUpPoller}, store notifier, and {Worker}s. # - # Does not own the process lifecycle — the caller ({Supervisor} or Falcon service) - # provides the task/fiber context via {#spawn_into}, and triggers shutdown - # via {#stop}. + # Does not own the process lifecycle — the caller provides the task/fiber + # context via {#spawn_into}, and triggers shutdown via {#stop}. # - # @example Usage with Supervisor - # dispatcher = Dispatcher.new(router: Sourced::Router, worker_count: 4) + # @example Usage with a task runner + # dispatcher = Sourced::Dispatcher.new(router: router, worker_count: 4) # executor.start do |task| # dispatcher.spawn_into(task) # end - # # On shutdown: # dispatcher.stop # # @example With custom queue for testing # queue = WorkQueue.new(max_per_reactor: 2, queue: Queue.new) - # dispatcher = Dispatcher.new(router: router, work_queue: queue) + # dispatcher = Sourced::Dispatcher.new(router: router, work_queue: queue) class Dispatcher - # Subscriber for the backend notifier pub/sub. Routes events to the {WorkQueue} + # Subscriber for the store notifier. Routes events to the {WorkQueue} # by resolving message types or group IDs to reactor classes. # # Handles two events: - # - +'messages_appended'+ — value is comma-separated type strings; + # - +'messages_appended'+ — comma-separated type strings; # maps types to interested reactors and pushes them - # - +'reactor_resumed'+ — value is a consumer group ID; + # - +'reactor_resumed'+ — a consumer group ID; # looks up the reactor and pushes it directly - # - # @example - # queuer = NotificationQueuer.new(work_queue: queue, reactors: [OrderReactor]) - # backend.notifier.subscribe(queuer) class NotificationQueuer MESSAGES_APPENDED = 'messages_appended' REACTOR_RESUMED = 'reactor_resumed' @@ -82,29 +78,45 @@ def build_type_lookup(reactors) # @return [Hash{String => Class}] mapping from group_id to reactor class def build_group_id_lookup(reactors) reactors.each_with_object({}) do |reactor, lookup| - lookup[reactor.consumer_info.group_id] = reactor + lookup[reactor.group_id] = reactor end end end - # @return [Array] worker instances managed by this dispatcher + # @return [Array] worker instances managed by this dispatcher attr_reader :workers - # @param router [Router] the router providing async_reactors and backend + def self.spawn_into(task) + config = Sourced.config + dispatcher = Sourced::Dispatcher.new( + router: Sourced.router, + worker_count: config.worker_count, + batch_size: config.batch_size, + max_drain_rounds: config.max_drain_rounds, + catchup_interval: config.catchup_interval, + housekeeping_interval: config.housekeeping_interval, + claim_ttl_seconds: config.claim_ttl_seconds, + logger: config.logger + ).spawn_into(task) + end + + # @param router [Sourced::Router] the router providing reactors and store # @param worker_count [Integer] number of worker fibers to spawn (default 2) - # @param batch_size [Integer] messages per backend fetch (default 1) - # @param max_drain_rounds [Integer] max drain iterations before a worker - # re-enqueues a reactor (default 10) + # @param batch_size [Integer] max messages per claim (default 50) + # @param max_drain_rounds [Integer] max drain iterations before re-enqueue (default 10) # @param catchup_interval [Numeric] seconds between catch-up polls (default 5) - # @param work_queue [WorkQueue, nil] optional pre-built queue (useful for testing - # with a plain +Queue.new+ to avoid Async dependency) + # @param housekeeping_interval [Numeric] seconds between heartbeat/reap cycles (default 30) + # @param claim_ttl_seconds [Integer] stale claim age threshold in seconds (default 120) + # @param work_queue [WorkQueue, nil] optional pre-built queue (useful for testing) # @param logger [Object] logger instance def initialize( router:, worker_count: 2, - batch_size: 1, + batch_size: 50, max_drain_rounds: 10, catchup_interval: 5, + housekeeping_interval: 30, + claim_ttl_seconds: 120, work_queue: nil, logger: Sourced.config.logger ) @@ -114,37 +126,49 @@ def initialize( return if worker_count.zero? - reactors = router.async_reactors.select { |r| r.handled_messages.any? }.to_a + reactors = router.reactors.select { |r| r.handled_messages.any? }.to_a @work_queue = work_queue || WorkQueue.new(max_per_reactor: worker_count) @workers = worker_count.times.map do |i| Worker.new( work_queue: @work_queue, - router: router, + router:, name: "worker-#{i}", - batch_size: batch_size, - max_drain_rounds: max_drain_rounds, - logger: logger + batch_size:, + max_drain_rounds:, + logger: ) end notification_queuer = NotificationQueuer.new(work_queue: @work_queue, reactors: reactors) - @backend_notifier = router.backend.notifier - @backend_notifier.subscribe(notification_queuer) + @store_notifier = router.store.notifier + @store_notifier.subscribe(notification_queuer) @catchup_poller = CatchUpPoller.new( work_queue: @work_queue, - reactors: reactors, + reactors:, + interval: catchup_interval, + logger: + ) + + @scheduled_message_poller = ScheduledMessagePoller.new( + store: router.store, interval: catchup_interval, - logger: logger + logger: + ) + + @stale_claim_reaper = StaleClaimReaper.new( + store: router.store, + interval: housekeeping_interval, + ttl_seconds: claim_ttl_seconds, + worker_ids_provider: -> { @workers.map(&:name) }, + logger: ) end # Spawn all component fibers into the caller's task context. - # Spawns: backend notifier (LISTEN), catch-up poller, and N workers. - # - # Supports both the executor's Task (+#spawn+) and Async::Task (+#async+). + # Spawns: store notifier (e.g. PG LISTEN), catch-up poller, and N workers. # # @param task [Object] an executor task or Async::Task to spawn fibers into # @return [void] @@ -153,32 +177,40 @@ def spawn_into(task) s = task.respond_to?(:spawn) ? :spawn : :async - # Backend notifier (PG LISTEN fiber — no-op for non-PG) - task.send(s) { @backend_notifier.start } + # Store notifier (start — no-op for InlineNotifier) + task.send(s) { @store_notifier.start } # CatchUp poller task.send(s) { @catchup_poller.run } + # Scheduled message poller + task.send(s) { @scheduled_message_poller.run } + + # Stale claim reaper + task.send(s) { @stale_claim_reaper.run } + # Workers @workers.each do |w| task.send(s) { w.run } end + + self end # Stop all components and close the work queue. - # Stops in order: backend notifier, catch-up poller, workers, then - # pushes shutdown sentinels into the queue to unblock any waiting workers. # # @return [void] def stop return if @workers.empty? - @logger.info "Dispatcher: stopping #{@workers.size} workers" - @backend_notifier.stop + @logger.info "Sourced::Dispatcher: stopping #{@workers.size} workers" + @store_notifier.stop @catchup_poller.stop + @scheduled_message_poller.stop + @stale_claim_reaper.stop @workers.each(&:stop) @work_queue.close(@workers.size) - @logger.info 'Dispatcher: all components stopped' + @logger.info 'Sourced::Dispatcher: all components stopped' end end end diff --git a/lib/sourced/durable_workflow.rb b/lib/sourced/durable_workflow.rb index a01a1f4b..c66edf13 100644 --- a/lib/sourced/durable_workflow.rb +++ b/lib/sourced/durable_workflow.rb @@ -1,38 +1,64 @@ # frozen_string_literal: true +require 'securerandom' + module Sourced + # Durable workflow base class. + # + # A workflow instance is identified by a +workflow_id+ string, which doubles + # as the partition key. All lifecycle events (WorkflowStarted, StepStarted, + # StepFailed, StepComplete, ContextUpdated, WaitStarted, WaitEnded, + # WorkflowComplete, WorkflowFailed) carry +workflow_id+ as their first + # payload attribute so {Sourced::Message#extracted_keys} indexes them for + # partition queries. + # + # The step-memoisation mechanism (@lookup + catch(:halt)) + # allows +durable+ / +wait+ / +context+ / +execute+ to be re-entered safely. class DurableWorkflow extend Sourced::Consumer + include Sourced::Evolve + + partition_by :workflow_id UnknownMessageError = Class.new(StandardError) + # Stable hash-based key for a given (method, args) pair. def self.step_key(step_name, args) [step_name, args].hash.to_s end def self.inherited(child) + super + child.partition_by(:workflow_id) cname = child.name.to_s.gsub(/::/, '.') .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') .gsub(/([a-z\d])([A-Z])/, '\1_\2') - .tr("-", "_") + .tr('-', '_') .downcase child.const_set(:WorkflowStarted, Sourced::Event.define("#{cname}.workflow.started") do + attribute :workflow_id, String attribute :args, Sourced::Types::Array.default([].freeze) end) child.const_set(:ContextUpdated, Sourced::Event.define("#{cname}.context.updated") do + attribute :workflow_id, String attribute :context, Sourced::Types::Any end) child.const_set(:WorkflowComplete, Sourced::Event.define("#{cname}.workflow.complete") do + attribute :workflow_id, String attribute :output, Sourced::Types::Any end) - child.const_set(:WorkflowFailed, Sourced::Event.define("#{cname}.workflow.failed")) + child.const_set(:WorkflowFailed, Sourced::Event.define("#{cname}.workflow.failed") do + attribute :workflow_id, String + end) child.const_set(:StepStarted, Sourced::Event.define("#{cname}.step.started") do + attribute :workflow_id, String attribute :key, String attribute :step_name, Sourced::Types::Lax::Symbol attribute :args, Sourced::Types::Array.default([].freeze) end) child.const_set(:StepFailed, Sourced::Event.define("#{cname}.step.failed") do + attribute :workflow_id, String attribute :key, String attribute :step_name, Sourced::Types::Lax::Symbol attribute :error_message, String @@ -40,71 +66,46 @@ def self.inherited(child) attribute :backtrace, Sourced::Types::Array[String] end) child.const_set(:StepComplete, Sourced::Event.define("#{cname}.step.complete") do + attribute :workflow_id, String attribute :key, String attribute :step_name, Sourced::Types::Lax::Symbol attribute :output, Sourced::Types::Any end) child.const_set(:WaitStarted, Sourced::Event.define("#{cname}.wait.started") do + attribute :workflow_id, String attribute :count, Integer attribute :at, Sourced::Types::Forms::Time end) - child.const_set(:WaitEnded, Sourced::Event.define("#{cname}.wait.ended")) - end + child.const_set(:WaitEnded, Sourced::Event.define("#{cname}.wait.ended") do + attribute :workflow_id, String + end) - def self.handled_messages + # Register all event classes so: + # - Router claims them on our consumer group (`handled_messages`). + # - `context_for(workflow_id:)` builds OR conditions for the partition + # read (one per event type, via `Message.to_conditions`). [ - self::WorkflowStarted, - self::WorkflowComplete, - self::StepStarted, - self::StepFailed, - self::StepComplete, - self::WaitStarted, - self::WaitEnded - ] - end - - def self.handle(message, history:, logger: Sourced.config.logger) - from(history, logger:).__handle(message) - end - - class Waiter - attr_reader :stream_id, :instance - - def initialize(reactor, stream_id, backend: Sourced.config.backend, logger: Sourced.config.logger) - @reactor, @stream_id, @backend = reactor, stream_id, backend - @instance = @reactor.new(logger:) - @value = nil - end - - def wait - while instance.status != :complete && instance.status != :failed - sleep 0.1 - load - end - instance - end - - def load - history = @backend.read_stream(@stream_id) - instance.__from(history) + child::WorkflowStarted, child::ContextUpdated, child::WorkflowComplete, + child::WorkflowFailed, child::StepStarted, child::StepFailed, + child::StepComplete, child::WaitStarted, child::WaitEnded + ].each do |klass| + child.handled_messages_for_evolve << klass unless child.handled_messages_for_evolve.include?(klass) end end - def self.from(history, logger: nil) - new(logger:).__from(history) - end - - def self.execute(*args) - stream_id = "workflow-#{SecureRandom.uuid}" - evt = self::WorkflowStarted.parse(stream_id:, payload: { args: }) - Sourced.config.backend.append_next_to_stream(stream_id, evt) - Waiter.new(self, stream_id, backend: Sourced.config.backend) + # Message types this consumer claims. Same set as evolve types because + # every workflow event both advances state and re-triggers the workflow. + def self.handled_messages + handled_messages_for_evolve end + # Define the initial context hash. Block receives no arguments. def self.context(&block) define_method :initial_context, &block end + # Wrap a method so the runtime memoises its result across workflow + # re-entries. def self.durable(method_name, retries: nil) source_method = :"__durable_source_#{method_name}" alias_method source_method, method_name @@ -115,38 +116,37 @@ def self.durable(method_name, retries: nil) case cached&.status when :complete cached.output - when :started # ready to call. + when :started begin output = send(source_method, *args) - @new_events << self.class::StepComplete.parse( - stream_id: id, - payload: { key:, step_name: method_name, output: } + @new_events << self.class::StepComplete.new( + payload: { workflow_id: id, key:, step_name: method_name, output: } ) throw :halt rescue StandardError => e - # TODO: this catches NameError? - # Syntax errors should immediatly stop the workflow, not retry - @new_events << self.class::StepFailed.parse( - stream_id: id, - payload: { key:, step_name: method_name, error_message: e.inspect, error_class: e.class.to_s, backtrace: e.backtrace } + @new_events << self.class::StepFailed.new( + payload: { + workflow_id: id, + key:, + step_name: method_name, + error_message: e.inspect, + error_class: e.class.to_s, + backtrace: e.backtrace + } ) if retries && cached.attempts == retries - @new_events << self.class::WorkflowFailed.parse(stream_id: id) + @new_events << self.class::WorkflowFailed.new(payload: { workflow_id: id }) end - throw :halt end - when :failed # retry. Exponential backoff, etc - @new_events << self.class::StepStarted.parse( - stream_id: id, - payload: { key:, step_name: method_name, args: } + when :failed + @new_events << self.class::StepStarted.new( + payload: { workflow_id: id, key:, step_name: method_name, args: } ) - throw :halt - when nil # first call. Schedule StepStarted - @new_events << self.class::StepStarted.parse( - stream_id: id, - payload: { key:, step_name: method_name, args: } + when nil + @new_events << self.class::StepStarted.new( + payload: { workflow_id: id, key:, step_name: method_name, args: } ) throw :halt end @@ -176,12 +176,106 @@ def complete_with(output) end end - attr_reader :id, :seq, :context, :args, :output, :status + # Kick off a new workflow instance. Appends a WorkflowStarted event and + # returns a {Waiter} that can poll for completion. + # + # @param args [Array] positional args passed to the workflow's #execute + # @param store [Sourced::Store] defaults to Sourced.store + # @return [Waiter] + def self.execute(*args, store: Sourced.store) + workflow_id = "workflow-#{SecureRandom.uuid}" + evt = self::WorkflowStarted.new(payload: { workflow_id:, args: }) + store.append([evt]) + Waiter.new(self, workflow_id, store:) + end + + # Router entry point. Drops claim.messages from the read history (the + # router's +store.read(conditions)+ returns the full partition, including + # messages being claimed) and delegates to {.handle_batch}. + def self.handle_claim(claim, history:) + claim_positions = claim.messages.map { |m| m.position if m.respond_to?(:position) }.compact.to_set + prior = history.messages.reject { |m| m.respond_to?(:position) && claim_positions.include?(m.position) } + prior_history = ReadResult.new(messages: prior, guard: history.guard) + values = claim.partition_value.transform_keys(&:to_sym) + handle_batch(values, claim.messages, history: prior_history) + end + + # GWT-compatible entry point. +history.messages+ must be disjoint from + # +new_messages+ — the caller owns that distinction. + def self.handle_batch(partition_values, new_messages, history:, replaying: false) + workflow_id = partition_values[:workflow_id] + instance = new([workflow_id]) + instance.__replay(history.messages) + + each_with_partial_ack(new_messages) do |msg| + instance.__evolve(msg) + actions = instance.__handle(msg, guard: history.guard) + [actions, msg] + end + end + + # Direct handler used by unit tests and by {Waiter}. Mirrors + # {Sourced::DurableWorkflow.handle}: +history+ should already contain + # +message+ as its last element. + def self.handle(message, history:) + from(history).__handle(message) + end + + # Rebuild a workflow instance by replaying +history+. + def self.from(history) + new.__replay(history) + end + + # Load a workflow instance for +workflow_id+ from the store. + def self.load(workflow_id, store: Sourced.store) + _inst, _rr = Sourced.load(self, store:, workflow_id: workflow_id) + end - def initialize(logger: nil) - @id = nil - @seq = 0 - @logger = logger + # Polls the store for terminal workflow events. + class Waiter + attr_reader :workflow_id, :instance + + def initialize(klass, workflow_id, store: Sourced.store) + @klass = klass + @workflow_id = workflow_id + @store = store + @instance = klass.new([workflow_id]) + end + + def wait(timeout: nil) + deadline = timeout ? Time.now + timeout : nil + until @instance.status == :complete || @instance.status == :failed + raise 'DurableWorkflow wait timed out' if deadline && Time.now > deadline + + sleep 0.05 + load + end + @instance + end + + def load + handled_types = @klass.handled_messages_for_evolve.map(&:type).uniq + result = @store.read_partition({ workflow_id: @workflow_id }, handled_types:) + @instance = @klass.new([@workflow_id]) + @instance.__replay(result.messages) + @instance + end + end + + attr_reader :id, :context, :args, :output, :status + + # +partition_values+ may be: + # - an Array like +['wf-id']+ (from {.handle_claim}) + # - a Hash like +{ workflow_id: 'wf-id' }+ (from {Sourced.load}) + # - a String +'wf-id'+ + # - nil + def initialize(partition_values = nil) + @id = case partition_values + when Array then partition_values.first + when Hash then partition_values[:workflow_id] + when String then partition_values + else nil + end @status = :new @args = [] @output = nil @@ -189,27 +283,28 @@ def initialize(logger: nil) @new_events = [] @wait_count = 0 @waiters = [] - @initial_context = nil @context = initial_context end def initial_context = nil - def __from(history) - history.each do |event| - __evolve(event) - end + def __replay(history) + Array(history).each { |m| __evolve(m) } self end - def __evolve(event) - @id = event.stream_id - @seq = event.seq + # Override Sourced::Evolve#evolve so {Sourced.load} (which calls +instance.evolve+) + # applies workflow events via our manual dispatcher. + def evolve(messages) + __replay(messages) + end + def __evolve(event) case event when self.class::ContextUpdated @context = deep_dup(event.payload.context) when self.class::WorkflowStarted + @id ||= event.payload.workflow_id @args = event.payload.args @status = :started when self.class::WorkflowFailed @@ -233,18 +328,16 @@ def __evolve(event) end end - def __handle(message) - return Sourced::Actions::OK if @status == :complete || @status == :failed + # Decide the next action given +message+. State is assumed to already + # reflect +message+ (caller replayed it). + def __handle(message, guard: nil) + return Actions::OK if @status == :complete || @status == :failed if message.is_a?(self.class::WaitStarted) - evt = self.class::WaitEnded.parse(stream_id: id) - return Sourced::Actions::Schedule.new([evt], at: message.payload.at) + evt = self.class::WaitEnded.new(payload: { workflow_id: id }) + return Actions::Schedule.new([evt], at: message.payload.at) end - # Raise if @waiting ? - # we don't allow handling any new messages until wait has ended. - - # TODO: all this deep-duping is not efficient. @initial_context = deep_dup(@context) completed = false @@ -255,47 +348,48 @@ def __handle(message) completed = true end - # If any step updated context, even on failure - # make sure to log a ContextUpdated event to keep track of that state - @new_events << self.class::ContextUpdated.parse( - stream_id: id, - payload: { context: deep_dup(@context) } - ) if @context != @initial_context - - @new_events << self.class::WorkflowComplete.parse( - stream_id: id, - payload: { output: } - ) if completed - - last_seq = @seq - events = @new_events.map { |e| e.with(seq: last_seq += 1 )} - Sourced::Actions::AppendAfter.new(id, events) + if @context != @initial_context + @new_events << self.class::ContextUpdated.new( + payload: { workflow_id: id, context: deep_dup(@context) } + ) + end + + if completed + @new_events << self.class::WorkflowComplete.new( + payload: { workflow_id: id, output: } + ) + end + + events = @new_events + @new_events = [] + return Actions::OK if events.empty? + + Actions::Append.new(events, guard: guard) end private - attr_reader :logger - def wait(seconds) @wait_count += 1 - if @waiters[@wait_count] # we're already waiting for this method call - return seconds - else # first time we call this method - @new_events << self.class::WaitStarted.parse( - stream_id: id, - payload: { count: @wait_count, at: Time.now + seconds } + if @waiters[@wait_count] + seconds + else + @new_events << self.class::WaitStarted.new( + payload: { workflow_id: id, count: @wait_count, at: Time.now + seconds } ) - throw :halt end end - def deep_dup(hash) - return hash.dup unless hash.is_a?(Hash) - - hash.each.with_object({}) do |(k, v), new_hash| - new_hash[k] = deep_dup(v) + def deep_dup(value) + case value + when Hash + value.each.with_object({}) { |(k, v), h| h[k] = deep_dup(v) } + when Array + value.map { |v| deep_dup(v) } + else + value.dup rescue value end end end diff --git a/lib/sourced/evolve.rb b/lib/sourced/evolve.rb index af0a963e..f99a4c0c 100644 --- a/lib/sourced/evolve.rb +++ b/lib/sourced/evolve.rb @@ -1,94 +1,44 @@ # frozen_string_literal: true module Sourced - # This mixin provides an .event macro - # to register event handlers for a class - # These event handlers are "evolvers", ie. they evolve - # a piece of state based on events. - # More here: https://ismaelcelis.com/posts/decide-evolve-react-pattern-in-ruby/#2-evolve - # - # Example: - # - # class Projector - # include Sourced::Evolve - # - # state do - # { status: 'new' } - # end - # - # event SomethingHappened do |state, event| - # state[:status] = 'done' - # end - # end - # - # pr = Projector.new - # state = pr.evolve([SomethingHappened.new]) - # state[:status] # => 'done' - # - # From the outside, this mixin exposes .handled_messages_for_evolve - # - # .handled_messages_for_evolve() Array - # - # It also provides a .before_evolve and .evolve_all macros - # See comments in code for details. + # Evolve mixin for reactors. + # State evolution from a history of messages. + # State block receives partition values hash instead of stream id. module Evolve - PREFIX = 'evolution' - NOOP_HANDLER = ->(*_) { nil } + PREFIX = 'sourced_evolution' def self.included(base) super base.extend ClassMethods end - # Initialize in-memory state for this evolver. - # Override this method to provide a custom initial state. - # @return [Any] the initial state - def init_state(_id) + def init_state(_partition_values) nil end def state - @state ||= init_state(id) + @state ||= init_state(partition_values) end - # Override this in host class - def id = nil + def partition_values + @partition_values ||= {} + end - # Apply a list of events to a piece of state - # by running event handlers registered in this class - # via the .event macro. - # - # @param events [Array] - # @return [Object] - def evolve(events) - Array(events).each do |event| - method_name = Sourced.message_method_name(Evolve::PREFIX, event.class.to_s) - # We might be evolving old events in history - # even if we don't have handlers for them anymore - # we still need to increment seq - __update_on_evolve(event) - if respond_to?(method_name) - before_evolve(state, event) - send(method_name, state, event) - end + # Apply messages to state via registered handlers. + # Skips messages without a registered handler. + def evolve(messages) + Array(messages).each do |msg| + method_name = Sourced.message_method_name(PREFIX, msg.class.to_s) + send(method_name, state, msg) if respond_to?(method_name) end - state end - private def before_evolve(*_) - nil - end - - private def __update_on_evolve(event) - # Noop - end - module ClassMethods def inherited(subclass) super - handled_messages_for_evolve.each do |evt_type| - subclass.handled_messages_for_evolve << evt_type + handled_messages_for_evolve.each do |klass| + subclass.handled_messages_for_evolve << klass end end @@ -96,67 +46,15 @@ def handled_messages_for_evolve @handled_messages_for_evolve ||= [] end - # Define an initial state factory for this evolver. - # @example - # - # state do - # { status: 'new' } - # end - # + # Define initial state factory. Block receives partition values hash. def state(&blk) define_method(:init_state, &blk) end - # This module only accepts registering event handlers - # with qualified event classes - # Decider overrides this method to allow - # defining event handlers with symbols - # which are registered as Event classes in the decider namespace. - # @example - # - # event SomethingHappened do |state, event| - # state[:status] = 'done' - # end - # - # @param event_class [Sourced::Message] - # @return [void] - def event(event_class, &block) - unless event_class.is_a?(Class) && event_class < Sourced::Message - raise ArgumentError, - "Invalid argument #{event_class.inspect} for #{self}.event" - end - - handled_messages_for_evolve << event_class - block = NOOP_HANDLER unless block_given? - define_method(Sourced.message_method_name(Evolve::PREFIX, event_class.to_s), &block) - end - - # Run this block before any of the registered event handlers - # Example: - # before_evolve do |state, event| - # state.udpated_at = event.created_at - # end - def before_evolve(&block) - define_method(:before_evolve, &block) - end - - # Example: - # # With an Array of event types - # evolve_all [:event_type1, :event_type2] do |state, event| - # state.updated_at = event.created_at - # end - # - # # From another Evolver that responds to #handled_messages_for_evolve - # evolve_all CartAggregate do |state, event| - # state.updated_at = event.created_at - # end - # - # @param event_list [Array, #handled_messages_for_evolve() [Array}] - def evolve_all(event_list, &block) - event_list = event_list.handled_messages_for_evolve if event_list.respond_to?(:handled_messages_for_evolve) - event_list.each do |event_type| - event(event_type, &block) - end + # Register an evolve handler for a Sourced::Message subclass. + def evolve(message_class, &block) + handled_messages_for_evolve << message_class + define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) end end end diff --git a/lib/sourced/falcon/environment.rb b/lib/sourced/falcon/environment.rb index 22ff606a..f30a77a3 100644 --- a/lib/sourced/falcon/environment.rb +++ b/lib/sourced/falcon/environment.rb @@ -2,13 +2,15 @@ module Sourced module Falcon - # Environment mixin for configuring a combined Falcon web server + Sourced workers service. + # Environment mixin for configuring a combined Falcon web server + workers service. # - # Include this module in a Falcon service definition to get Sourced worker defaults - # alongside the standard Falcon server environment. + # Include this module in a Falcon service definition to get workers defaults + # alongside the standard Falcon server environment. All settings are read from + # {Sourced.config} — no per-service config methods needed. # - # Housekeeping settings are read from Sourced.config by default (configured in your - # boot/environment file). They can be overridden per-service in falcon.rb if needed. + # The Service automatically calls {Sourced.setup!} at the start of +run+ to + # re-establish database connections after Falcon forks (SQLite connections + # are not fork-safe). This replays the block passed to {Sourced.configure}. # # @example falcon.rb # #!/usr/bin/env falcon-host @@ -20,38 +22,11 @@ module Falcon # include Falcon::Environment::Rackup # # url "http://[::]:9292" - # sourced_worker_count 4 # end module Environment include ::Falcon::Environment::Server - def service_class - Sourced::Falcon::Service - end - - # Number of Sourced worker fibers to spawn per container process. - # @return [Integer] - def sourced_worker_count = Sourced.config.worker_count - - # Number of housekeeper fibers to spawn per container process. - # @return [Integer] - def sourced_housekeeping_count = Sourced.config.housekeeping_count - - # Seconds between housekeeper scheduling cycles. - # @return [Numeric] - def sourced_housekeeping_interval = Sourced.config.housekeeping_interval - - # Seconds between worker heartbeats. - # @return [Numeric] - def sourced_housekeeping_heartbeat_interval = Sourced.config.housekeeping_heartbeat_interval - - # Seconds before a claim is considered stale and can be reaped. - # @return [Numeric] - def sourced_housekeeping_claim_ttl_seconds = Sourced.config.housekeeping_claim_ttl_seconds - - # Number of messages to fetch per lock cycle for batch processing. - # @return [Integer] - def sourced_worker_batch_size = Sourced.config.worker_batch_size + def service_class = Sourced::Falcon::Service end end end diff --git a/lib/sourced/falcon/service.rb b/lib/sourced/falcon/service.rb index d5e099fc..0135d0a2 100644 --- a/lib/sourced/falcon/service.rb +++ b/lib/sourced/falcon/service.rb @@ -1,44 +1,26 @@ # frozen_string_literal: true require 'sourced/dispatcher' -require 'sourced/house_keeper' module Sourced module Falcon # A Falcon service that runs both the web server and Sourced background workers # as sibling fibers within the same Async reactor. # - # Uses a Dispatcher for signal-driven worker dispatch instead of blind polling. - # Lifecycle is managed by Falcon's container — no need for a separate Supervisor - # process or signal handling. + # Uses a Sourced::Dispatcher for signal-driven worker dispatch. The Dispatcher + # already embeds the StaleClaimReaper, so no separate HouseKeeper is needed + # (unlike {Sourced::Falcon::Service}). + # + # All configuration is read from {Sourced.config}. class Service < ::Falcon::Service::Server def run(instance, evaluator) - server = evaluator.make_server(@bound_endpoint) - - @dispatcher = Sourced::Dispatcher.new( - router: Sourced::Router, - worker_count: evaluator.sourced_worker_count, - batch_size: evaluator.sourced_worker_batch_size, - logger: Sourced.config.logger - ) + Sourced.setup! - housekeepers = evaluator.sourced_housekeeping_count.times.map do |i| - Sourced::HouseKeeper.new( - backend: Sourced.config.backend, - name: "falcon-hk-#{i}", - interval: evaluator.sourced_housekeeping_interval, - heartbeat_interval: evaluator.sourced_housekeeping_heartbeat_interval, - claim_ttl_seconds: evaluator.sourced_housekeeping_claim_ttl_seconds, - worker_ids_provider: -> { @dispatcher.workers.map(&:name) } - ) - end - - @sourced_housekeepers = housekeepers + server = evaluator.make_server(@bound_endpoint) Async do |task| server.run - housekeepers.each { |hk| task.async { hk.work } } - @dispatcher.spawn_into(task) + Sourced::Dispatcher.spawn_into(task) task.children.each(&:wait) end @@ -47,7 +29,6 @@ def run(instance, evaluator) def stop(...) @dispatcher&.stop - @sourced_housekeepers&.each(&:stop) super end end diff --git a/lib/sourced/handler.rb b/lib/sourced/handler.rb deleted file mode 100644 index 0123bc72..00000000 --- a/lib/sourced/handler.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module Handler - PREFIX = 'handle' - - def self.included(base) - base.send :extend, Consumer - base.send :extend, ClassMethods - end - - def handle(message, history:, replaying: false) - handler_name = Sourced.message_method_name(PREFIX, message.class.name) - return Actions::OK unless respond_to?(handler_name) - - args = {history:, replaying:} - expected_args = self.class.__args_lookup[handler_name] - handler_args = expected_args.each.with_object({}) do |key, memo| - memo[key] = args[key] - end - - send(handler_name, message, **handler_args) - end - - module ClassMethods - def handled_messages - @handled_messsages ||= [] - end - - def on(*args, &block) - case args - in [Symbol => msg_name, Hash => payload_schema] - __register_named_message_handler(msg_name, payload_schema, &block) - in [Symbol => msg_name] - __register_named_message_handler(msg_name, &block) - in [Class => msg_type] if msg_type < Sourced::Message - __register_class_message_handler(msg_type, &block) - else - args.each do |arg| - on(*arg, &block) - end - end - end - - def handle(message, history: [], replaying: false) - results = new.handle(message, history:, replaying:) - Actions.build_for(results) - end - - def __args_lookup - @__args_lookup ||= {} - end - - private - - def __register_named_message_handler(msg_name, payload_schema = nil, &block) - msg_class = Sourced::Message.define(__message_type(msg_name), payload_schema:) - klass_name = msg_name.to_s.split('_').map(&:capitalize).join - const_set(klass_name, msg_class) - __register_class_message_handler(msg_class, &block) - end - - def __register_class_message_handler(msg_type, &block) - handled_messages << msg_type - handler_name = Sourced.message_method_name(PREFIX, msg_type.name) - __args_lookup[handler_name] = Sourced::Injector.resolve_args(block) - define_method(handler_name, &block) - end - - # TODO: these are in Actor too - def __message_type(msg_name) - [__message_type_prefix, msg_name].join('.').downcase - end - - def message_namespace - Types::ModuleToMessageType.parse(name.to_s) - end - - def __message_type_prefix - @__message_type_prefix ||= message_namespace - end - end - end -end diff --git a/lib/sourced/house_keeper.rb b/lib/sourced/house_keeper.rb deleted file mode 100644 index b1b6a938..00000000 --- a/lib/sourced/house_keeper.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Sourced - class HouseKeeper - attr_reader :name - - def initialize( - logger: Sourced.config.logger, - interval: 3, - heartbeat_interval: 5, - claim_ttl_seconds: 120, - backend:, - name:, - worker_ids_provider: nil - ) - @logger = logger - @interval = interval - @heartbeat_interval = heartbeat_interval - @claim_ttl_seconds = claim_ttl_seconds - @backend = backend - @name = name - @running = false - @worker_ids_provider = worker_ids_provider || -> { [] } - end - - def work - @running = true - - # On start wait for a random period - # in order to space out multiple house-keepers - sleep rand(5) - logger.info "HouseKeeper #{name}: starting" - - # Reap stale claims on startup (from previous runs where workers were killed) - released = backend.release_stale_claims(ttl_seconds: @claim_ttl_seconds) - logger.info "HouseKeeper #{name}: startup cleanup released #{released} stale claims" if released && released > 0 - - last_heartbeat = Time.at(0) - last_stale_reaping = Time.now - while @running - sleep @interval - - # 1) Schedule due messages - schcount = backend.update_schedule! - logger.info "HouseKeeper #{name}: appended #{schcount} scheduled messages" if schcount > 0 - - now = Time.now - - # 2) Heartbeat alive workers (bulk upsert) - if now - last_heartbeat >= @heartbeat_interval - ids = Array(@worker_ids_provider.call).uniq - hb = backend.worker_heartbeat(ids) - logger.debug "HouseKeeper #{name}: heartbeated #{hb} workers" if hb > 0 - last_heartbeat = now - end - - # 3) Reap stale claims (only every claim_ttl_seconds, since claims can't be stale until then) - if now - last_stale_reaping >= @claim_ttl_seconds - released = backend.release_stale_claims(ttl_seconds: @claim_ttl_seconds) - logger.info "HouseKeeper #{name}: released #{released} stale claims" if released && released > 0 - last_stale_reaping = now - end - end - - logger.info "HouseKeeper #{name}: stopped" - end - - def stop - @running = false - end - - private - - attr_reader :logger, :backend, :interval - end -end diff --git a/lib/sourced/installer.rb b/lib/sourced/installer.rb new file mode 100644 index 00000000..f1675abc --- /dev/null +++ b/lib/sourced/installer.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'sequel' +require 'sequel/extensions/migration' +require 'erb' + +module Sourced + class Installer + TABLE_SUFFIXES = %i[messages key_pairs message_key_pairs scheduled_messages consumer_groups offsets offset_key_pairs workers].freeze + + attr_reader :messages_table, :key_pairs_table, :message_key_pairs_table, + :scheduled_messages_table, :consumer_groups_table, :offsets_table, + :offset_key_pairs_table, :workers_table + + def initialize(db, logger:, prefix: 'sourced', migration_template: '001_create_sourced_tables.rb.erb') + raise ArgumentError, "invalid prefix: #{prefix}" unless prefix.match?(/\A[a-zA-Z_]\w*\z/) + + @db = db + @logger = logger + @prefix = prefix + @migration_template = migration_template + + TABLE_SUFFIXES.each do |suffix| + instance_variable_set(:"@#{suffix}_table", :"#{prefix}_#{suffix}") + end + end + + # Eval the rendered migration and apply :up directly. + def install + migration.apply(db, :up) + logger.info("Sourced tables installed (prefix: #{prefix})") + end + + # Check that all expected tables exist. + def installed? + all_table_names.all? { |t| db.table_exists?(t) } + end + + # Apply :down on the migration to drop tables. + def uninstall + raise 'Not in test environment' unless ENV['ENVIRONMENT'] == 'test' + + migration.apply(db, :down) + end + + # Render the migration to a file for use with the host app's Sequel::Migrator. + # + # installer.copy_migration_to("db/migrations") + # installer.copy_migration_to { "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_ccc_tables.rb" } + # + def copy_migration_to(dir = nil, &block) + path = block ? block.call : File.join(dir, '001_create_sourced_tables.rb') + File.write(path, rendered_migration) + logger.info("Copied Sourced migration to #{path}") + path + end + + private + + attr_reader :db, :logger, :prefix + + def migration + @migration ||= eval(rendered_migration) # rubocop:disable Security/Eval + end + + def rendered_migration + @rendered_migration ||= begin + template_path = File.join(__dir__, 'migrations', @migration_template) + ERB.new(File.read(template_path)).result(binding) + end + end + + # Returns actual symbols for use in installed? checks. + def all_table_names + TABLE_SUFFIXES.map { |suffix| instance_variable_get(:"@#{suffix}_table") } + end + end +end diff --git a/lib/sourced/message.rb b/lib/sourced/message.rb index b5d7789d..bd9d66db 100644 --- a/lib/sourced/message.rb +++ b/lib/sourced/message.rb @@ -2,84 +2,66 @@ require 'sourced/types' -# A superclass and registry to define event types -# for example for an event-driven or event-sourced system. -# All events have an "envelope" set of attributes, -# including unique ID, stream_id, type, timestamp, causation ID, -# event subclasses have a type string (ex. 'users.name.updated') and an optional payload -# This class provides a `.define` method to create new event types with a type and optional payload struct, -# a `.from` method to instantiate the correct subclass from a hash, ex. when deserializing from JSON or a web request. -# and a `#follow` method to produce new events based on a previous event's envelope, where the #causation_id and #correlation_id -# are set to the parent event -# @example -# -# # Define event struct with type and payload -# UserCreated = Message.define('users.created') do -# attribute :name, Types::String -# attribute :email, Types::Email -# end -# -# # Instantiate a full event with .new -# user_created = UserCreated.new(stream_id: 'user-1', payload: { name: 'Joe', email: '...' }) -# -# # Use the `.from(Hash) => Message` factory to lookup event class by `type` and produce the right instance -# user_created = Message.from(type: 'users.created', stream_id: 'user-1', payload: { name: 'Joe', email: '...' }) -# -# # Use #follow(payload Hash) => Message to produce events following a command or parent event -# create_user = CreateUser.new(...) -# user_created = create_user.follow(UserCreated, name: 'Joe', email: '...') -# user_created.causation_id == create_user.id -# user_created.correlation_id == create_user.correlation_id -# user_created.stream_id == create_user.stream_id -# -# ## Message registries -# Each Message class has its own registry of sub-classes. -# You can use the top-level Sourced::Message.from(hash) to instantiate all message types. -# You can also scope the lookup by sub-class. -# -# @example -# -# class PublicCommand < Sourced::Message; end -# -# DoSomething = PublicCommand.define('commands.do_something') -# -# # Use .from scoped to PublicCommand subclass -# # to ensure that only PublicCommand subclasses are accessible. -# cmd = PublicCommand.from(type: 'commands.do_something', payload: { ... }) -# -# ## JSON Schemas -# Plumb data structs support `.to_json_schema`, so you can document all events in the registry with something like -# -# Message.subclasses.map(&:to_json_schema) -# module Sourced - UnknownMessageError = Class.new(ArgumentError) - PastMessageDateError = Class.new(ArgumentError) - + # A query condition for reading messages from the store. + # Matches on (message_type AND all attrs key-value pairs). + # Multiple conditions are OR'd when passed to {Store#read}. + QueryCondition = Data.define(:message_type, :attrs) + + # Returned by {Store#read} and {Store#claim_next} for optimistic concurrency. + # Pass to {Store#append} via +guard:+ to detect conflicting writes. + ConsistencyGuard = Data.define(:conditions, :last_position) + + # Base message class. Messages have no stream_id or seq — they go into a + # flat, globally-ordered log. + # + # Supports +causation_id+ and +correlation_id+ for tracing causal chains. + # + # Define message types via {.define}: + # + # CourseCreated = Sourced::Message.define('course.created') do + # attribute :course_name, String + # end + # class Message < Types::Data + EMPTY_ARRAY = [].freeze + attribute :id, Types::AutoUUID - attribute :stream_id, Types::String.present attribute :type, Types::String.present - attribute :created_at, Types::Forms::Time.default { Time.now } #Types::JSON::AutoUTCTime attribute? :causation_id, Types::UUID::V4 attribute? :correlation_id, Types::UUID::V4 - attribute :seq, Types::Integer.default(1) + attribute :created_at, Types::Forms::Time.default { Time.now } attribute :metadata, Types::Hash.default(Plumb::BLANK_HASH) attribute :payload, Types::Static[nil] + # Lookup table mapping type strings to message subclasses. + # Separate from {Sourced::Message}'s registry. class Registry + # @param message_class [Class] the root message class for this registry def initialize(message_class) @message_class = message_class @lookup = {} end + # @return [Array] registered type strings def keys = @lookup.keys + + # @return [Array] direct subclasses of the root message class def subclasses = message_class.subclasses + # Register a message class under a type string. + # + # @param key [String] message type string + # @param klass [Class] message subclass def []=(key, klass) @lookup[key] = klass end + # Look up a message class by type string. + # Searches this registry first, then recurses into subclass registries. + # + # @param key [String] message type string + # @return [Class, nil] def [](key) klass = lookup[key] return klass if klass @@ -91,8 +73,15 @@ def [](key) nil end - def inspect - %(<#{self.class}:#{object_id} #{lookup.size} keys, #{subclasses.size} child registries>) + # All registered message classes across this registry and subclass registries. + # + # @return [Enumerator] if no block given + # @yield [Class] each registered message class + def all(&block) + return enum_for(:all) unless block + + lookup.each_value(&block) + subclasses.each { |c| c.registry.all(&block) } end private @@ -100,124 +89,166 @@ def inspect attr_reader :lookup, :message_class end + # @return [Registry] the message type registry for this class def self.registry @registry ||= Registry.new(self) end + # Base class for typed message payloads. class Payload < Types::Data + # @param key [Symbol] attribute name + # @return [Object] attribute value def [](key) = attributes[key] + + # @see Hash#fetch def fetch(...) = to_h.fetch(...) end - def self.define(type_str, payload_schema: nil, &payload_block) + # Define a new message type. Registers it in the {.registry} and + # optionally defines a typed payload. + # + # @param type_str [String] unique message type identifier (e.g. 'course.created') + # @yield optional block to define payload attributes via +attribute+ DSL + # @return [Class] the new message subclass + # + # @example + # UserJoined = Sourced::Message.define('user.joined') do + # attribute :course_name, String + # attribute :user_id, String + # end + def self.define(type_str, &payload_block) type_str.freeze unless type_str.frozen? - if registry[type_str] - Sourced.config.logger.warn("Message '#{type_str}' already defined") - end registry[type_str] = Class.new(self) do def self.node_name = :data define_singleton_method(:type) { type_str } attribute :type, Types::Static[type_str] - if payload_schema - const_set(:Payload, Payload[payload_schema]) - attribute :payload, self::Payload - elsif block_given? - const_set(:Payload, Class.new(Payload, &payload_block)) - attribute :payload, self::Payload + if block_given? + payload_class = Class.new(Payload, &payload_block) + const_set(:Payload, payload_class) + attribute :payload, payload_class + names = payload_class._schema.to_h.keys.map(&:to_sym).freeze + define_singleton_method(:payload_attribute_names) { names } end end end + # Instantiate the correct message subclass from a hash with a +:type+ key. + # + # @param attrs [Hash] must include +:type+ matching a registered type string + # @return [Message] instance of the appropriate subclass + # @raise [Sourced::UnknownMessageError] if the type string is not registered def self.from(attrs) klass = registry[attrs[:type]] - raise UnknownMessageError, "Unknown event type: #{attrs[:type]}" unless klass + raise Sourced::UnknownMessageError, "Unknown message type: #{attrs[:type]}" unless klass klass.new(attrs) end - def self.build(stream_id, payload = nil) - attrs = {stream_id:} - attrs[:payload] = payload if payload - parse(attrs) - end - def initialize(attrs = {}) - unless attrs[:payload] - attrs = attrs.merge(payload: {}) - end + attrs = attrs.merge(payload: {}) unless attrs[:payload] super(attrs) end - def with_metadata(meta = {}) - return self if meta.empty? + # Identity implementation of the +to_message+ contract — see + # {.===} and {Sourced::PositionedMessage#to_message}. + def to_message = self - attrs = metadata.merge(meta) - with(metadata: attrs) - end + # Make +case/when+ transparent to {Sourced::PositionedMessage} (or any + # wrapper implementing +#to_message+). Ruby's default +Module#===+ + # is implemented in C and ignores +is_a?+ overrides, so wrapped + # messages would otherwise fall through the +else+ branch. + def self.===(other) + return true if super + return false unless other.respond_to?(:to_message) - def follow(event_class, payload_attrs = nil) - follow_with_attributes( - event_class, - payload: payload_attrs - ) + unwrapped = other.to_message + !unwrapped.equal?(other) && super(unwrapped) end - def follow_with_seq(event_class, seq, payload_attrs = nil) - follow_with_attributes( - event_class, - attrs: { seq: }, - payload: payload_attrs - ) + def with_metadata(meta = {}) + return self if meta.empty? + + with(metadata: metadata.merge(meta)) end - def follow_with_stream_id(event_class, stream_id, payload_attrs = nil) - follow_with_attributes( - event_class, - attrs: { stream_id: }, - payload: payload_attrs - ) + def with_payload(attrs = {}) + hash = to_h + (hash[:payload] ||= {}).merge!(attrs) + self.class.new(hash) end - def follow_with_attributes(event_class, attrs: {}, payload: nil, metadata: nil) - meta = self.metadata - meta = meta.merge(metadata) if metadata - attrs = { stream_id:, causation_id: id, correlation_id:, metadata: meta }.merge(attrs) - attrs[:payload] = payload.to_h if payload - event_class.parse(attrs) + def at(datetime) + if datetime < created_at + raise Sourced::PastMessageDateError, "Message #{type} can't be delayed to a date in the past" + end + + with(created_at: datetime) end + # Set causation and correlation IDs on another message, establishing + # a causal link from this message to +message+. Merges metadata. + # + # @param message [Message] the message to correlate + # @return [Message] a copy of +message+ with causation/correlation set + # + # @example + # caused = source_event.correlate(SomeCommand.new(payload: { ... })) + # caused.causation_id # => source_event.id + # caused.correlation_id # => source_event.correlation_id def correlate(message) attrs = { causation_id: id, - correlation_id:, + correlation_id: correlation_id, metadata: metadata.merge(message.metadata || Plumb::BLANK_HASH) } message.with(attrs) end - # A copy of a message with a new stream_id - # @param stream_id [String, #stream_id] - # @return [Message] - def to(stream_id) - stream_id = stream_id.stream_id if stream_id.respond_to?(:stream_id) - with(stream_id:) + # Returns the declared payload attribute names for this message class. + # Subclasses created via {.define} override this with a cached frozen array. + # + # @return [Array] attribute names (e.g. +[:course_name, :user_id]+) + def self.payload_attribute_names = EMPTY_ARRAY + + # Build a {QueryCondition} for the intersection of this message's declared + # attributes and the given key-value pairs. Attributes not declared on this + # message class are silently ignored. Returns an array with a single condition + # containing all matching attrs, or an empty array if none match. + # + # @param attrs [Hash{Symbol => String}] partition attribute values + # @return [Array] + # + # @example + # CourseCreated.to_conditions(course_name: 'Algebra', user_id: 'joe') + # # => [QueryCondition('course.created', { course_name: 'Algebra' })] + # # user_id ignored — CourseCreated doesn't declare it + def self.to_conditions(**attrs) + supported = payload_attribute_names + matched = attrs.select { |key, _| supported.include?(key) } + .transform_values(&:to_s) + return [] if matched.empty? + + [QueryCondition.new(message_type: type, attrs: matched)] end - def at(datetime) - if datetime < created_at - raise PastMessageDateError, "Message #{type} can't be delayed to a date in the past" - end - with(created_at: datetime) - end + # Auto-extract key-value pairs from all top-level payload attributes. + # Used by {Store#append} to index messages for querying. + # + # @return [Array] pairs of [name, value], skipping nils + def extracted_keys + return [] unless payload - def to_json(*) - to_h.to_json(*) + payload.to_h.filter_map { |k, v| + [k.to_s, v.to_s] unless v.nil? + } end private + # Hook called by Plumb after schema parsing, when +:id+ has been resolved. + # Defaults +causation_id+ and +correlation_id+ to the message's own +id+. def prepare_attributes(attrs) attrs[:correlation_id] = attrs[:id] unless attrs[:correlation_id] attrs[:causation_id] = attrs[:id] unless attrs[:causation_id] diff --git a/lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb b/lib/sourced/migrations/001_create_sourced_tables.rb.erb similarity index 100% rename from lib/sourced/ccc/migrations/001_create_ccc_tables.rb.erb rename to lib/sourced/migrations/001_create_sourced_tables.rb.erb diff --git a/lib/sourced/projector.rb b/lib/sourced/projector.rb index a8f329f1..a62541a6 100644 --- a/lib/sourced/projector.rb +++ b/lib/sourced/projector.rb @@ -1,253 +1,84 @@ # frozen_string_literal: true module Sourced - # Projectors react to events - # and update views of current state somewhere (a DB, files, etc) + # Reactor base class for Sourced read-model projectors. class Projector - include React - include Evolve - include Sync - extend Consumer - - REACTION_WITH_STATE_PREFIX = 'reaction_with_state' - BLANK_ARRAY = [].freeze + include Sourced::Evolve + include Sourced::React + include Sourced::Sync + extend Sourced::Consumer class << self - # The Reactor interface - def handled_messages = handled_messages_for_evolve - - # Override this check defined in React mixin - private def __validate_message_for_reaction!(event_class) - if event_class && !handled_messages_for_evolve.include?(event_class) - raise ArgumentError, '.reaction only works with event types handled by this class via .event(event_type)' - end - end - - # The Ractor Interface - # @param message [Sourced::Message] - # @option replaying [Boolean] - # @option history [Enumerable] - # @return [Array] - def handle(message, replaying:, history: BLANK_ARRAY) - new(id: identity_from(message)).handle(message, replaying:, history:) - end - - # Override this in subclasses - # to make an actor take its @id from an arbitrary - # field in the message + # Projectors claim events they evolve from + events they react to. # - # @param message [Sourced::Message] - # @return [Object] - def identity_from(message) = message.stream_id + # @return [Array] evolved and reacted-to message classes + def handled_messages + (handled_messages_for_evolve + handled_messages_for_react).uniq + end private - # Shared batch finalization for both StateStored and EventSourced projectors. - # Runs reaction handlers first (which may issue 3rd party requests), - # then builds deferred sync actions last (executed in backend DB transaction). - # - # Unlike Actors (which create a new instance per message), Projectors use an - # evolve-all-sync-once optimization: all batch messages are evolved into a - # single instance's state before reactions run. This means on partial failure, - # the sync must be rebuilt for only the successfully processed messages. - # The block is called with the partial message list to produce correct sync_actions - # (e.g. StateStored rebuilds a fresh instance with partial evolve). - # - # @param instance [Projector] the projector instance (already evolved) - # @param batch [Array<[Message, Boolean]>] original batch pairs - # @param messages [Array] extracted messages from batch - # @param all_replaying [Boolean] whether all messages in the batch are replaying - # @yield [partial_messages] called on reaction error to build partial sync_actions - # @yieldparam partial_messages [Array] messages successfully processed - # @yieldreturn [Array] sync actions for partial state - # @return [Array<[actions, source_message]>] action pairs - def sync_and_react(instance, batch, messages, all_replaying, &on_partial_sync) - reaction_pairs = if all_replaying - BLANK_ARRAY + def build_action_pairs(instance, messages, replaying:) + sync_actions = instance.collect_actions( + state: instance.state, messages: messages, replaying: replaying + ) + + reaction_pairs = if replaying + [] else - each_with_partial_ack(batch) do |msg, replaying| - next if replaying + each_with_partial_ack(messages) do |msg| next unless instance.reacts_to?(msg) - - reaction_cmds = instance.react(msg) - reaction_cmds.any? ? [Actions.build_for(reaction_cmds), msg] : nil + reaction_msgs = Array(instance.react(msg)) + actions = Actions.build_for(reaction_msgs) + actions.any? ? [actions, msg] : nil end end - # All reactions succeeded. Build deferred sync for all messages (runs last in backend transaction). - sync_actions = instance.sync_actions_with( - state: instance.state, events: messages, replaying: all_replaying - ) - reaction_pairs + [[sync_actions, messages.last]] - rescue PartialBatchError => e - # Augment partial reaction results with sync for the successfully processed messages. - fail_idx = messages.index { |m| m.id == e.failed_message.id } - partial_messages = messages[0...fail_idx] - if on_partial_sync && partial_messages.any? - sync_actions = on_partial_sync.call(partial_messages) - e.action_pairs << [sync_actions, partial_messages.last] if sync_actions.any? - end - raise end end - attr_reader :id, :seq - - def initialize(id:, logger: Sourced.config.logger) - @id = id - @seq = 0 - @logger = logger - end + attr_reader :partition_values - def inspect - %(<#{self.class} id:#{id} seq:#{seq}>) + # @param partition_values [Hash{Symbol => String}] partition key-value pairs + def initialize(partition_values = {}) + @partition_values = partition_values end - def handle(message, replaying:) - raise NotImplementedError, 'implement me in subclasses' - end - - private - - attr_reader :logger - - # Override Evolve#__update_on_evolve - def __update_on_evolve(event) - @seq = event.seq - end - - # A StateStored projector fetches initial state from - # storage somewhere (DB, files, API) - # And then after reacting to events and updating state, - # it can save it back to the same or different storage. - # @example - # - # class CartListings < Sourced::Projector::StateStored - # # Fetch listing record from DB, or new one. - # state do |id| - # CartListing.find_or_initialize(id) - # end - # - # # Evolve listing record from events - # evolve Carts::ItemAdded do |listing, event| - # listing.total += event.payload.price - # end - # - # # Sync listing record back to DB - # sync do |state:, events:, replaying:| - # state.save! - # end - # end + # Projector variant that evolves only the claimed messages on top of stored state. class StateStored < self class << self - # State-stored version doesn't load :history - def handle(message, replaying: false) - new(id: identity_from(message)).handle(message, replaying:) - end - - # Optimized batch processing: one state load, one sync for entire batch. - # On partial failure, rebuilds a fresh instance with only the successfully - # processed messages so the sync persists correct partial state. - # @param batch [Array<[Message, Boolean]>] array of [message, replaying] pairs - # @return [Array<[actions, source_message]>] action pairs - def handle_batch(batch) - first_msg, _ = batch.first - instance = new(id: identity_from(first_msg)) - messages = batch.map(&:first) - all_replaying = batch.all? { |_, r| r } - - instance.state - instance.evolve(messages) - sync_and_react(instance, batch, messages, all_replaying) do |partial_messages| - partial = new(id: identity_from(first_msg)) - partial.state - partial.evolve(partial_messages) - partial.sync_actions_with(state: partial.state, events: partial_messages, replaying: all_replaying) - end + def handle_batch(partition_values, new_messages, history: nil, replaying: false) + instance = new(partition_values) + instance.evolve(new_messages) + build_action_pairs(instance, new_messages, replaying: replaying) end - end - - def handle(message, replaying:) - # Load state from storage - state - # Evolve new message - evolve(message) - # Collect sync actions - actions = sync_actions_with(state:, events: [message], replaying:) - # Replaying? Just return sync action - return actions if replaying - # Not replaying. Also run reactions - if reacts_to?(message) - actions += Actions.build_for(react(message)) + # @param claim [ClaimResult] claimed partition batch + # @return [Array, PositionedMessage)>] action/source pairs + def handle_claim(claim) + values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } + handle_batch(values, claim.messages, replaying: claim.replaying) end - - actions end end - # An EventSourced projector fetches initial state from - # past events in the event store. - # And then after reacting to events and updating state, - # it can save it to a DB table, a file, etc. - # @example - # - # class CartListings < Sourced::Projector::EventSourced - # # Initial in-memory state - # state do |id| - # { id:, total: 0 } - # end - # - # # Evolve listing record from events - # evolve Carts::ItemAdded do |listing, event| - # listing[:total] += event.payload.price - # end - # - # # Sync listing record to a file - # sync do |state:, events:, replaying:| - # File.write("/listings/#{state[:id]}.json", JSON.dump(state)) - # end - # end + # Projector variant that rebuilds state from full history each time. class EventSourced < self - BLANK_HISTORY = [].freeze - class << self - # Optimized batch processing: one history evolve, one sync for entire batch. - # On partial failure, reuses the already-evolved instance for sync since - # EventSourced state is rebuilt from history on every invocation (idempotent). - # @param batch [Array<[Message, Boolean]>] array of [message, replaying] pairs - # @param history [Array] full stream history - # @return [Array<[actions, source_message]>] action pairs - def handle_batch(batch, history: BLANK_HISTORY) - first_msg, _ = batch.first - instance = new(id: identity_from(first_msg)) - messages = batch.map(&:first) - all_replaying = batch.all? { |_, r| r } - - instance.evolve(history) - sync_and_react(instance, batch, messages, all_replaying) do |partial_messages| - # EventSourced state is rebuilt from history, so sync is idempotent. - # Reuse the already-evolved instance but sync only partial_messages. - instance.sync_actions_with(state: instance.state, events: partial_messages, replaying: all_replaying) - end + def handle_batch(partition_values, new_messages, history:, replaying: false) + instance = new(partition_values) + instance.evolve(history.messages) + build_action_pairs(instance, new_messages, replaying: replaying) end - end - def handle(message, replaying:, history:) - # Evolve new message from history - evolve(history) - # Collect sync actions - actions = sync_actions_with(state:, events: [message], replaying:) - # Replaying? Just return sync action - return actions if replaying - - # Not replaying. Also run reactions - if reacts_to?(message) - actions += Actions.build_for(react(message)) + # @param claim [ClaimResult] claimed partition batch + # @param history [ReadResult] full partition history + # @return [Array, PositionedMessage)>] action/source pairs + def handle_claim(claim, history:) + values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } + handle_batch(values, claim.messages, history:, replaying: claim.replaying) end - - actions end end end diff --git a/lib/sourced/pubsub/pg.rb b/lib/sourced/pubsub/pg.rb deleted file mode 100644 index ea23b6b5..00000000 --- a/lib/sourced/pubsub/pg.rb +++ /dev/null @@ -1,253 +0,0 @@ -# frozen_string_literal: true - -require 'sequel' -require 'thread' -require 'json' - -module Sourced - module PubSub - # a PubSub implementation using Postgres' LISTEN/NOTIFY - # - # This class provides a publish-subscribe mechanism using Postgres LISTEN/NOTIFY, - # with connection pooling - each channel name reuses a single database - # listener per process, allowing multiple subscribers to share the same listener thread / fiber. - # Relies on Sourced's configured Executor to use Threads or Fibers for concurrency - # - # @example Publishing a message - # pubsub = Sourced.config.pubsub - # event = MyEvent.new(stream_id: '111', payload: 'hello') - # pubsub.publish('my_channel', event) - # - # @example Subscribing to messages - # pubsub = Sourced.config.pubsub - # channel = pubsub.subscribe('my_channel') - # channel.start do |event, ch| - # case event - # when MyEvent - # puts "Received: #{event}" - # end - # end - # - # @example Multiple subscribers to the same channel - # # Both threads listen to the same channel but receive all messages - # Thread.new do - # ch1 = pubsub.subscribe('events') - # ch1.start { |event| puts "Subscriber 1: #{event}" } - # end - # - # Thread.new do - # ch2 = pubsub.subscribe('events') - # ch2.start { |event| puts "Subscriber 2: #{event}" } - # end - # - class PG - # Listener holds a single DB connection per channel name/process - # Channel instances with the same name instantiated in different threads/fibers but the same process - # share a Listener instance - # The Listener receives messages and dispatches them to all channels. - class Listener - # @param db [Sequel::Database] the database connection for listening - # @param channel_name [String] the name of the Postgres channel to listen on - # @param timeout [Integer] the timeout in seconds for listen operations (default: 2) - # @param logger [Logger] - def initialize(db:, channel_name:, timeout: 2, logger:) - @db = db - @channel_name = channel_name - @logger = logger - @channels = {} - @timeout = timeout - @queue = Sourced.config.executor.new_queue - @running = true - @info = "[#{[self.class.name, @channel_name, Process.pid, object_id].join(' ')}]" - end - - # Subscribe a channel to this listener - # - # @param channel [Channel] the channel to subscribe - # @return [self] - def subscribe(channel) - start - @queue << [:subscribe, channel] - self - end - - # Unsubscribe a channel from this listener - # - # @param channel [Channel] the channel to unsubscribe - # @return [self] - def unsubscribe(channel) - @queue << [:unsubscribe, channel] - self - end - - # Start the listener threads for database listening and message dispatching - # - # @return [self] - def start - return if @control - - @control = Sourced.config.executor.start(wait: false) do |t| - t.spawn do - while (msg = @queue.pop) - case msg - in :stop - @running = false - # Stop all channels? - @logger.info "#{@info} stopping" - @channels.values.each(&:stop) - in [:unsubscribe, channel] - if @channels.delete(channel.object_id) - @logger.info { "#{@info} unsubscribe channel #{channel.object_id}" } - if @channels.empty? - @logger.info { "#{@info} all channels unsubscribed." } - end - end - in [:subscribe, channel] - @logger.info { "#{@info} subscribe channel #{channel.object_id}" } - @channels[channel.object_id] ||= channel - in [:dispatch, message] - @channels.values.each { |ch| ch << message } - end - end - - @logger.info "#{@info} Stopped" - end - - t.spawn do - @db.listen(@channel_name, timeout: @timeout, loop: true) do |_channel, _pid, payload| - break unless @running - - message = parse(payload) - @queue << [:dispatch, message] - end - end - end - - @logger.info { "#{@info} Started" } - end - - # Stop the listener threads and unsubscribe all channels - # - # @return [self] - def stop - @queue << :stop - @control&.wait - @control = nil - @logger.info "#{@info} Stopped" - self - end - - private def parse(payload) - data = JSON.parse(payload, symbolize_names: true) - Sourced::Message.from(data) - end - end - - # Initialize a new PubSub instance with database and logger - # - # @param db [Sequel::Database] the database connection for publishing and listening - # @param logger [Logger] the logger instance for recording events - def initialize(db:, logger:) - @db = db - @logger = logger - @mutex = Mutex.new - @listeners ||= {} - end - - # Subscribe to messages on a channel - # - # Creates or reuses a listener for the given channel name and returns a new channel object - # that can be used to receive messages. Multiple subscribers to the same channel share a - # single database listener per process. - # - # @param channel_name [String] the name of the channel to subscribe to - # @return [Channel] a new channel object for receiving messages - def subscribe(channel_name) - @mutex.synchronize do - listener = @listeners[channel_name] ||= Listener.new(db: @db, channel_name:, logger: @logger) - ch = Channel.new(name: channel_name, listener:) - listener.subscribe(ch) - ch - end - end - - # Publish a message to a channel - # - # Sends an event to all subscribers on the given channel via Postgres NOTIFY. - # - # @param channel_name [String] the name of the channel to publish to - # @param event [Sourced::Message] the message to publish - # @return [self] - def publish(channel_name, event) - event_data = JSON.dump(event.to_h) - @db.run(Sequel.lit('SELECT pg_notify(?, ?)', channel_name, event_data)) - self - end - end - - class Channel - NOTIFY_CHANNEL = 'sourced-scheduler-ch' - - attr_reader :name - - # Initialize a new channel for receiving messages - # - # @param name [String] the name of the channel (default: NOTIFY_CHANNEL) - # @param listener [Listener] the listener managing this channel - def initialize(name: NOTIFY_CHANNEL, listener:) - @name = name - @running = false - @listener = listener - @queue = Sourced.config.executor.new_queue - end - - # Add a message to this channel's queue - # - # @param message [Sourced::Message] the message to queue - # @return [self] - def <<(message) - @queue << message - self - end - - # Start listening to incoming events on this channel - # - # Blocks until the channel is stopped. Messages are passed to either the provided - # handler or the given block. - # - # @param handler [#call, nil] a callable object to use as an event handler - # @yieldparam message [Sourced::Message] the incoming message - # @yieldparam channel [Channel] this channel instance - # @return [self] - def start(handler: nil, &block) - return self if @running - - @running = true - - handler ||= block - - while (msg = @queue.pop) - handler.call(msg, self) - end - - @running = false - - self - end - - # Stop listening on this channel - # - # Unsubscribes from the listener and marks the channel to stop processing messages. - # The stop takes effect on the next iteration of the message loop. - # - # @return [self] - def stop - return self unless @running - - @listener.unsubscribe self - @queue << nil - self - end - end - end -end diff --git a/lib/sourced/pubsub/test.rb b/lib/sourced/pubsub/test.rb deleted file mode 100644 index 8a6a48d5..00000000 --- a/lib/sourced/pubsub/test.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module PubSub - # An in-memory pubsub implementation for testing and development. - # Thread/fiber-safe. Each subscriber gets its own queue, - # so a slow consumer won't block other subscribers. - class Test - def initialize - @mutex = Mutex.new - @subscribers = {} - end - - # @param channel_name [String] - # @return [Channel] - def subscribe(channel_name) - channel = Channel.new(name: channel_name, pubsub: self) - @mutex.synchronize do - @subscribers[channel_name] = (@subscribers[channel_name] || []) + [channel] - end - channel - end - - # Remove a channel from the subscriber list. - # @param channel [Channel] - def unsubscribe(channel) - @mutex.synchronize do - arr = @subscribers[channel.name] - @subscribers[channel.name] = arr - [channel] if arr - end - end - - # @param channel_name [String] - # @param event [Sourced::Message] - # @return [self] - def publish(channel_name, event) - channels = @subscribers[channel_name] - return self unless channels - - channels.each { |ch| ch << event } - self - end - - class Channel - attr_reader :name - - def initialize(name:, pubsub:) - @name = name - @pubsub = pubsub - @queue = Sourced.config.executor.new_queue - end - - # Push a message into this channel's queue. - # @param message [Sourced::Message] - # @return [self] - def <<(message) - @queue << message - self - end - - # Block and process messages from the queue. - # @param handler [#call, nil] - # @yieldparam message [Sourced::Message] - # @yieldparam channel [Channel] - # @return [self] - def start(handler: nil, &block) - handler ||= block - - while (msg = @queue.pop) - handler.call(msg, self) - end - - self - end - - # Stop processing and unsubscribe from the pubsub. - # @return [self] - def stop - @pubsub.unsubscribe(self) - @queue << nil - self - end - end - end - end -end diff --git a/lib/sourced/rails/install_generator.rb b/lib/sourced/rails/install_generator.rb deleted file mode 100644 index 8f9a0ea3..00000000 --- a/lib/sourced/rails/install_generator.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'rails/generators' -require 'rails/generators/active_record' - -module Sourced - module Rails - class InstallGenerator < ::Rails::Generators::Base - include ActiveRecord::Generators::Migration - - source_root File.expand_path('templates', __dir__) - - class_option :prefix, type: :string, default: 'sourced' - - def copy_initializer_file - create_file 'config/initializers/sourced.rb' do - <<~CONTENT - # frozen_string_literal: true - - require 'sourced' - require 'sourced/backends/active_record_backend' - - # This table prefix is used to generate the initial database migrations. - # If you change the table prefix here, - # make sure to migrate your database to the new table names. - Sourced::Backends::ActiveRecordBackend.table_prefix = '#{table_prefix}' - - # Configure Sors to use the ActiveRecord backend - Sourced.configure do |config| - config.backend = Sourced::Backends::ActiveRecordBackend.new - config.logger = Rails.logger - end - CONTENT - end - end - - def copy_bin_file - copy_file 'bin_sourced', 'bin/sourced' - chmod 'bin/sourced', 0o755 - end - - def create_migration_file - migration_template 'create_sourced_tables.rb.erb', File.join(db_migrate_path, 'create_sourced_tables.rb') - end - - private - - def migration_version - "[#{ActiveRecord::VERSION::STRING.to_f}]" - end - - def table_prefix - options['prefix'] - end - end - end -end diff --git a/lib/sourced/rails/railtie.rb b/lib/sourced/rails/railtie.rb deleted file mode 100644 index 4eb069d7..00000000 --- a/lib/sourced/rails/railtie.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Sourced - module Rails - class Railtie < ::Rails::Railtie - # TODO: review this. - # Workers use Async, so this is needed - # but not sure this can be safely used with non Async servers like Puma. - # config.active_support.isolation_level = :fiber - - generators do - require 'sourced/rails/install_generator' - end - end - end -end diff --git a/lib/sourced/rails/templates/bin_sors b/lib/sourced/rails/templates/bin_sors deleted file mode 100644 index 4400e39b..00000000 --- a/lib/sourced/rails/templates/bin_sors +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env ruby - -require_relative "../config/environment" -require "sourced" - -ActiveRecord::Base.logger = nil - -Sourced::Supervisor.start diff --git a/lib/sourced/rails/templates/create_sors_tables.rb.erb b/lib/sourced/rails/templates/create_sors_tables.rb.erb deleted file mode 100644 index 211ccdb7..00000000 --- a/lib/sourced/rails/templates/create_sors_tables.rb.erb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -class CreateSorsTables < ActiveRecord::Migration<%= migration_version %> - def change - # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support - # enable_extension 'pgcrypto' - - if connection.class.name == 'ActiveRecord::ConnectionAdapters::SQLite3Adapter' - create_table :<%= table_prefix %>_events, id: false do |t| - t.string :id, null: false, index: { unique: true } - t.bigint :global_seq, primary_key: true - t.bigint :seq - t.string :stream_id, null: false, index: true - t.string :type, null: false - t.datetime :created_at - t.string :producer - t.string :causation_id, index: true - t.string :correlation_id - t.text :payload - end - else - create_table :<%= table_prefix %>_events, id: :uuid do |t| - t.bigserial :global_seq, index: true - t.bigint :seq - t.string :stream_id, null: false, index: true - t.string :type, null: false - t.datetime :created_at - t.string :producer - t.uuid :causation_id, index: true - t.uuid :correlation_id - t.jsonb :payload - end - end - - add_index :<%= table_prefix %>_events, %i[stream_id seq], unique: true - - create_table :<%= table_prefix %>_streams do |t| - t.text :stream_id, null: false, index: { unique: true } - t.boolean :locked, default: false, null: false - end - - create_table :<%= table_prefix %>_commands do |t| - t.string :stream_id, null: false - if t.class.name == 'ActiveRecord::ConnectionAdapters::SQLite3::TableDefinition' - t.text :data, null: false - t.datetime :scheduled_at, null: false, default: -> { 'CURRENT_TIMESTAMP' } - else - t.jsonb :data, null: false - t.datetime :scheduled_at, null: false, default: -> { 'NOW()' } - end - end - - add_foreign_key :<%= table_prefix %>_commands, :<%= table_prefix %>_streams, column: :stream_id, primary_key: :stream_id - end -end diff --git a/lib/sourced/react.rb b/lib/sourced/react.rb index 1e19cc76..72adb3fb 100644 --- a/lib/sourced/react.rb +++ b/lib/sourced/react.rb @@ -1,39 +1,13 @@ # frozen_string_literal: true +require 'set' + module Sourced - # This mixin provides a .react macro to register - # message handlers for a class - # These message handlers are "reactions", ie. they react to - # messages by producing new commands which will initiate new Decider flows. - # More here: https://ismaelcelis.com/posts/decide-evolve-react-pattern-in-ruby/#3-react - # - # Example: - # - # class Saga - # include Sourced::React - # - # # Host class must implement a #state method - # # which will be passed to reaction handlers - # attr_reader :state - # - # def initialize(id:) - # @state = { id: } - # end - # - # # React to an event and return a new command. - # # This command will be scheduled for processing by a Decider. - # # Using Sourced::Event#follow copies over metadata from the event - # # including causation and correlation IDs. - # reaction SomethingHappened do |state, event| - # event.follow(DoSomethingElse, field1: 'value1') - # end - # end - # - # saga = Saga.new(id: '123') - # commands = saga.react([something_happened]) - # + # React mixin for reactors. + # Supports the same dispatch-based reaction DSL as Sourced::React, + # adapted to Sourced's stream-less messages. module React - PREFIX = 'reaction' + PREFIX = 'sourced_reaction' EMPTY_ARRAY = [].freeze def self.included(base) @@ -41,51 +15,47 @@ def self.included(base) base.extend ClassMethods end - # @param events [Array] - # @return [Array] - def react(events) - __handling_reactions(Array(events)) do |event| - method_name = Sourced.message_method_name(React::PREFIX, event.class.to_s) + # Run reaction handlers for one or more messages. + # Supports both explicit message returns and dispatch(...) calls. + def react(messages) + __handling_reactions(Array(messages)) do |message| + method_name = Sourced.message_method_name(PREFIX, message.class.to_s) if respond_to?(method_name) - Array(send(method_name, state, event)).compact + Array(send(method_name, state, message)).compact else EMPTY_ARRAY end end end - # TODO: O(1) lookup def reacts_to?(message) self.class.handled_messages_for_react.include?(message.class) end private - def __handling_reactions(events, &) - @__stream_dispatchers = [] - events.each do |event| - @__event_for_reaction = event - yield event + def __handling_reactions(messages) + messages.flat_map do |message| + @__reaction_dispatchers = [] + @__message_for_reaction = message + explicit = Array(yield(message)).compact.reject { |value| value.is_a?(Dispatcher) } + dispatched = @__reaction_dispatchers.map(&:message) + explicit + dispatched end - cmds = @__stream_dispatchers.map(&:message) - @__stream_dispatchers.clear - cmds + ensure + @__reaction_dispatchers = [] + @__message_for_reaction = nil end class Dispatcher attr_reader :message - def initialize(msg) - @message = msg + def initialize(message) + @message = message end def inspect = %(<#{self.class} #{@message}>) - def to(stream_id) - @message = @message.to(stream_id) - self - end - def at(datetime) @message = @message.at(datetime) self @@ -97,28 +67,47 @@ def with_metadata(attrs = {}) end end - def dispatch(command_class, payload = {}) - command_class = self.class[command_class] if command_class.is_a?(Symbol) - cmd = @__event_for_reaction - .follow(command_class, payload) - .with_metadata(producer: self.class.consumer_info.group_id) - - dispatcher = Dispatcher.new(cmd) - @__stream_dispatchers << dispatcher + # Queue a follow-up message from within a reaction block. + # + # The returned {Dispatcher} can be chained to delay the message + # or add metadata before it is appended. + # + # @param message_class [Class, Symbol] messages class, or a symbol + # resolved via .[] + # @param payload [Hash] message payload attributes + # @return [Dispatcher] chainable wrapper around the dispatched message + # + # @example Dispatch by class + # reaction StudentEnrolled do |_state, event| + # dispatch(NotifyStudent, student_id: event.payload.student_id) + # end + # + # @example Dispatch by symbol with delay and metadata + # reaction StudentEnrolled do |_state, event| + # dispatch(:notify_student, student_id: event.payload.student_id) + # .with_metadata(channel: 'email') + # .at(Time.now + 300) + # end + def dispatch(message_class, payload = {}) + message_class = self.class[message_class] if message_class.is_a?(Symbol) + message = @__message_for_reaction + .correlate(message_class.new(payload: payload)) + .with_metadata(producer: self.class.group_id) + + dispatcher = Dispatcher.new(message) + @__reaction_dispatchers << dispatcher dispatcher end module ClassMethods def inherited(subclass) super - handled_messages_for_react.each do |evt_type| - subclass.handled_messages_for_react << evt_type + handled_messages_for_react.each do |klass| + subclass.handled_messages_for_react << klass + end + catch_all_react_events.each do |klass| + subclass.catch_all_react_events << klass end - end - - # Override this with extend Sourced::Consumer - def consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: name) end def handled_messages_for_react @@ -129,77 +118,47 @@ def catch_all_react_events @catch_all_react_events ||= Set.new end - # Define a reaction to an event - # @example - # reaction SomethingHappened do |state, event| - # stream = stream_for(event) - # # stream = stream_for("new-stream-id") - # stream.command DoSomethingElse - # end + # Register a reaction handler for one or more messages types. # - # The host class is expected to define a #state method - # These handlers will load the decider's state from past events, and yield the state and the event to the block. - # @example - # reaction SomethingHappened do |state, event| - # if state[:count] % 3 == 0 - # steam_for(event).command DoSomething - # end - # end + # Accepts message classes, symbols resolved via .[], + # multiple arguments, or no arguments for a catch-all reaction across + # all evolve types without an explicit handler. # - # If no event class given, the handler is registered for all events - # set to evolve in .handled_messaged_for_evolve, unless - # specific reactions have already been registered for them - # The host class is expected to support .handled_messaged_for_evolve - # see Evolve mixin - # @example - # reaction do |state, event| - # LOGGER.info state + # @example React to a specific event class + # reaction StudentEnrolled do |state, event| + # dispatch(NotifyStudent, student_id: event.payload.student_id) # end # - # @overload reaction do |state, event| - # @overload reaction(event_symbol) do |state, event| - # @param event_symbol [Symbol] Symbolised message name - # @overload reaction(event_class) do |state, event| - # @param event_class [Class] Must be subclass of Sourced::Message - # @overload reaction(*events) do |state, event| - # @param *events [Array] List of event classes or symbols - # @return [void] + # @example React to a symbol-resolved message class + # reaction :student_enrolled do |state, event| + # dispatch(:notify_student, student_id: event.payload.student_id) + # end def reaction(*args, &block) case args in [] - handled_messages_for_evolve.each do |e| - method_name = Sourced.message_method_name(React::PREFIX, e.to_s) - if !instance_methods.include?(method_name.to_sym) - catch_all_react_events << e - reaction e, &block - end + handled_messages_for_evolve.each do |message_class| + method_name = Sourced.message_method_name(PREFIX, message_class.to_s) + next if instance_methods.include?(method_name.to_sym) + + catch_all_react_events << message_class + reaction(message_class, &block) end in [Symbol => message_symbol] - message_class = __resolve_message_class(message_symbol).tap do |klass| + message_class = self[message_symbol].tap do |klass| raise( ArgumentError, - "Cannot resolve message symbol #{message_symbol.inspect} " \ - "for #{self}.reaction" + "Cannot resolve message symbol #{message_symbol.inspect} for #{self}.reaction" ) unless klass end reaction(message_class, &block) in [Class => message_class] if message_class < Sourced::Message __validate_message_for_reaction!(message_class) - unless message_class.is_a?(Class) && message_class < Sourced::Message - raise( - ArgumentError, - "Invalid argument #{message_class.inspect} for #{self}.reaction" - ) - end - - self.handled_messages_for_react << message_class - define_method(Sourced.message_method_name(React::PREFIX, message_class.to_s), &block) if block_given? - in Array => args if args.none?(&:nil?) - args.each do |k| - reaction k, &block - end + handled_messages_for_react << message_class + define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) if block_given? + in Array => values if values.none?(&:nil?) + values.each { |value| reaction(value, &block) } else raise( ArgumentError, @@ -208,17 +167,8 @@ def reaction(*args, &block) end end - # Run this hook before registering a reaction - # Actor can override this to make sure that the same message is not - # also handled as a command - def __validate_message_for_reaction!(event_class) - # no-op. - end - - private - - def __resolve_message_class(message_symbol) - raise ArgumentError, "#{self} doesn't support resolving #{message_symbol.inspect} into a message class" + def __validate_message_for_reaction!(_message_class) + # no-op end end end diff --git a/lib/sourced/router.rb b/lib/sourced/router.rb index 957d0839..9ff95d75 100644 --- a/lib/sourced/router.rb +++ b/lib/sourced/router.rb @@ -1,197 +1,178 @@ # frozen_string_literal: true -require 'singleton' require 'sourced/injector' module Sourced - # The Router is the central dispatch mechanism in Sourced, responsible for: - # - Registering Reactors (actors and projectors) - # - Routing events to appropriate reactors - # - Managing the execution of asynchronous reactors - # - Coordinating with the backend for event storage and retrieval - # - # The Router uses the Singleton pattern to ensure a single global registry - # of all registered components in the system. - # - # @example Register components - # Sourced::Router.register(MyActor) - # Sourced::Router.register(MyProjector) - # class Router - include Singleton + attr_reader :store, :reactors - PID = Process.pid - - class << self - public :new + def initialize(store:) + @store = store + @reactors = [] + @needs_history = {} + end - # Register an actor or projector for command/event handling. - # @param args [Object] Arguments passed to the instance register method - # @return [void] - # @see #register - def register(...) - instance.register(...) - end + def register(reactor_class) + @reactors << reactor_class + store.register_consumer_group( + reactor_class.group_id, + partition_by: reactor_class.partition_keys.map(&:to_s) + ) + @needs_history[reactor_class] = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) + end - # @return [Boolean] true if the class is registered as a decider or reactor - def registered?(...) - instance.registered?(...) - end + def handle_next_for(reactor_class, worker_id: 'default', batch_size: nil) + handled_types = reactor_class.handled_messages.map(&:type).uniq + + claim = store.claim_next( + reactor_class.group_id, + partition_by: reactor_class.partition_keys.map(&:to_s), + handled_types: handled_types, + worker_id: worker_id, + batch_size: batch_size + ) + return false unless claim + + begin + kwargs = {} + if @needs_history[reactor_class] + attrs = claim.partition_value.transform_keys(&:to_sym) + conditions = reactor_class.context_for(attrs) + kwargs[:history] = store.read(conditions) + end - # Get all registered asynchronous reactors. - # @return [Set] Set of async reactor classes - # @see #async_reactors - def async_reactors - instance.async_reactors - end + action_pairs = reactor_class.handle_claim(claim, **kwargs) - # Handle the next available event for a specific reactor. - # @param reactor [Class] The reactor class to get events for - # @param process_name [String, nil] Optional process identifier for logging - # @return [Boolean] true if an event was handled, false if no events available - # @see #handle_next_event_for_reactor - def handle_next_event_for_reactor(...) - instance.handle_next_event_for_reactor(...) - end + if action_pairs == Actions::RETRY + store.release(reactor_class.group_id, offset_id: claim.offset_id) + return true + end - def backend = instance.backend - end + execute_actions(action_pairs, claim, reactor_class.group_id) + true - # @!attribute [r] async_reactors - # @return [Set] Reactors that run asynchronously in background workers - # @!attribute [r] backend - # @return [Object] The configured backend for event storage - # @!attribute [r] logger - # @return [Object] The configured logger instance - attr_reader :async_reactors, :backend, :logger, :needs_history - - # Initialize a new Router instance. - # @param backend [Object] Backend for event storage (defaults to configured backend) - # @param logger [Object] Logger instance (defaults to configured logger) - def initialize(backend: Sourced.config.backend, logger: Sourced.config.logger) - @backend = backend - @logger = logger - @registered_lookup = {} - @needs_history = {} - @async_reactors = Set.new + rescue Sourced::PartialBatchError => e + execute_actions(e.action_pairs, claim, reactor_class.group_id) + store.updating_consumer_group(reactor_class.group_id) do |group| + reactor_class.on_exception(e, e.failed_message, group) + end + true + rescue Sourced::ConcurrentAppendError + store.release(reactor_class.group_id, offset_id: claim.offset_id) + true + rescue StandardError => e + store.release(reactor_class.group_id, offset_id: claim.offset_id) + store.updating_consumer_group(reactor_class.group_id) do |group| + reactor_class.on_exception(e, claim.messages.first, group) + end + true + end end - # Register a Reactor with the router. + # Stop a consumer group and invoke the reactor's {Consumer#on_stop} callback. # - # During registration, the router analyzes the reactor's #handle_batch method signature - # to determine whether it needs stream history. Reactors that declare a `history:` keyword - # will receive the full stream history when processing batches. + # Marks the group as stopped in the store so workers will no longer claim + # work for it, then calls +on_stop+ on the reactor class. # - # @param thing [Class] Reactor object to register. + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @param message [String, nil] optional reason for stopping (persisted in the group's error_context) # @return [void] - # @raise [InvalidReactorError] if the class doesn't implement required interfaces - # - # @example Register an actor that handles both commands and events - # router.register(CartActor) + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor # - # @example Reactors with different handle_batch signatures - # # Reactor that doesn't need history (default Consumer wrapper) - # class SimpleReactor - # extend Sourced::Consumer - # def self.handle(event) = Sourced::Actions::OK - # end + # @example Stop with a reactor class + # router.stop_consumer_group(CourseDecider, 'maintenance window') # - # # Reactor that needs access to full stream history - # class HistoryReactor - # extend Sourced::Consumer - # def self.handle_batch(batch, history:) - # # batch is Array of [message, replaying] pairs - # # history is Array of all messages in the stream - # end - # end - def register(thing) - unless ReactorInterface === thing - raise InvalidReactorError, "#{thing.inspect} is not a valid Reactor interface" - end - - # Analyze the reactor's handle_batch signature to determine if it needs history - @needs_history[thing] = Injector.resolve_args(thing, :handle_batch).include?(:history) - @async_reactors << thing - - group_id = thing.consumer_info.group_id - @registered_lookup[group_id] = true - backend.register_consumer_group(group_id) + # @example Stop with a string group_id + # router.stop_consumer_group('CourseDecider') + def stop_consumer_group(reactor_or_id, message = nil) + reactor_class = resolve_reactor_class(reactor_or_id) + store.stop_consumer_group(reactor_class.group_id, message) + reactor_class.on_stop(message) end - def registered?(thing) - !!@registered_lookup[thing.consumer_info.group_id] + # Reset a consumer group and invoke the reactor's {Consumer#on_reset} callback. + # + # Clears all partition offsets and resets the discovery position to 0, + # so the group will reprocess messages from the beginning. Does not + # change the group's status (a stopped group remains stopped after reset). + # Then calls +on_reset+ on the reactor class. + # + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # + # @example + # router.reset_consumer_group(CourseDecider) + def reset_consumer_group(reactor_or_id) + reactor_class = resolve_reactor_class(reactor_or_id) + store.reset_consumer_group(reactor_class.group_id) + reactor_class.on_reset end - # Handle the next available batch of messages for a specific reactor. + # Start a consumer group and invoke the reactor's {Consumer#on_start} callback. # - # Fetches a batch of messages from the backend and calls reactor.handle_batch(batch, **kargs). - # If the reactor's handle_batch signature includes `history:`, the full stream history - # is fetched from the backend and passed through. + # Marks the group as active in the store so workers can claim work for it + # again, then calls +on_start+ on the reactor class. # - # @param reactor [Class] The reactor class to get events for - # @param worker_id [String, nil] Optional process identifier for logging - # @param raise_on_error [Boolean] Raise error immediately instead of notifying Reactor#on_exception - # @param batch_size [Integer] Number of messages to fetch per lock cycle - # @return [Boolean] true if a batch was handled, false if no messages available - def handle_next_event_for_reactor(reactor, worker_id = nil, raise_on_error = false, batch_size: 1) - effective_batch_size = reactor.consumer_info.batch_size || batch_size - found = false - - backend.reserve_next_for_reactor(reactor, batch_size: effective_batch_size, with_history: @needs_history[reactor], worker_id:) do |batch, history| - found = true - first_msg = batch.first&.first - log_event("handling batch(#{batch.size})", reactor, first_msg, worker_id) if first_msg - - kargs = {} - kargs[:history] = history if @needs_history[reactor] - reactor.handle_batch(batch, **kargs) - rescue PartialBatchError => e - raise e if raise_on_error - - logger.warn "[#{PID}]: partial batch failure for reactor #{reactor}: #{e.message}" - backend.updating_consumer_group(reactor.consumer_info.group_id) do |group| - reactor.on_exception(e, e.failed_message, group) - end - e.action_pairs - rescue StandardError => e - raise e if raise_on_error - - logger.warn "[#{PID}]: error handling batch with reactor #{reactor} #{e}" - backend.updating_consumer_group(reactor.consumer_info.group_id) do |group| - reactor.on_exception(e, batch.first&.first, group) - end - Actions::RETRY - end - found + # @param reactor_or_id [Class, String] a registered reactor class, or its +group_id+ string + # @return [void] + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + # + # @example + # router.start_consumer_group(CourseDecider) + def start_consumer_group(reactor_or_id) + reactor_class = resolve_reactor_class(reactor_or_id) + store.start_consumer_group(reactor_class.group_id) + reactor_class.on_start end - # Handle messages for reactors in this router - # until there's none left in the backend - # Useful for testing workflows - # @param limit [Numeric] How many times to loop fetching new messages def drain(limit = Float::INFINITY) - pid = Process.pid - have_messages = @async_reactors.each.with_index.with_object({}) { |(_, i), m| m[i] = true } - count = 0 loop do count += 1 - @async_reactors.each.with_index do |r, idx| - found = handle_next_event_for_reactor(r, pid, true) - have_messages[idx] = found - end - break if have_messages.values.none? || count >= limit + found_any = @reactors.any? { |r| handle_next_for(r) } + break unless found_any && count < limit end end private - def log_event(label, reactor, event, process_name = PID) - logger.info "[#{process_name}]: #{reactor.consumer_info.group_id} #{label} #{event_info(event)}" + # Resolve a reactor class or group_id string to a registered reactor class. + # + # @param reactor_or_id [Class, String] a reactor class (returned as-is) or a +group_id+ string + # @return [Class] the matching registered reactor class + # @raise [ArgumentError] if +reactor_or_id+ is a String that doesn't match any registered reactor + def resolve_reactor_class(reactor_or_id) + return reactor_or_id if reactor_or_id.is_a?(Module) + + @reactors.find { |r| r.group_id == reactor_or_id } || + raise(ArgumentError, "No reactor registered with group_id '#{reactor_or_id}'") end - def event_info(event) - %([#{event.type}] stream_id:#{event.stream_id} seq:#{event.seq}) + def execute_actions(action_pairs, claim, group_id) + after_sync_actions = [] + + store.db.transaction do + last_position = nil + Array(action_pairs).each do |(actions, source_message)| + Array(actions).each do |action| + if action.is_a?(Actions::AfterSync) + after_sync_actions << action + elsif action != Actions::OK + action.execute(store, source_message) + end + end + last_position = source_message.position if source_message.respond_to?(:position) + end + + if last_position + store.ack(group_id, offset_id: claim.offset_id, position: last_position) + else + store.release(group_id, offset_id: claim.offset_id) + end + end + + after_sync_actions.each(&:call) end end end diff --git a/lib/sourced/scheduled_message_poller.rb b/lib/sourced/scheduled_message_poller.rb new file mode 100644 index 00000000..ada03289 --- /dev/null +++ b/lib/sourced/scheduled_message_poller.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Sourced + # Periodically promotes due scheduled messages into the main log. + class ScheduledMessagePoller + # @param store [Sourced::Store] the store containing scheduled messages + # @param interval [Numeric] polling interval in seconds + # @param logger [Object] logger instance + def initialize(store:, interval: 5, logger: Sourced.config.logger) + @store = store + @interval = interval + @logger = logger + @running = false + end + + # Run the polling loop until {#stop} is called. + # + # @return [void] + def run + @running = true + while @running + promoted = @store.update_schedule! + @logger.info "Sourced::ScheduledMessagePoller: appended #{promoted} scheduled messages" if promoted > 0 + sleep @interval + end + @logger.info 'Sourced::ScheduledMessagePoller: stopped' + end + + # Signal the poller to stop after the current sleep cycle. + # + # @return [void] + def stop + @running = false + end + end +end diff --git a/lib/sourced/stale_claim_reaper.rb b/lib/sourced/stale_claim_reaper.rb new file mode 100644 index 00000000..73f84ab3 --- /dev/null +++ b/lib/sourced/stale_claim_reaper.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Sourced + # Periodic loop that heartbeats active workers and releases claims + # held by workers that have stopped heartbeating (crashed or killed). + # + # Combines heartbeating and reaping in one loop since Sourced doesn't have + # a separate HouseKeeper like the main Sourced module. + # + # +worker_ids_provider+ is a proc that returns current worker names — + # injected by the {Dispatcher} which owns the Worker instances. + # + # @example + # reaper = StaleClaimReaper.new( + # store: store, + # interval: 30, + # ttl_seconds: 120, + # worker_ids_provider: -> { workers.map(&:name) }, + # logger: logger + # ) + # # In a fiber/thread: + # reaper.run # blocks, heartbeating + reaping every 30s + # # From another fiber/thread: + # reaper.stop # breaks the loop + class StaleClaimReaper + # @param store [Sourced::Store] the store + # @param interval [Numeric] seconds between heartbeat/reap cycles (default 30) + # @param ttl_seconds [Integer] age threshold for stale claims (default 120) + # @param worker_ids_provider [Proc] returns Array of active worker IDs + # @param logger [Object] logger instance + def initialize(store:, interval: 30, ttl_seconds: 120, worker_ids_provider: -> { [] }, logger: Sourced.config.logger) + @store = store + @interval = interval + @ttl_seconds = ttl_seconds + @worker_ids_provider = worker_ids_provider + @logger = logger + @running = false + end + + # Run the heartbeat/reap loop. Blocks until {#stop} is called. + # Reaps on startup (from previous runs where workers were killed). + # + # @return [void] + def run + @running = true + reap # reap on startup for claims left by previously killed workers + while @running + sleep @interval + heartbeat if @running + reap if @running + end + @logger.info 'Sourced::StaleClaimReaper: stopped' + end + + # Signal the reaper to stop after the current sleep cycle. + # + # @return [void] + def stop + @running = false + end + + private + + def heartbeat + ids = Array(@worker_ids_provider.call).uniq + count = @store.worker_heartbeat(ids) + @logger.debug "Sourced::StaleClaimReaper: heartbeated #{count} workers" if count > 0 + end + + def reap + released = @store.release_stale_claims(ttl_seconds: @ttl_seconds) + @logger.info "Sourced::StaleClaimReaper: released #{released} stale claims" if released > 0 + end + end +end diff --git a/lib/sourced/store.rb b/lib/sourced/store.rb new file mode 100644 index 00000000..d334a191 --- /dev/null +++ b/lib/sourced/store.rb @@ -0,0 +1,1303 @@ +# frozen_string_literal: true + +require 'json' +require 'set' +require 'sourced/inline_notifier' +require 'sourced/installer' + +module Sourced + # Wraps a Message with a storage position. Delegates all message methods. + class PositionedMessage < SimpleDelegator + attr_reader :position + + # @param message [Sourced::Message] the wrapped message instance + # @param position [Integer] global log position + def initialize(message, position) + super(message) + @position = position + end + + def class = __getobj__.class + def is_a?(klass) = __getobj__.is_a?(klass) || super + def kind_of?(klass) = is_a?(klass) + def instance_of?(klass) = __getobj__.instance_of?(klass) + + # Unwrap to the underlying {Sourced::Message}. Part of the +to_message+ + # contract honoured by {Sourced::Message.===} so that +case/when+ works + # transparently across wrapped and unwrapped messages. + def to_message = __getobj__ + end + + # Returned by {Store#claim_next} with everything needed to process and ack a partition. + ClaimResult = Data.define(:offset_id, :key_pair_ids, :partition_key, :partition_value, :messages, :replaying, :guard) + + # Returned by {Store#read} with messages and a consistency guard. + # Supports array destructuring via #to_ary for backwards compatibility: + # messages, guard = store.read(conditions) + ReadResult = Data.define(:messages, :guard) do + def to_ary = [messages, guard] + end + + ReadAllResult = Data.define(:messages, :last_position, :fetcher) do + include Enumerable + + def to_ary = [messages, last_position] + + # Iterates messages in the current page. + def each(&block) = messages.each(&block) + + # Returns an Enumerator that lazily paginates through all messages, + # fetching subsequent pages as needed. + def to_enum + Enumerator.new do |y| + result = self + loop do + break if result.messages.empty? + + result.messages.each { |m| y << m } + result = result.fetcher.call(result.messages.last.position) + end + end + end + end + + Stats = Data.define(:max_position, :groups) + + OffsetsResult = Data.define(:offsets, :total_count, :fetcher) do + include Enumerable + + def to_ary = [offsets, total_count] + + # Iterates offsets in the current page. + def each(&block) = offsets.each(&block) + + # Returns an Enumerator that lazily paginates through all offsets, + # fetching subsequent pages as needed. + def to_enum + Enumerator.new do |y| + result = self + loop do + break if result.offsets.empty? + + result.offsets.each { |o| y << o } + result = result.fetcher.call(result.offsets.last[:id] + 1) + end + end + end + end + + # SQLite-backed store for Sourced's flat, globally-ordered message log. + # Provides message storage with automatic key-pair indexing, + # consumer group management, and partition-based offset tracking + # for parallel background processing. + class Store + ACTIVE = 'active' + STOPPED = 'stopped' + FAILED = 'failed' + DISCOVERY_BATCH_SIZE = 100 + + # @return [Sequel::SQLite::Database] + attr_reader :db + + # @return [Sourced::InlineNotifier] + attr_reader :notifier + + # @return [Logger] + attr_reader :logger + + # @return [Sourced::Installer] + attr_reader :installer + + # @param db [Sequel::SQLite::Database] a Sequel SQLite connection + # @param notifier [#notify_new_messages, #notify_reactor_resumed, nil] optional notifier for dispatch signals + # @param logger [Logger, nil] optional logger (defaults to Sourced.config.logger) + # @param prefix [String] table name prefix (default 'sourced') + def initialize(db, notifier: nil, logger: nil, prefix: 'sourced') + @db = db + @notifier = notifier || Sourced::InlineNotifier.new + @logger = logger || Sourced.config.logger + Sequel.extension(:fiber_concurrency) + @db.run('PRAGMA foreign_keys = ON') + @db.run('PRAGMA journal_mode = WAL') + @db.run('PRAGMA busy_timeout = 5000') + + @installer = Installer.new(db, logger: @logger, prefix: prefix) + + # Source table name symbols from the installer + @messages_table = @installer.messages_table + @key_pairs_table = @installer.key_pairs_table + @message_key_pairs_table = @installer.message_key_pairs_table + @scheduled_messages_table = @installer.scheduled_messages_table + @consumer_groups_table = @installer.consumer_groups_table + @offsets_table = @installer.offsets_table + @offset_key_pairs_table = @installer.offset_key_pairs_table + @workers_table = @installer.workers_table + + # Cache of registered consumer groups for eager offset creation in append. + # Populated by register_consumer_group. + # { group_id => { cg_id: Integer, partition_by: Array | nil } } + @registered_groups = {} + end + + # Whether all required tables exist. + # @return [Boolean] + def installed? + installer.installed? + end + + # Create all required tables and indexes. Idempotent. + # @return [void] + def install! + installer.install + end + + # Drop all tables. Test-only guard. + # @return [void] + def uninstall + installer.uninstall + end + + # Render the migration to a file for use with the host app's Sequel::Migrator. + # @see Installer#copy_migration_to + def copy_migration_to(dir = nil, &block) + installer.copy_migration_to(dir, &block) + end + + # Append messages to the store. Extracts and indexes key-value pairs + # from each message's payload automatically. + # + # When a {ConsistencyGuard} is provided, checks for conflicting messages + # before inserting (optimistic concurrency). + # + # @param messages [Sourced::Message, Array] one or more messages to append + # @param guard [ConsistencyGuard, nil] optional guard for conflict detection + # @return [Integer] the last assigned position + # @raise [Sourced::ConcurrentAppendError] if conflicting messages found after guard position + def append(messages, guard: nil) + messages = Array(messages) + return latest_position if messages.empty? + + last_position = nil + + db.transaction do + if guard + conflicts = check_conflicts(guard.conditions, guard.last_position) + raise Sourced::ConcurrentAppendError, "Conflicting messages found after position #{guard.last_position}" if conflicts.any? + end + + messages.each do |msg| + payload_json = msg.payload ? JSON.dump(msg.payload.to_h) : '{}' + metadata_json = msg.metadata.empty? ? nil : JSON.dump(msg.metadata) + + # insert returns last_insert_rowid on SQLite — no need for a separate SELECT + last_position = db[@messages_table].insert( + message_id: msg.id, + message_type: msg.type, + causation_id: msg.causation_id, + correlation_id: msg.correlation_id, + payload: payload_json, + metadata: metadata_json, + created_at: msg.created_at.iso8601 + ) + + # Upsert key pairs and link to message in 2 statements (was 3): + # 1. INSERT OR IGNORE the key_pair + # 2. INSERT message_key_pair with key_pair_id resolved via subquery + msg.extracted_keys.each do |name, value| + db.run("INSERT OR IGNORE INTO #{@key_pairs_table} (name, value) VALUES (#{db.literal(name)}, #{db.literal(value)})") + db.run(<<~SQL) + INSERT INTO #{@message_key_pairs_table} (message_position, key_pair_id) + SELECT #{db.literal(last_position)}, id + FROM #{@key_pairs_table} + WHERE name = #{db.literal(name)} AND value = #{db.literal(value)} + SQL + end + end + + ensure_offsets_for_registered_groups(messages) + end + + notifier.notify_new_messages(messages.map(&:type).uniq) + + last_position + end + + # Persist messages for future promotion into the main log. + # + # @param messages [Sourced::Message, Array] one or more delayed messages + # @param at [Time] when the messages should become available + # @return [Boolean] false when no messages were provided, true otherwise + def schedule_messages(messages, at:) + messages = Array(messages) + return false if messages.empty? + + now = Time.now + rows = messages.map do |message| + data = message.to_h + data[:metadata] = message.metadata.merge(scheduled_at: now) + { + created_at: now.iso8601, + available_at: at.iso8601, + message: JSON.dump(data) + } + end + + db.transaction do + db[@scheduled_messages_table].multi_insert(rows) + end + + true + end + + # Promote due scheduled messages into the main log. + # + # Appended messages are re-inserted through {#append} so they are indexed, + # assigned fresh positions, and announced through the store notifier. + # + # @return [Integer] number of scheduled messages promoted + def update_schedule! + now = Time.now + + db.transaction do + rows = db[@scheduled_messages_table] + .where { available_at <= now.iso8601 } + .order(:id) + .limit(100) + .all + + return 0 if rows.empty? + + messages = rows.map do |row| + data = JSON.parse(row[:message], symbolize_names: true) + data[:created_at] = now + Message.from(data) + end + + append(messages) + + row_ids = rows.map { |row| row[:id] } + db[@scheduled_messages_table].where(id: row_ids).delete + + rows.size + end + end + + # Paginate the global event log in position order. + # + # @example First page + # messages = store.read_all(limit: 20) + # + # @example Next page (using the last position from the previous page) + # messages = store.read_all(from_position: 20, limit: 20) + # + # @example Filtered by conditions (OR semantics, same as #read) + # messages = store.read_all(conditions: [cond1, cond2], limit: 20) + # + # @param from_position [Integer] return messages from this position, inclusive (default 0) + # @param conditions [Array, nil] optional conditions to filter by + # @param limit [Integer] max number of messages to return (default 50) + # @return [ReadAllResult] messages and last global position + def read_all(from_position: nil, conditions: [], limit: 50, order: :asc) + desc = order == :desc + conditions = Array(conditions).compact + after_position = from_position ? from_position - 1 : nil + + if conditions.any? + messages = query_messages(conditions, after_position:, limit:, order:) + else + ds = db[@messages_table] + ds = desc ? ds.where { position <= from_position } : ds.where { position >= from_position } if from_position + messages = ds.order(desc ? Sequel.desc(:position) : :position).limit(limit).map { |row| deserialize(row) } + end + + fetcher = ->(pos) { read_all(from_position: desc ? pos - 1 : pos + 1, conditions:, limit:, order:) } + ReadAllResult.new(messages:, last_position: latest_position, fetcher:) + end + + # Query messages by conditions. Each condition matches on + # (message_type AND key_name AND key_value). Multiple conditions are OR'd. + # + # @param conditions [QueryCondition, Array] query conditions + # @param after_position [Integer, nil] only return messages after this position (exclusive) + # @param limit [Integer, nil] max number of messages to return + # @return [ReadResult] messages and a guard + def read(conditions, after_position: nil, limit: nil) + conditions = Array(conditions) + if conditions.empty? + guard = ConsistencyGuard.new(conditions:, last_position: after_position || latest_position) + return ReadResult.new(messages: [], guard:) + end + + messages = query_messages(conditions, after_position:, limit:) + last_position = messages.any? ? messages.last.position : (after_position || latest_position) + guard = ConsistencyGuard.new(conditions:, last_position:) + ReadResult.new(messages:, guard:) + end + + # Read messages for a specific partition using AND semantics. + # A message is included only when every partition attribute it declares + # matches the given value. Messages that don't declare a partition + # attribute pass through (same logic as {#claim_next}). + # + # @example Single partition attribute + # result = store.read_partition( + # { device_id: 'dev-1' }, + # handled_types: ['device.registered', 'device.bound'] + # ) + # result.messages # => [#, ...] + # result.guard # => # + # + # @example Composite partition (AND semantics — messages must match all attributes they declare) + # result = store.read_partition( + # { course_name: 'Algebra', user_id: 'joe' }, + # handled_types: ['course.created', 'user.joined_course'] + # ) + # # Returns CourseCreated(course_name: 'Algebra') — matches on its only attribute + # # Returns UserJoinedCourse(course_name: 'Algebra', user_id: 'joe') — matches both + # # Excludes UserJoinedCourse(course_name: 'Algebra', user_id: 'jane') — user_id mismatch + # + # @example Resuming from a position (e.g. after processing a batch) + # result = store.read_partition( + # { device_id: 'dev-1' }, + # handled_types: ['device.registered'], + # after_position: 42 + # ) + # # Only returns messages with position > 42 + # + # @example Using the guard for optimistic concurrency on append + # result = store.read_partition( + # { device_id: 'dev-1' }, + # handled_types: ['device.registered', 'device.bound'] + # ) + # # ... build new events from result.messages ... + # store.append(new_events, guard: result.guard) + # # Raises Sourced::ConcurrentAppendError if conflicting writes occurred + # + # @param partition_attrs [Hash{Symbol|String => String}] partition attribute values + # @param handled_types [Array] message type strings to include + # @param after_position [Integer] fetch messages after this position (exclusive, default 0) + # @return [ReadResult] messages and a guard for optimistic concurrency + def read_partition(partition_attrs, handled_types:, after_position: 0) + # Resolve key_pair_ids for each partition attribute + key_pair_ids = partition_attrs.filter_map do |name, value| + db[@key_pairs_table].where(name: name.to_s, value: value.to_s).get(:id) + end + + # If any key pair doesn't exist in the store, no messages can match + if key_pair_ids.size < partition_attrs.size + guard = ConsistencyGuard.new(conditions: [], last_position: after_position) + return ReadResult.new(messages: [], guard:) + end + + messages = fetch_partition_messages(key_pair_ids, after_position, handled_types) + + # Build guard conditions from handled_types, scoped to partition attrs. + # These use OR semantics so the guard detects any concurrent write + # in the broader partition context (e.g. another student enrolling). + partition_sym = partition_attrs.transform_keys(&:to_sym) + guard_conditions = handled_types.filter_map do |type| + klass = Message.registry[type] + klass&.to_conditions(**partition_sym) + end.flatten + + # The guard's last_position must cover the full OR-context, not just + # the AND-filtered messages. Otherwise a message that passes the OR + # conditions but was excluded by AND filtering would look like a conflict. + last_pos = max_position_for(guard_conditions, after_position: after_position) + + guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) + ReadResult.new(messages: messages, guard: guard) + end + + # Conflict detection: returns messages matching conditions that appeared + # after the given position. Empty array means no conflicts. + # + # @param conditions [Array] conditions to check + # @param position [Integer] check for messages after this position + # @return [ReadResult] + def messages_since(conditions, position) + read(conditions, after_position: position) + end + + # Register a consumer group. Idempotent. + # When +partition_by+ is provided, offsets are created eagerly during {#append} + # instead of lazily via discovery in {#claim_next}. + # + # @param group_id [String] unique identifier for the consumer group + # @param partition_by [Array, nil] attribute names defining partitions + # @return [void] + def register_consumer_group(group_id, partition_by: nil) + partition_by_sorted = partition_by ? Array(partition_by).map(&:to_s).sort : nil + partition_by_json = partition_by_sorted ? JSON.dump(partition_by_sorted) : nil + now = Time.now.iso8601 + db.run(<<~SQL) + INSERT INTO #{@consumer_groups_table} (group_id, status, highest_position, partition_by, created_at, updated_at) + VALUES (#{db.literal(group_id)}, '#{ACTIVE}', 0, #{db.literal(partition_by_json)}, #{db.literal(now)}, #{db.literal(now)}) + ON CONFLICT(group_id) DO UPDATE SET partition_by = #{db.literal(partition_by_json)}, updated_at = #{db.literal(now)} + SQL + + # Cache for hot-path use in append + cg = db[@consumer_groups_table].where(group_id: group_id).first + @registered_groups[group_id] = { cg_id: cg[:id], partition_by: partition_by_sorted } + end + + # Whether the consumer group exists and is active. + # + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ + # @return [Boolean] + def consumer_group_active?(group_id) + group_id = resolve_group_id(group_id) + row = db[@consumer_groups_table].where(group_id: group_id).select(:status).first + return false unless row + + row[:status] == ACTIVE + end + + # Stop a consumer group intentionally. Stopped groups are skipped by {#claim_next}. + # + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ + # @param message [String, nil] optional operator-supplied reason + # @return [void] + def stop_consumer_group(group_id, message = nil) + group_id = resolve_group_id(group_id) + updating_consumer_group(group_id) do |group| + group.stop(message:) + end + end + + # Re-activate a stopped or failed consumer group, clearing retry state. + # + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ + # @return [void] + def start_consumer_group(group_id) + group_id = resolve_group_id(group_id) + db[@consumer_groups_table] + .where(group_id: group_id) + .update(status: ACTIVE, retry_at: nil, error_context: nil, updated_at: Time.now.iso8601) + notifier.notify_reactor_resumed(group_id) + end + + # Load a consumer group row, yield a {GroupUpdater} for mutation, + # then persist the accumulated updates atomically. + # Mirrors SequelBackend#updating_consumer_group. + # + # @param group_id [String] + # @yieldparam group [Sourced::GroupUpdater] + # @return [void] + def updating_consumer_group(group_id) + dataset = db[@consumer_groups_table].where(group_id: group_id) + row = dataset.first + raise ArgumentError, "Consumer group #{group_id} not found" unless row + + ctx = row[:error_context] ? JSON.parse(row[:error_context], symbolize_names: true) : {} + row[:error_context] = ctx + + group = Sourced::GroupUpdater.new(group_id, row, logger) + yield group + + updates = group.updates.dup + updates[:error_context] = JSON.dump(updates[:error_context]) + dataset.update(updates) + end + + # Delete all offsets for a consumer group, resetting it to process from the beginning. + # + # @param group_id [String, #group_id] identifier or object responding to +#group_id+ + # @return [void] + def reset_consumer_group(group_id) + group_id = resolve_group_id(group_id) + cg = db[@consumer_groups_table].where(group_id: group_id).first + return unless cg + + db[@offsets_table].where(consumer_group_id: cg[:id]).delete + db[@consumer_groups_table].where(id: cg[:id]).update( + discovery_position: 0, + last_nil_types_max_pos: 0, + updated_at: Time.now.iso8601 + ) + end + + # Claim the next available partition for processing. + # + # Bootstraps partition offsets (discovering new partitions from messages with + # ALL +partition_by+ attributes), finds the unclaimed partition with the earliest + # pending message, claims it, and fetches messages using conditional AND semantics. + # + # Returns a {ConsistencyGuard} alongside the messages, built from each handled + # message class's declared payload attributes via {Message.to_conditions}. + # + # The +replaying+ flag indicates whether the returned messages have been + # processed by this consumer group before. A message is replaying when its + # position is at or before the consumer group's +highest_position+ — the + # furthest position ever successfully acked. After a reset, re-claimed + # messages are correctly flagged as replaying. + # + # @param group_id [String] consumer group identifier + # @param partition_by [String, Array] attribute name(s) defining partitions + # @param handled_types [Array] message type strings this consumer handles + # @param worker_id [String] identifier for the claiming worker + # @param batch_size [Integer, nil] max messages to fetch per claim (nil = unlimited) + # @return [Hash, nil] +{ offset_id:, key_pair_ids:, partition_key:, partition_value:, messages:, replaying:, guard: }+ or nil + def claim_next(group_id, partition_by:, handled_types:, worker_id:, batch_size: nil) + partition_by = Array(partition_by).sort + now = Time.now.iso8601 + cg = db[@consumer_groups_table] + .where(group_id: group_id, status: ACTIVE) + .where { Sequel.|({retry_at: nil}, Sequel.lit('retry_at <= ?', now)) } + .first + return nil unless cg + + # Short-circuit: no new messages since the last nil claim. + types_max_pos = db[@messages_table] + .where(message_type: handled_types) + .max(:position) || 0 + + return nil if types_max_pos <= cg[:last_nil_types_max_pos] + + claimed = nil + group_info = @registered_groups[group_id] + + if group_info&.fetch(:partition_by, nil) + # Eager path: offsets were created by append. Try fast claim first, + # fall back to discovery only for catch-up (new group against existing log). + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + unless claimed + discover_new_partitions(cg[:id], partition_by, handled_types) + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end + else + # Legacy path: lazy discovery + has_offsets = db[@offsets_table].where(consumer_group_id: cg[:id]).limit(1).any? + if has_offsets && types_max_pos <= cg[:discovery_position] + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end + unless claimed + discover_new_partitions(cg[:id], partition_by, handled_types) + claimed = find_and_claim_partition(cg[:id], handled_types, worker_id) + end + end + + unless claimed + # Remember types_max_pos so next poll short-circuits instantly + db[@consumer_groups_table].where(id: cg[:id]) + .update(last_nil_types_max_pos: types_max_pos) + return nil + end + + key_pair_ids = db[@offset_key_pairs_table] + .where(offset_id: claimed[:offset_id]) + .select_map(:key_pair_id) + + messages = fetch_partition_messages(key_pair_ids, claimed[:last_position], handled_types, limit: batch_size) + + # If no messages pass the conditional AND filter, release and return nil + if messages.empty? + release(group_id, offset_id: claimed[:offset_id]) + return nil + end + + # Build partition_value hash from key_pairs + partition_value = {} + db[@key_pairs_table].where(id: key_pair_ids).each do |kp| + partition_value[kp[:name]] = kp[:value] + end + + # Build guard conditions from handled_types. + # Each class's to_conditions only generates conditions for attributes it actually has. + # We use handled_types (not just fetched messages) so the guard also covers + # message types that haven't appeared yet but would be conflicts. + partition_attrs = partition_value.transform_keys(&:to_sym) + guard_conditions = handled_types.filter_map do |type| + klass = Message.registry[type] + klass&.to_conditions(**partition_attrs) + end.flatten + + last_pos = messages.last.position + guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) + + # replaying: true when all messages are at or below the highest position + # ever acked by this consumer group (i.e. they've been processed before). + replaying = messages.last.position <= cg[:highest_position] + + ClaimResult.new( + offset_id: claimed[:offset_id], + key_pair_ids: key_pair_ids, + partition_key: claimed[:partition_key], + partition_value: partition_value, + messages: messages, + replaying: replaying, + guard: guard + ) + end + + # Acknowledge processing: advance the offset to +position+ and release the claim. + # Also advances the consumer group's +highest_position+ watermark (never decreases), + # which drives the {#claim_next} +replaying+ flag. + # + # @param group_id [String] consumer group identifier + # @param offset_id [Integer] offset ID from the claim result + # @param position [Integer] position of the last processed message + # @return [void] + def ack(group_id, offset_id:, position:) + cg = db[@consumer_groups_table].where(group_id: group_id).first + return unless cg + + db[@offsets_table].where(id: offset_id, consumer_group_id: cg[:id]).update( + last_position: position, + claimed: 0, + claimed_at: nil, + claimed_by: nil + ) + + # Advance the high watermark (never decrease) + if position > cg[:highest_position] + db[@consumer_groups_table].where(id: cg[:id]).update( + highest_position: position, + updated_at: Time.now.iso8601 + ) + end + end + + # Release a claim without advancing the offset. Use for error recovery + # so the partition can be re-claimed and retried. + # + # @param group_id [String] consumer group identifier + # @param offset_id [Integer] offset ID from the claim result + # @return [void] + def release(group_id, offset_id:) + cg = db[@consumer_groups_table].where(group_id: group_id).first + return unless cg + + db[@offsets_table].where(id: offset_id, consumer_group_id: cg[:id]).update( + claimed: 0, + claimed_at: nil, + claimed_by: nil + ) + end + + # Upsert heartbeat timestamps for active workers. + # + # @param worker_ids [Array] worker identifiers + # @param at [Time] timestamp to record (default Time.now) + # @return [Integer] number of workers heartbeated + def worker_heartbeat(worker_ids, at: Time.now) + ids = Array(worker_ids).uniq + return 0 if ids.empty? + + now = at.iso8601 + ids.each do |id| + db.run(<<~SQL) + INSERT INTO #{@workers_table} (id, last_seen) VALUES (#{db.literal(id)}, #{db.literal(now)}) + ON CONFLICT(id) DO UPDATE SET last_seen = #{db.literal(now)} + SQL + end + ids.size + end + + # Release claims held by workers that haven't heartbeated within ttl_seconds. + # + # @param ttl_seconds [Integer] age threshold + # @return [Integer] number of claims released + def release_stale_claims(ttl_seconds: 120) + cutoff = (Time.now - ttl_seconds).iso8601 + + stale_worker_ids = db[@workers_table] + .where(Sequel.lit('last_seen <= ?', cutoff)) + .select_map(:id) + + return 0 if stale_worker_ids.empty? + + db[@offsets_table] + .where(claimed: 1) + .where(claimed_by: stale_worker_ids) + .update(claimed: 0, claimed_at: nil, claimed_by: nil) + end + + # Advance a consumer group's offset for a specific partition to at least +position+. + # Bootstraps the offset row if it doesn't exist yet. + # Unlike {#ack}, this does not require a prior claim. + # + # @param group_id [String] consumer group identifier + # @param partition [Hash{String => String}] partition attribute names and values + # @param position [Integer] advance offset to at least this position + # @return [void] + def advance_offset(group_id, partition:, position:) + cg = db[@consumer_groups_table].where(group_id: group_id).first + return unless cg + + offset_id = ensure_offset_for_partition(cg[:id], partition) + return unless offset_id + + offset = db[@offsets_table].where(id: offset_id).first + return if offset[:last_position] >= position + + db[@offsets_table].where(id: offset_id).update(last_position: position) + + if position > cg[:highest_position] + db[@consumer_groups_table].where(id: cg[:id]).update( + highest_position: position, + updated_at: Time.now.iso8601 + ) + end + end + + # System-wide diagnostics for monitoring and debugging. + # + # @example + # stats = store.stats + # stats.max_position # => 42 + # stats.groups + # # => [ + # # { + # # group_id: "my_decider", + # # status: "active", + # # retry_at: nil, + # # error_context: {}, + # # oldest_processed: 10, + # # newest_processed: 42, + # # partition_count: 3 + # # }, + # # { + # # group_id: "failing_decider", + # # status: "failed", + # # retry_at: nil, + # # error_context: { exception_class: "RuntimeError", exception_message: "boom" }, + # # oldest_processed: 5, + # # newest_processed: 30, + # # partition_count: 2 + # # } + # # ] + # + # @return [Sourced::Stats] max_position and per-group processing state + def stats + groups = db.fetch(<<~SQL).all + SELECT + cg.group_id, + cg.status, + cg.retry_at, + cg.error_context, + COALESCE(MIN(CASE WHEN o.last_position > 0 THEN o.last_position END), 0) AS oldest_processed, + COALESCE(MAX(o.last_position), 0) AS newest_processed, + COUNT(o.id) AS partition_count + FROM #{@consumer_groups_table} cg + LEFT JOIN #{@offsets_table} o ON o.consumer_group_id = cg.id + GROUP BY cg.id, cg.group_id, cg.status, cg.retry_at, cg.error_context + ORDER BY cg.group_id + SQL + + groups.each do |g| + g[:retry_at] = Time.parse(g[:retry_at]) if g[:retry_at] + g[:error_context] = g[:error_context] ? JSON.parse(g[:error_context], symbolize_names: true) : {} + end + + Stats.new(max_position: latest_position, groups: groups) + end + + # List offsets with optional group filtering and cursor-based pagination. + # + # @param group_id [String, nil] filter by consumer group (nil = all groups) + # @param limit [Integer] max offsets per page (default 50) + # @param from_id [Integer, nil] cursor — return offsets with id >= from_id (inclusive) + # @return [Sourced::OffsetsResult] paginated offsets with total_count and fetcher for auto-pagination + def read_offsets(group_id: nil, limit: 50, from_id: nil) + dataset = db[@offsets_table].join(@consumer_groups_table, id: :consumer_group_id) + .select( + Sequel[@offsets_table][:id], + Sequel[@consumer_groups_table][:group_id].as(:group_name), + Sequel[@consumer_groups_table][:status].as(:group_status), + Sequel[@offsets_table][:partition_key], + Sequel[@offsets_table][:last_position], + Sequel[@offsets_table][:claimed], + Sequel[@offsets_table][:claimed_at], + Sequel[@offsets_table][:claimed_by] + ) + .order(Sequel[@offsets_table][:id]) + + count_dataset = db[@offsets_table].join(@consumer_groups_table, id: :consumer_group_id) + + if group_id + dataset = dataset.where(Sequel[@consumer_groups_table][:group_id] => group_id) + count_dataset = count_dataset.where(Sequel[@consumer_groups_table][:group_id] => group_id) + end + + if from_id + dataset = dataset.where(Sequel[@offsets_table][:id] >= from_id) + end + + total_count = count_dataset.count + offsets = dataset.limit(limit).all + + offsets.each do |o| + o[:claimed] = o[:claimed] == 1 + end + + fetcher = ->(next_from_id) { read_offsets(group_id: group_id, limit: limit, from_id: next_from_id) } + + OffsetsResult.new(offsets: offsets, total_count: total_count, fetcher: fetcher) + end + + # Fetch all messages sharing the same correlation_id as the given message. + # Useful for tracing causal chains (command -> events -> reactions). + # + # @param message_id [String] UUID of any message in the correlation chain + # @return [Array] correlated messages ordered by position, or [] if not found + def read_correlation_batch(message_id) + correlation_id = db[@messages_table] + .where(message_id: message_id) + .get(:correlation_id) + return [] unless correlation_id + + db[@messages_table] + .where(correlation_id: correlation_id) + .order(:position) + .map { |row| deserialize(row) } + end + + # Current max position in the message log. + # + # @return [Integer] max position, or 0 if the store is empty + def latest_position + db[@messages_table].max(:position) || 0 + end + + # Delete all data from all tables and reset autoincrement. For testing only. + # + # @return [void] + def clear! + db[@offset_key_pairs_table].delete + db[@offsets_table].delete + db[@consumer_groups_table].delete + db[@message_key_pairs_table].delete + db[@key_pairs_table].delete + db[@messages_table].delete + db[@scheduled_messages_table].delete + db[@workers_table].delete + db.run('DELETE FROM sqlite_sequence') if db.table_exists?(:sqlite_sequence) + end + + private + + # Resolve a group_id argument that is either a String + # or an object responding to +#group_id+. + # + # @param group_id [String, #group_id] + # @return [String] + def resolve_group_id(group_id) + group_id.respond_to?(:group_id) ? group_id.group_id : group_id + end + + # Create offsets eagerly for all registered consumer groups. + # Called inside the append transaction after messages and key_pairs are inserted. + # + # @param messages [Array] messages being appended + # @return [void] + def ensure_offsets_for_registered_groups(messages) + return if @registered_groups.empty? + + # Collect all partition attribute names across registered groups + attr_names = @registered_groups.each_value.flat_map { |gi| gi[:partition_by] || [] }.uniq + + # Pre-fetch relevant key_pair IDs in one query, keyed by "name:value" + kp_id_cache = {} + db[@key_pairs_table].where(name: attr_names).each do |row| + kp_id_cache["#{row[:name]}:#{row[:value]}"] = row[:id] + end + + @registered_groups.each_value do |group_info| + partition_by = group_info[:partition_by] + next unless partition_by + + cg_id = group_info[:cg_id] + seen = Set.new + + messages.each do |msg| + keys = msg.extracted_keys.to_h # {"device_id"=>"dev-1", "name"=>"A"} + next unless partition_by.all? { |attr| keys.key?(attr) } + + values = partition_by.to_h { |attr| [attr, keys[attr]] } + pk = build_partition_key(partition_by, values) + next if seen.include?(pk) + seen << pk + + kp_ids = partition_by.map { |attr| kp_id_cache["#{attr}:#{values[attr]}"] } + create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) + end + end + end + + # Build canonical partition key string from attribute names and values. + # Sorted by attribute name for deterministic uniqueness. + # + # @param partition_by [Array] attribute names + # @param values [Hash{String => String}] attribute values keyed by name + # @return [String] e.g. "course_name:Algebra|user_id:joe" + def build_partition_key(partition_by, values) + partition_by.sort.map { |attr| "#{attr}:#{values[attr]}" }.join('|') + end + + # Scan a bounded window of messages forward from the consumer group's + # discovery_position watermark, find new partition tuples, create offsets + # for them, then advance the watermark. + # + # @param cg_id [Integer] consumer group internal ID + # @param partition_by [Array] sorted attribute names + # @param handled_types [Array] message type strings + # @return [void] + def discover_new_partitions(cg_id, partition_by, handled_types) + cg = db[@consumer_groups_table].where(id: cg_id).first + discovery_pos = cg[:discovery_position] + + types_list = handled_types.map { |t| db.literal(t) }.join(', ') + + # CTE per partition attribute: pre-joins message_key_pairs with key_pairs + # so the main query only self-joins on the CTEs (N joins instead of 2N). + ctes = [] + selects = [] + joins = [] + partition_by.each_with_index do |attr, i| + ctes << <<~CTE + attr#{i} AS ( + SELECT mkp.message_position, kp.id AS kp_id, kp.value AS val + FROM #{@message_key_pairs_table} mkp + JOIN #{@key_pairs_table} kp ON mkp.key_pair_id = kp.id AND kp.name = #{db.literal(attr)} + ) + CTE + selects << "a#{i}.kp_id AS kp_id_#{i}, a#{i}.val AS val_#{i}" + joins << "JOIN attr#{i} a#{i} ON m.position = a#{i}.message_position" + end + + group_by = partition_by.each_index.map { |i| "a#{i}.kp_id" }.join(', ') + + # Discover all partition tuples in the window (no NOT EXISTS — fast). + # Duplicates are filtered in Ruby against known partition_keys, and + # INSERT OR IGNORE handles any remaining races. + sql = <<~SQL + WITH #{ctes.join(",\n")} + SELECT #{selects.join(', ')}, + MIN(m.position) AS min_pos, + MAX(m.position) AS max_pos + FROM #{@messages_table} m + #{joins.join("\n")} + WHERE m.message_type IN (#{types_list}) + AND m.position > #{db.literal(discovery_pos)} + GROUP BY #{group_by} + ORDER BY min_pos ASC + LIMIT #{DISCOVERY_BATCH_SIZE} + SQL + + rows = db.fetch(sql).all + + max_discovered_pos = 0 + + if rows.any? + db.transaction do + rows.each do |row| + values = {} + kp_ids = [] + partition_by.each_with_index do |attr, i| + values[attr] = row[:"val_#{i}"] + kp_ids << row[:"kp_id_#{i}"] + end + + # INSERT OR IGNORE handles duplicates — no need to pre-filter. + # At most DISCOVERY_BATCH_SIZE (100) tuples per call, so the + # cost of re-attempting known offsets is bounded. + create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) + max_discovered_pos = row[:max_pos] if row[:max_pos] > max_discovered_pos + end + end + end + + # Advance watermark to the max position seen in the window (whether new or known). + # This ensures we don't re-scan these messages on the next discovery call. + max_window_pos = rows.any? ? rows.map { |r| r[:max_pos] }.max : 0 + new_watermark = [max_discovered_pos, max_window_pos, latest_position].select { |p| p > 0 }.min || discovery_pos + # If we found a full batch, advance only to the batch's max (more may follow). + # If we found fewer than a batch, we've scanned to the end — advance to latest. + new_watermark = latest_position if rows.size < DISCOVERY_BATCH_SIZE + + if new_watermark > discovery_pos + db[@consumer_groups_table].where(id: cg_id).update( + discovery_position: new_watermark, + updated_at: Time.now.iso8601 + ) + end + end + + # Ensure an offset row exists for a specific partition (by attribute values). + # Resolves key_pair IDs from the key_pairs table; returns nil if any + # partition attribute has no corresponding key_pair (meaning no messages + # with that attribute value exist yet). + # + # @param cg_id [Integer] consumer group internal ID + # @param partition [Hash{String => String}] attribute names and values + # @return [Integer, nil] offset ID, or nil if key_pairs not found + def ensure_offset_for_partition(cg_id, partition) + partition_by = partition.keys.sort + kp_ids = [] + partition_by.each do |attr| + kp = db[@key_pairs_table].where(name: attr, value: partition[attr].to_s).first + return nil unless kp + + kp_ids << kp[:id] + end + + create_offset_with_key_pairs(cg_id, partition_by, partition, kp_ids) + end + + # Create an offset row and its key_pair associations. Idempotent via INSERT OR IGNORE. + # + # @param cg_id [Integer] consumer group internal ID + # @param partition_by [Array] sorted attribute names + # @param values [Hash{String => String}] attribute values keyed by name + # @param kp_ids [Array] key_pair IDs + # @return [Integer] offset ID + def create_offset_with_key_pairs(cg_id, partition_by, values, kp_ids) + partition_key = build_partition_key(partition_by, values) + + db.run(<<~SQL) + INSERT OR IGNORE INTO #{@offsets_table} (consumer_group_id, partition_key, last_position, claimed) + VALUES (#{db.literal(cg_id)}, #{db.literal(partition_key)}, 0, 0) + SQL + + offset_id = db[@offsets_table].where(consumer_group_id: cg_id, partition_key: partition_key).get(:id) + + kp_ids.each do |kp_id| + db.run(<<~SQL) + INSERT OR IGNORE INTO #{@offset_key_pairs_table} (offset_id, key_pair_id) + VALUES (#{db.literal(offset_id)}, #{db.literal(kp_id)}) + SQL + end + + offset_id + end + + # Find the next unclaimed partition with pending messages and claim it. + # Uses OR joins for candidate detection (any matching key_pair), then + # a NOT EXISTS clause to exclude messages that conflict on a shared + # attribute name (same name, different value). This gives AND semantics: + # a message only counts as pending for a partition when every attribute + # it shares with the partition has the same value. + # + # @param cg_id [Integer] consumer group internal ID + # @param handled_types [Array] message type strings + # @param worker_id [String] claiming worker identifier + # @return [Hash, nil] +{ offset_id:, partition_key:, last_position: }+ or nil + def find_and_claim_partition(cg_id, handled_types, worker_id) + types_list = handled_types.map { |t| db.literal(t) }.join(', ') + + row = nil + db[@offsets_table] + .where(consumer_group_id: cg_id, claimed: 0) + .select(:id, :partition_key, :last_position) + .order(:last_position) + .each do |offset| + + pending = db.fetch(<<~SQL).first + SELECT 1 + FROM #{@offset_key_pairs_table} okp + JOIN #{@message_key_pairs_table} mkp ON okp.key_pair_id = mkp.key_pair_id + JOIN #{@messages_table} m ON mkp.message_position = m.position + WHERE okp.offset_id = #{db.literal(offset[:id])} + AND m.position > #{db.literal(offset[:last_position])} + AND m.message_type IN (#{types_list}) + GROUP BY m.position + HAVING COUNT(*) = ( + SELECT COUNT(*) FROM #{@offset_key_pairs_table} WHERE offset_id = #{db.literal(offset[:id])} + ) + LIMIT 1 + SQL + + if pending + row = { offset_id: offset[:id], partition_key: offset[:partition_key], last_position: offset[:last_position] } + break + end + end + + return nil unless row + + now = Time.now.iso8601 + updated = db[@offsets_table] + .where(id: row[:offset_id], claimed: 0) + .update(claimed: 1, claimed_at: now, claimed_by: worker_id) + + return nil if updated == 0 + + row + end + + # Fetch messages for a partition using conditional AND semantics. + # For each candidate message, it must match ALL of the partition's attributes + # that the message itself has. Messages with a single partition attribute match + # on that one; messages with multiple must match all of them. + # + # @param key_pair_ids [Array] partition key_pair IDs + # @param last_position [Integer] fetch messages after this position + # @param handled_types [Array] message type strings + # @param limit [Integer, nil] max messages to return (nil = unlimited) + # @return [Array] + def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: nil) + return [] if key_pair_ids.empty? + + kp_ids_list = key_pair_ids.map { |id| db.literal(id) }.join(', ') + types_list = handled_types.map { |t| db.literal(t) }.join(', ') + + # CTE pre-resolves which attribute names the partition's key_pairs cover. + # The main query uses this to count-match: a message qualifies when it + # matches as many partition key_pairs as it has attributes in common with + # the partition (AND semantics for shared attributes). + sql = <<~SQL + WITH partition_attr_names AS ( + SELECT DISTINCT name FROM #{@key_pairs_table} WHERE id IN (#{kp_ids_list}) + ) + SELECT DISTINCT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at + FROM #{@messages_table} m + WHERE m.position > #{db.literal(last_position)} + AND m.message_type IN (#{types_list}) + AND EXISTS ( + SELECT 1 FROM #{@message_key_pairs_table} mkp + WHERE mkp.message_position = m.position + AND mkp.key_pair_id IN (#{kp_ids_list}) + ) + AND ( + SELECT COUNT(*) FROM #{@message_key_pairs_table} mkp + WHERE mkp.message_position = m.position + AND mkp.key_pair_id IN (#{kp_ids_list}) + ) = ( + SELECT COUNT(DISTINCT kp_msg.name) + FROM #{@message_key_pairs_table} mkp2 + JOIN #{@key_pairs_table} kp_msg ON mkp2.key_pair_id = kp_msg.id + WHERE mkp2.message_position = m.position + AND kp_msg.name IN (SELECT name FROM partition_attr_names) + ) + ORDER BY m.position ASC + SQL + sql += " LIMIT #{db.literal(limit)}" if limit + + db.fetch(sql).map { |row| deserialize(row) } + end + + # Core query logic shared by {#read}, {#read_all}, and {#check_conflicts}. + # Resolves key_pair IDs from conditions, then queries messages. + # Attributes within each condition are AND'd; conditions are OR'd. + # + # @param conditions [Array] + # @param after_position [Integer, nil] only include messages after this position (exclusive) + # @param limit [Integer, nil] + # @param order [:asc, :desc] position order (default :asc) + # @return [Array] + def query_messages(conditions, after_position: nil, limit: nil, order: :asc) + subqueries = condition_position_subqueries(conditions, after_position: after_position) + return [] if subqueries.empty? + + union = subqueries.join(" UNION ") + direction = order == :desc ? "DESC" : "ASC" + + sql = <<~SQL + SELECT m.position, m.message_id, m.message_type, m.causation_id, m.correlation_id, m.payload, m.metadata, m.created_at + FROM #{@messages_table} m + WHERE m.position IN (#{union}) + ORDER BY m.position #{direction} + SQL + sql += " LIMIT #{db.literal(limit)}" if limit + + db.fetch(sql).map { |row| deserialize(row) } + end + + # Check for conflicting messages after a given position. + # + # @param conditions [Array] + # @param after_position [Integer] + # @return [Array] + def check_conflicts(conditions, after_position) + return [] if conditions.empty? + + query_messages(conditions, after_position:) + end + + # Max position among messages matching the given conditions. + # Attributes within each condition are AND'd; conditions are OR'd. + # Returns after_position (or latest_position) if no matches. + # + # @param conditions [Array] + # @param after_position [Integer, nil] only consider messages after this position (exclusive) + # @return [Integer] + def max_position_for(conditions, after_position: nil) + return after_position || latest_position if conditions.empty? + + subqueries = condition_position_subqueries(conditions, after_position: after_position) + return after_position || latest_position if subqueries.empty? + + union = subqueries.join(" UNION ") + row = db.fetch("SELECT MAX(position) AS max_pos FROM (#{union})").first + row[:max_pos] || after_position || latest_position + end + + # Build per-condition position subqueries with AND-within/OR-across semantics. + # Resolves key_pair IDs, then builds one SQL subquery per condition. + # Each subquery selects positions where the message matches ALL attrs in the condition. + # + # @param conditions [Array] + # @param after_position [Integer, nil] only include positions after this (exclusive) + # @return [Array] SQL subquery strings (empty if no conditions can match) + def condition_position_subqueries(conditions, after_position: nil) + all_lookups = conditions.flat_map { |c| c.attrs.map { |k, v| [k.to_s, v.to_s] } }.uniq + return [] if all_lookups.empty? + + or_clauses = all_lookups.map { |n, v| "(name = #{db.literal(n)} AND value = #{db.literal(v)})" } + key_rows = db.fetch("SELECT id, name, value FROM #{@key_pairs_table} WHERE #{or_clauses.join(' OR ')}").all + + key_pair_index = {} + key_rows.each { |r| key_pair_index[[r[:name], r[:value]]] = r[:id] } + + position_filter = after_position ? "AND m.position > #{db.literal(after_position)}" : "" + + conditions.filter_map do |c| + kp_ids = c.attrs.filter_map { |k, v| key_pair_index[[k.to_s, v.to_s]] } + next if kp_ids.size < c.attrs.size + + kp_ids_list = kp_ids.map { |id| db.literal(id) }.join(', ') + + <<~SQL + SELECT m.position + FROM #{@messages_table} m + JOIN #{@message_key_pairs_table} mkp ON m.position = mkp.message_position + WHERE m.message_type = #{db.literal(c.message_type)} + AND mkp.key_pair_id IN (#{kp_ids_list}) + #{position_filter} + GROUP BY m.position + HAVING COUNT(DISTINCT mkp.key_pair_id) = #{kp_ids.size} + SQL + end + end + + # Deserialize a database row into a {PositionedMessage}. + # Looks up the message class from the registry; falls back to base {Message}. + # + # @param row [Hash] database row with :position, :message_id, :message_type, :causation_id, :correlation_id, :payload, :metadata, :created_at + # @return [PositionedMessage] + def deserialize(row) + payload = JSON.parse(row[:payload], symbolize_names: true) + metadata = row[:metadata] ? JSON.parse(row[:metadata], symbolize_names: true) : {} + + klass = Message.registry[row[:message_type]] + attrs = { + id: row[:message_id], + type: row[:message_type], + causation_id: row[:causation_id], + correlation_id: row[:correlation_id], + created_at: row[:created_at], + metadata: metadata, + payload: payload + } + + msg = if klass + klass.new(attrs) + else + Message.new(attrs) + end + + PositionedMessage.new(msg, row[:position]) + end + end +end diff --git a/lib/sourced/supervisor.rb b/lib/sourced/supervisor.rb index d1883188..e4b620e9 100644 --- a/lib/sourced/supervisor.rb +++ b/lib/sourced/supervisor.rb @@ -1,24 +1,18 @@ # frozen_string_literal: true -require 'async' -require 'console' require 'sourced/dispatcher' -require 'sourced/house_keeper' module Sourced - # The Supervisor manages background workers that process events and commands. - # It uses a Dispatcher for signal-driven worker dispatch and HouseKeepers - # for maintenance tasks (heartbeats, stale claim release, scheduled messages). + # Top-level process entry point for background workers. + # Creates a {Dispatcher} (which embeds Workers, CatchUpPoller, notifier, + # and StaleClaimReaper) and spawns it into an executor. # - # The supervisor automatically sets up signal handlers for INT and TERM signals - # to ensure workers shut down cleanly when the process is terminated. - # - # @example Start a supervisor with 10 workers - # Sourced::Supervisor.start(count: 10) + # @example Start with defaults + # Sourced::Supervisor.start(router: my_router) # # @example Create and start manually - # supervisor = Sourced::Supervisor.new(count: 5) - # supervisor.start # This will block until interrupted + # supervisor = Sourced::Supervisor.new(router: my_router, count: 4) + # supervisor.start class Supervisor # Start a new supervisor instance with the given options. # @@ -28,84 +22,64 @@ def self.start(...) new(...).start end - # Initialize a new supervisor instance. - # + # @param router [Sourced::Router] the router providing reactors and store # @param logger [Object] Logger instance for supervisor output # @param count [Integer] Number of worker fibers to spawn # @param batch_size [Integer] Messages per backend fetch # @param max_drain_rounds [Integer] Max drain iterations per reactor pickup # @param catchup_interval [Numeric] Seconds between catch-up polls + # @param housekeeping_interval [Numeric] Seconds between heartbeat/reap cycles + # @param claim_ttl_seconds [Integer] Stale claim age threshold in seconds # @param executor [Object] Executor instance for running concurrent workers def initialize( + router: Sourced.router, logger: Sourced.config.logger, count: Sourced.config.worker_count, - batch_size: Sourced.config.worker_batch_size, + batch_size: Sourced.config.batch_size, max_drain_rounds: Sourced.config.max_drain_rounds, catchup_interval: Sourced.config.catchup_interval, - housekeeping_count: Sourced.config.housekeeping_count, housekeeping_interval: Sourced.config.housekeeping_interval, - housekeeping_heartbeat_interval: Sourced.config.housekeeping_heartbeat_interval, - housekeeping_claim_ttl_seconds: Sourced.config.housekeeping_claim_ttl_seconds, - executor: Sourced.config.executor, - router: Sourced::Router + claim_ttl_seconds: Sourced.config.claim_ttl_seconds, + executor: Sourced.config.executor ) + @router = router @logger = logger @count = count @batch_size = batch_size @max_drain_rounds = max_drain_rounds @catchup_interval = catchup_interval - @housekeeping_count = housekeeping_count @housekeeping_interval = housekeeping_interval - @housekeeping_heartbeat_interval = housekeeping_heartbeat_interval - @housekeeping_claim_ttl_seconds = housekeeping_claim_ttl_seconds + @claim_ttl_seconds = claim_ttl_seconds @executor = executor - @router = router end - # Start the supervisor, dispatcher, and housekeepers. + # Start the supervisor and dispatcher. # This method blocks until the supervisor receives a shutdown signal. def start - logger.info("Starting supervisor with #{@count} workers and #{@executor} executor") + logger.info("Sourced::Supervisor: starting with #{@count} workers and #{@executor} executor") set_signal_handlers @dispatcher = Dispatcher.new( - router: router, + router: @router, worker_count: @count, batch_size: @batch_size, max_drain_rounds: @max_drain_rounds, catchup_interval: @catchup_interval, + housekeeping_interval: @housekeeping_interval, + claim_ttl_seconds: @claim_ttl_seconds, logger: logger ) - @housekeepers = @housekeeping_count.times.map do |i| - HouseKeeper.new( - logger:, - backend: router.backend, - name: "HouseKeeper-#{i}", - interval: @housekeeping_interval, - heartbeat_interval: @housekeeping_heartbeat_interval, - claim_ttl_seconds: @housekeeping_claim_ttl_seconds, - worker_ids_provider: -> { @dispatcher.workers.map(&:name) } - ) - end - @executor.start do |task| - @housekeepers.each do |hk| - task.spawn do - hk.work - end - end - @dispatcher.spawn_into(task) end end # Stop all components gracefully. def stop - logger.info("Stopping dispatcher and #{@housekeepers&.size || 0} house-keepers") + logger.info('Sourced::Supervisor: stopping dispatcher') @dispatcher&.stop - @housekeepers&.each(&:stop) - logger.info('All workers stopped') + logger.info('Sourced::Supervisor: all workers stopped') end # Set up signal handlers for graceful shutdown. @@ -116,6 +90,6 @@ def set_signal_handlers private - attr_reader :logger, :router + attr_reader :logger end end diff --git a/lib/sourced/sync.rb b/lib/sourced/sync.rb index 150ab4bb..5ba4b647 100644 --- a/lib/sourced/sync.rb +++ b/lib/sourced/sync.rb @@ -1,87 +1,88 @@ # frozen_string_literal: true module Sourced - # This mixin provides a .sync macro to registering blocks - # that will run within a transaction. Ie. in Actors when appending events to the backend, - # or projectors when persisting their state. - # @example - # - # class CartActor < Sourced::Actor - # # Run this block within a transaction - # # when appending messages to storage - # sync do |state:, command:, events:| - # # Do something here, like updating a view, sending an email, etc. - # end - # end - # - # When given a ReactorInterface, it will run that reactor synchronously - # and ACK the offsets for the embedded reactor and consumed events. - # This is so that reactors can be run in a strong consistency manner - # within an Actor lifecycle. - # ACKing the events will ensure that the events are not reprocessed - # if the child reactor is later moved to eventually consistent processing. - # @example - # - # class CartActor < Sourced::Actor - # # The CartListings projector - # # will be run synchronously when events are appended by this Actor - # # Any error raised by the projector will cause the transaction to rollback - # sync CartListings - # end + # Sync mixin for reactors. + # Registers blocks that run within the store transaction (+sync+) + # or after the transaction commits (+after_sync+). module Sync - CallableInterface = Sourced::Types::Interface[:call] - def self.included(base) super base.extend ClassMethods end - def sync_blocks_with(**args) - self.class.sync_blocks.map do |callable| - case callable - when Proc - proc { instance_exec(**args, &callable) } - when CallableInterface - proc { callable.call(**args) } - else - raise ArgumentError, "Not a valid sync block: #{callable.inspect}" - end + # Build {Actions::Sync} wrappers for all registered +sync+ blocks. + # + # @param args [Hash] keyword arguments forwarded to each block + # @return [Array] + def sync_actions(**args) + self.class.sync_blocks.map do |block| + Actions::Sync.new(proc { instance_exec(**args, &block) }) end end - def sync_actions_with(**args) - sync_blocks_with(**args).map do |bl| - Actions::Sync.new(bl) + # Build {Actions::AfterSync} wrappers for all registered +after_sync+ blocks. + # + # @param args [Hash] keyword arguments forwarded to each block + # @return [Array] + def after_sync_actions(**args) + self.class.after_sync_blocks.map do |block| + Actions::AfterSync.new(proc { instance_exec(**args, &block) }) end end + # Build all sync and after_sync actions together. + # + # @param args [Hash] keyword arguments forwarded to each block + # @return [Array] + def collect_actions(**args) + sync_actions(**args) + after_sync_actions(**args) + end + module ClassMethods + # @api private def inherited(subclass) super sync_blocks.each do |blk| subclass.sync_blocks << blk end + after_sync_blocks.each do |blk| + subclass.after_sync_blocks << blk + end end + # @return [Array] registered sync blocks def sync_blocks @sync_blocks ||= [] end - # The .sync macro - # @example + # Register a block to run inside the store transaction. + # + # The block receives the same keyword arguments as the reactor's + # action-building step (e.g. +state:+, +messages:+, +events:+ + # for deciders, or +state:+, +messages:+, +replaying:+ for projectors). + # + # @yield [**args] side-effect block executed within the transaction + # @return [void] + def sync(&block) + sync_blocks << block + end + + # @return [Array] registered after_sync blocks + def after_sync_blocks + @after_sync_blocks ||= [] + end + + # Register a block to run after the store transaction commits. # - # sync do |state, command, events| - # # Do something here, like updating a view, sending an email, etc. - # end + # Use this for side effects that should only happen on successful + # commit (e.g. sending emails, HTTP calls, pushing to external queues). # - # sync CartListings + # The block receives the same keyword arguments as +sync+. # - # @param callable [Nil, Proc, ReactorInterface, CallableInterface] the block to run - # @yieldparam state [Object] the state of the host class - # @yieldparam command [Object, Nil] the command being processed - # @yieldparam events [Array] the events being appended - def sync(callable = nil, &block) - sync_blocks << (block || callable) + # @yield [**args] side-effect block executed after transaction commit + # @return [void] + def after_sync(&block) + after_sync_blocks << block end end end diff --git a/lib/sourced/testing/rspec.rb b/lib/sourced/testing/rspec.rb index af0de260..1f07cb6c 100644 --- a/lib/sourced/testing/rspec.rb +++ b/lib/sourced/testing/rspec.rb @@ -1,101 +1,44 @@ # frozen_string_literal: true -require 'sourced/message' - -# An RSpec module with helpers to test Sourced reactors -# RSpec.describe Payment do -# subject(:payment) { Payment.new(id: 'payment-1') } -# -# it 'starts a payment' do -# with_actor(payment) -# .when(Payment::Start, order_id: 'o1', amount: 1000) -# .then(Payment::Started.build(payment.id, order_id: 'o1', amount: 1000)) -# end -# -# it 'is a no-op if payment already started' do -# with_actor(payment) -# .given(Payment::Started, order_id: 'o1', amount: 1000) -# .when(Payment::Start, order_id: 'o1', amount: 1000) -# .then([]) -# end -# -# it 'confirms a started payment' do -# with_actor(payment) -# .given(Payment::Started, order_id: 'o1', amount: 1000) -# .when(Payment::Confirm) -# .then(Payment::Confirmed.build(payment.id)) -# end -# -# it 'does not confirm a payment that has not started' do -# with_actor(payment) -# .when(Payment::Confirm) -# .then([]) -# end -# -# # Evaluating block with .then -# it 'calls API' do -# with_actor(payment) -# .when(Payment::Confirm) -# .then do |actions| -# expect(api_request).to have_been_requested -# end -# end -# end +require 'sourced' +require 'sourced/store' + module Sourced module Testing module RSpec - ERROR = proc do |args| - raise ArgumentError, "no support for #{args.inspect}" - end - NONE = [].freeze - module MessageBuilder - private - - def stream_id = nil - - def build_messages(*messages) - messages = build_message(*messages) do |arr| - Array(arr).map { |e| build_message(*e) } - end - Array(messages) - end - - def build_message(*args, &fallback) - fallback ||= ERROR - - case args - in [Class => klass, Hash => payload] - klass.build(stream_id, payload) - in [Class => klass] - klass.build(stream_id) - in [Sourced::Message => mm] - mm - in [NONE] - NONE - else - fallback.(args) - end - end - - def run_sync_actions(actions) - return if @sync_run - - actions.filter { |a| Sourced::Actions::Sync === a }.each(&:call) - @sync_run = true - end - - def partition_actions(actions) - actions.partition do |a| - a.respond_to?(:messages) - end - end + # Entry point for reactors GWT tests. + # + # Works with any reactor that responds to the standard + # handle_batch(partition_values, new_messages, history:, replaying:) + # contract (Deciders, Projectors, DurableWorkflows). + # + # @param reactor_class [Class] a reactors class + # @param partition_attrs [Hash] partition key-value pairs (e.g. device_id: 'd1') + # @return [GWT] + # + # @example Decider + # with_reactor(MyDecider, device_id: 'd1') + # .given(DeviceRegistered, device_id: 'd1', name: 'Sensor') + # .when(BindDevice, device_id: 'd1', asset_id: 'a1') + # .then(DeviceBound, device_id: 'd1', asset_id: 'a1') + # + # @example Projector — assert state via block + # with_reactor(MyProjector, list_id: 'L1') + # .given(ItemAdded, list_id: 'L1', name: 'Apple') + # .then { |r| expect(r.state[:items]).to eq(['Apple']) } + def with_reactor(reactor_class, **partition_attrs) + GWT.new(reactor_class, **partition_attrs) end - class MessageMatcher - MessageArray = Sourced::Types::Array[Sourced::Message] + # Uniform result yielded to +.then+ / +.then!+ block callbacks. + # Gives access to both the reactor's produced action pairs / messages + # (what deciders typically assert on) and the evolved state (what + # projectors typically assert on). + RunResult = Data.define(:pairs, :messages, :state) + class MessageMatcher def initialize(expected_messages) @expected_messages = Array(expected_messages) @errors = [] @@ -103,11 +46,6 @@ def initialize(expected_messages) end def matches?(actual_messages) - if !(MessageArray === actual_messages) - @errors << "expected an array of Sourced messages, but got #{actual_messages.inspect}" - return false - end - if @expected_messages.size != actual_messages.size @errors << "Expected #{@expected_messages.size} messages, but got #{actual_messages.size}" @errors << actual_messages.inspect @@ -116,8 +54,7 @@ def matches?(actual_messages) @expected_messages.each.with_index do |expected, idx| actual = actual_messages[idx] - @mismatching[idx] << "expected a #{expected.class}, got #{actual.class}" unless actual.class === expected - @mismatching[idx] << "expected stream_id '#{expected.stream_id}', got '#{actual.stream_id}'" unless expected.stream_id == actual.stream_id + @mismatching[idx] << "expected a #{expected.class}, got #{actual.class}" unless actual.class == expected.class @mismatching[idx] << "expected payload #{expected.payload.to_h.inspect}, got #{actual.payload.to_h.inspect}" unless expected.payload == actual.payload end @@ -139,231 +76,206 @@ def failure_message end end - def match_sourced_messages(expected_messages) - MessageMatcher.new(expected_messages) - end - - FinishedTestCase = Class.new(StandardError) - class GWT - include MessageBuilder - - def initialize(actor) - @actor = actor - @when = nil - @actual = nil - @sync_run = false + def initialize(reactor_class, **partition_attrs) + @reactor_class = reactor_class + @partition_values = partition_attrs + @given_messages = [] + @when_messages = [] + @asserted = false end - def given(*args) - raise FinishedTestCase, 'test case already asserted, cannot add more events to it' if @actual + # Accumulate history / context messages. These become +history.messages+ + # passed to the reactor's +handle_batch+. + # + # @param klass_or_instance [Class, Sourced::Message] + # @param payload [Hash] + # @return [self] + def given(klass_or_instance = nil, **payload) + raise 'test case already asserted' if @asserted - message = build_message(*args) - @actor.evolve(message) + @given_messages << build_message(klass_or_instance, **payload) self end - def when(*args) - raise FinishedTestCase, 'test case already asserted, cannot add more events to it' if @actual + alias_method :and, :given + + # The batch of new messages to process via +handle_batch+. Can be + # called multiple times to supply several messages. + # + # @param klass_or_instance [Class, Sourced::Message] + # @param payload [Hash] + # @return [self] + def when(klass_or_instance = nil, **payload) + raise 'test case already asserted' if @asserted - @when = build_message(*args) + @when_messages << build_message(klass_or_instance, **payload) self end - def and(...) = given(...) - - # Like #then, but run any sync actions - def then!(*expected, &block) - run_then(true, *expected, &block) + # Assert expected outcomes. + # + # - Pass message class + payload pairs (or instances) to assert + # produced messages. + # - Pass +[]+ or +NONE+ to assert no messages. + # - Pass an Exception class (+ optional message) to assert the + # reactor raised. + # - Pass a block to receive a {RunResult} for custom assertions. + # + # @return [self] + def then(*expected, **payload, &block) + run_then(false, *expected, **payload, &block) end - def then(*expected, &block) - run_then(false, *expected, &block) + # Like #then, but runs Sync / AfterSync actions before computing + # the result yielded to the block (or before extracting messages). + def then!(*expected, **payload, &block) + run_then(true, *expected, **payload, &block) end private - def stream_id = @actor.id - - def run_then(sync, *expected, &block) - # If expecting an exception class or instance, assert that handle raises it - if expected.size == 1 && exception_expectation?(expected[0]) - expect_exception(expected[0]) - return self + def build_message(klass_or_instance, **payload) + if klass_or_instance.is_a?(Sourced::Message) + klass_or_instance + else + klass_or_instance.new(payload: payload) end + end - # Actor instances maintain their own state (#given above calls #evolve on them) - # ReactorAdapter also keeps its own history via #evolve - # So here we satisfy the expected :history arg, but don't provide historical messages - @actual ||= @actor.handle(@when, history: []) - actions_with_messages, actions_without_messages = partition_actions(@actual) - run_sync_actions(actions_without_messages) if sync - - if block_given? - block.call(@actual) - return self - end + def run_then(sync, *expected, **payload, &block) + @asserted = true - expected = build_messages(*expected) - matcher = MessageMatcher.new(expected) - actual_messages = actions_with_messages.flat_map(&:messages) - if !matcher.matches?(actual_messages) - ::RSpec::Expectations.fail_with(matcher.failure_message) + # Shorthand: .then(Class, key: val) → build message from class + payload + if expected.size == 1 && expected[0].is_a?(Class) && !(expected[0] < ::Exception) && payload.any? + expected = [expected[0].new(payload: payload)] end - self - end - - def exception_expectation?(arg) - case arg - when Class - arg < ::Exception - when ::Exception - true - else - false + # Exception expectation + if expected.size >= 1 && exception_expectation?(expected[0]) + expect_exception(expected[0], expected[1]) + return self end - end - def expect_exception(expected) - exception_class = expected.is_a?(Class) ? expected : expected.class + pairs = run_handle_batch - begin - @actor.handle(@when, history: []) - rescue exception_class => e - if expected.is_a?(::Exception) - if e.message != expected.message - ::RSpec::Expectations.fail_with( - "expected #{exception_class} with message #{expected.message.inspect}, " \ - "but got #{e.message.inspect}" - ) - end + if sync + pairs.each do |actions, _| + Array(actions).select { |a| + a.is_a?(Sourced::Actions::Sync) || a.is_a?(Sourced::Actions::AfterSync) + }.each(&:call) end - return end - ::RSpec::Expectations.fail_with("expected #{exception_class} to be raised, but nothing was raised") - end - end - - # Make a base Reactor .handle() interface support #evolve, #decide - class ReactorAdapter - attr_reader :id - - def initialize(reactor, id) - @reactor = reactor - @id = id - @history = [] - end - - def evolve(events) - @history += Array(events) - end - - # Include :history for compatibility - # but we keep out own history - def handle(command, history: []) - @reactor.handle(command, history: [*@history, command]) - end - end - - ActorInterface = Sourced::Types::Interface[:decide, :evolve, :handle] - - def with_reactor(*args) - actor = case args - in [ActorInterface => a] - a - in [Sourced::ReactorInterface => reactor, String => id] - ReactorAdapter.new(reactor, id) - in [Sourced::ReactorInterface => reactor] - ReactorAdapter.new(reactor, Sourced.new_stream_id) - end - - GWT.new(actor) - end + if block_given? + block.call(RunResult.new(pairs: pairs, messages: extract_messages(pairs), state: compute_state(sync: sync))) + return self + end - class Stage - include MessageBuilder + actual_messages = extract_messages(pairs) + expected_msgs = build_expected(*expected) - attr_reader :router + if expected_msgs.empty? + unless actual_messages.empty? + ::RSpec::Expectations.fail_with( + "Expected no messages, but got #{actual_messages.size}: #{actual_messages.inspect}" + ) + end + return self + end - def initialize(reactors, router: nil, logger: Logger.new(nil)) - @reactors = reactors - @router ||= Sourced::Router.new( - backend: Sourced::Backends::TestBackend.new, - logger: - ) - @reactors.each do |r| - @router.register(r) + matcher = MessageMatcher.new(expected_msgs) + unless matcher.matches?(actual_messages) + ::RSpec::Expectations.fail_with(matcher.failure_message) end - @stream_id = nil - @called = false - @when = nil - @sync_run = false - @given_count = 0 - end - def with_stream(stream_id) - @stream_id = stream_id self end - def given(*args) - raise FinishedTestCase, 'test case already asserted, cannot add more events to it' if @called - - message = build_message(*args) - @router.backend.append_next_to_stream(message.stream_id, [message]) - @given_count += 1 - self + def run_handle_batch + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 0) + history = Sourced::ReadResult.new(messages: @given_messages, guard: guard) + @reactor_class.handle_batch( + @partition_values, + @when_messages, + history: history, + replaying: false + ) end - def when(*args) - raise FinishedTestCase, 'test case already asserted, cannot add more events to it' if @called - - @when = build_message(*args) - @router.backend.append_next_to_stream(@when.stream_id, [@when]) - self + # Build an instance and evolve it with all known messages so the + # caller can assert on state regardless of reactor type. For reactors + # whose +handle_batch+ evolves its own instance (Decider, Projector, + # DurableWorkflow), this is an independent, predictable computation. + # When +sync+ is true, also runs the reactor's Sync / AfterSync + # blocks against this instance so their state mutations are visible. + def compute_state(sync: false) + instance = @reactor_class.new(@partition_values) + return nil unless instance.respond_to?(:evolve) + + messages = @given_messages + @when_messages + instance.evolve(messages) + run_sync_on(instance, messages) if sync + instance.state end - def and(...) = given(...) - - def run - @called = true - router.drain - - self + # Invoke Sync / AfterSync blocks against +instance+. Per-block + # kwarg signatures vary by reactor type (deciders expect +events:+, + # projectors expect +replaying:+); we inspect each block's + # parameters and pass only what it declares. + def run_sync_on(instance, messages) + all_args = { state: instance.state, messages: messages, events: [], replaying: false } + klass = instance.class + blocks = [] + blocks.concat(klass.sync_blocks) if klass.respond_to?(:sync_blocks) + blocks.concat(klass.after_sync_blocks) if klass.respond_to?(:after_sync_blocks) + blocks.each do |block| + wanted = block.parameters.select { |type, _| type == :keyreq || type == :key }.map(&:last) + instance.instance_exec(**all_args.slice(*wanted), &block) + end end - # Like #then, but run any sync actions - def then!(&block) - self.then(&block) + def extract_messages(pairs) + pairs.flat_map { |actions, _| + Array(actions) + .select { |a| a.respond_to?(:messages) } + .flat_map(&:messages) + } end - def then(&block) - run + def build_expected(*args) + return [] if args == [[]] || args == [NONE] + return [] if args.empty? - if block_given? - if block.arity == 2 - new_messages = backend.messages[(@given_count - 1)..] - block.call(self, new_messages) + args.map do |arg| + case arg + when Sourced::Message + arg else - block.call(self) + raise ArgumentError, "unsupported expected message: #{arg.inspect}" end - return self end - - self end - def backend = router.backend - - private + def exception_expectation?(arg) + arg.is_a?(Class) && arg < ::Exception + end - attr_reader :stream_id - end + def expect_exception(exception_class, message = nil) + begin + run_handle_batch + rescue exception_class => e + if message && e.message != message + ::RSpec::Expectations.fail_with( + "expected #{exception_class} with message #{message.inspect}, " \ + "but got #{e.message.inspect}" + ) + end + return + end - def with_reactors(*reactors) - Stage.new(reactors) + ::RSpec::Expectations.fail_with("expected #{exception_class} to be raised, but nothing was raised") + end end end end diff --git a/lib/sourced/topology.rb b/lib/sourced/topology.rb index 1f9f1455..489b7eae 100644 --- a/lib/sourced/topology.rb +++ b/lib/sourced/topology.rb @@ -7,6 +7,9 @@ end module Sourced + # Analyzes registered reactors (Deciders and Projectors) and builds a + # flat array of node structs describing the message flow graph. Enables + # visualization, introspection, and Event Modeling diagram generation. module Topology CommandNode = Struct.new(:type, :id, :name, :group_id, :produces, :schema, keyword_init: true) EventNode = Struct.new(:type, :id, :name, :group_id, :produces, :schema, keyword_init: true) @@ -14,24 +17,24 @@ module Topology ReadModelNode = Struct.new(:type, :id, :name, :group_id, :consumes, :produces, :schema, keyword_init: true) # Analyze registered reactors and build the topology graph. - # @param reactors [Enumerable] reactor classes (Actors, Projectors) - # @return [Array] + # + # @param reactors [Enumerable] reactor classes (Deciders and/or Projectors) + # @return [Array] def self.build(reactors) analyzer = SourceAnalyzer.new nodes = [] - command_ids = {} # type_string => CommandNode (dedup across reactors) - event_nodes = {} # type_string => EventNode (dedup) + command_ids = {} + event_nodes = {} reactors.each do |reactor| - group_id = reactor.consumer_info.group_id + group_id = reactor.group_id - # Command nodes (actors only) if reactor.respond_to?(:handled_commands) reactor.handled_commands.each do |cmd_class| next if command_ids.key?(cmd_class.type) produced_refs = analyzer.events_produced_by(reactor, cmd_class) - produced_types = resolve_refs(produced_refs, reactor, :event) + produced_types = resolve_refs(produced_refs, reactor) schema = extract_schema(cmd_class) cmd_node = CommandNode.new( @@ -45,12 +48,11 @@ def self.build(reactors) nodes << cmd_node command_ids[cmd_class.type] = cmd_node - # Register event nodes from produces produced_types.each do |evt_type| next if event_nodes.key?(evt_type) next if command_ids.key?(evt_type) - evt_class = find_event_class(evt_type, reactor) + evt_class = find_event_class(evt_type) next unless evt_class event_nodes[evt_type] = EventNode.new( @@ -65,10 +67,8 @@ def self.build(reactors) end end - # Event nodes from evolve handlers (covers projectors and events not yet seen) if reactor.respond_to?(:handled_messages_for_evolve) reactor.handled_messages_for_evolve.each do |evt_class| - # Skip command classes that ended up in evolve handlers next if evt_class < Sourced::Command evt_type = evt_class.type @@ -90,15 +90,13 @@ def self.build(reactors) rm_id = is_projector ? "#{Sourced::Types::ModuleToMessageType.parse(group_id)}-rm" : nil projector_aut_ids = [] - # Automation nodes from reactions if reactor.respond_to?(:handled_messages_for_react) catch_all_events = reactor.respond_to?(:catch_all_react_events) ? reactor.catch_all_react_events : Set.new specific_events = reactor.handled_messages_for_react.reject { |e| catch_all_events.include?(e) } - # Specific reactions: one automation node per event specific_events.each do |evt_class| produced_refs = analyzer.commands_dispatched_by(reactor, evt_class) - produced_types = resolve_refs(produced_refs, reactor, :command) + produced_types = resolve_refs(produced_refs, reactor) aut_id = "#{evt_class.type}-#{group_id}-aut" projector_aut_ids << aut_id if is_projector @@ -113,10 +111,9 @@ def self.build(reactors) ) end - # Catch-all reaction: single automation node for all catch-all events if catch_all_events.any? produced_refs = analyzer.commands_dispatched_by(reactor, catch_all_events.first) - produced_types = resolve_refs(produced_refs, reactor, :command) + produced_types = resolve_refs(produced_refs, reactor) group_type_id = Sourced::Types::ModuleToMessageType.parse(group_id) aut_id = "#{group_type_id}-aut" @@ -139,7 +136,6 @@ def self.build(reactors) end end - # ReadModel nodes (projectors only) if is_projector consumes = reactor.handled_messages_for_evolve .reject { |c| c < Sourced::Command } @@ -160,22 +156,18 @@ def self.build(reactors) nodes + event_nodes.values end - # Resolve AST references to message type strings. - # @param refs [Array] e.g. [[:const, "ThingCreated"], [:symbol, "notify"]] - # @param reactor [Class] reactor class for namespace resolution - # @param kind [Symbol] :event or :command - # @return [Array] resolved type strings - def self.resolve_refs(refs, reactor, kind) + # @api private + def self.resolve_refs(refs, reactor) refs.filter_map do |ref_type, ref_value| - resolve_ref(ref_type, ref_value, reactor, kind) + resolve_ref(ref_type, ref_value, reactor) end.uniq end # @api private - def self.resolve_ref(ref_type, ref_value, reactor, kind) + def self.resolve_ref(ref_type, ref_value, reactor) case ref_type when :symbol - resolve_symbol_ref(ref_value, reactor, kind) + resolve_symbol_ref(ref_value, reactor) when :const resolve_const_ref(ref_value, reactor) when :const_path @@ -185,17 +177,11 @@ def self.resolve_ref(ref_type, ref_value, reactor, kind) end end - def self.resolve_symbol_ref(symbol_name, reactor, kind) + def self.resolve_symbol_ref(symbol_name, reactor) sym = symbol_name.to_sym - if kind == :event && reactor.respond_to?(:resolve_message_class) + if reactor.respond_to?(:[]) begin - reactor.resolve_message_class(sym).type - rescue StandardError - nil - end - elsif reactor.respond_to?(:[]) - begin - reactor[sym].type + reactor[sym]&.type rescue StandardError nil end @@ -203,9 +189,8 @@ def self.resolve_symbol_ref(symbol_name, reactor, kind) end def self.resolve_const_ref(const_name, reactor) - # Search in reactor namespace, then enclosing modules, then top-level klass = resolve_constant_in_context(const_name, reactor) - klass&.type + klass&.respond_to?(:type) ? klass.type : nil end def self.resolve_const_path_ref(path) @@ -215,9 +200,6 @@ def self.resolve_const_path_ref(path) nil end - # Resolve Reactor[:symbol] bracket-accessor references. - # @param ref [Hash] with :receiver (const name string) and :index (symbol name string) - # @param reactor [Class] reactor class for namespace resolution def self.resolve_const_index_ref(ref, reactor) receiver_name = ref[:receiver] klass = if receiver_name.include?('::') @@ -232,14 +214,11 @@ def self.resolve_const_index_ref(ref, reactor) nil end - # Try to find a constant by unqualified name within the reactor's module hierarchy. def self.resolve_constant_in_context(const_name, reactor) - # 1. Check reactor's own constants if reactor.const_defined?(const_name, false) return reactor.const_get(const_name, false) end - # 2. Walk enclosing modules parts = reactor.name.split('::') while parts.length > 1 parts.pop @@ -253,19 +232,11 @@ def self.resolve_constant_in_context(const_name, reactor) end end - # 3. Top-level Object.const_get(const_name) rescue nil end - def self.find_event_class(type_string, reactor) - # Try reactor's event registry first - if reactor.respond_to?(:const_get) && reactor.const_defined?(:Event, false) - klass = reactor::Event.registry[type_string] - return klass if klass - end - - # Fall back to global Event registry - Sourced::Event.registry[type_string] + def self.find_event_class(type_string) + Sourced::Message.registry[type_string] end def self.extract_schema(msg_class) @@ -294,21 +265,13 @@ def initialize @prism_available = check_prism end - # Extract event references from a command handler block. - # @param reactor [Class] - # @param cmd_class [Class] - # @return [Array] e.g. [[:const, "ThingCreated"]] def events_produced_by(reactor, cmd_class) return [] unless @prism_available - method_name = Sourced.message_method_name(Actor::PREFIX, cmd_class.name) + method_name = Sourced.message_method_name('sourced_decide', cmd_class.name) extract_calls_from_handler(reactor, method_name, :event) end - # Extract dispatch references from a reaction handler block. - # @param reactor [Class] - # @param evt_class [Class] - # @return [Array] e.g. [[:const, "NotifyThing"]] def commands_dispatched_by(reactor, evt_class) return [] unless @prism_available @@ -339,14 +302,12 @@ def extract_calls_from_handler(reactor, method_name, call_name) [] end - # Find the block node at a specific line in the AST. def find_block_at_line(program, target_line) finder = BlockAtLineFinder.new(target_line) program.accept(finder) finder.found_block end - # Collect all calls to the target method within a block, traversing all branches. def collect_calls(block_node, target_name) collector = CallCollector.new(target_name) block_node.accept(collector) @@ -354,7 +315,6 @@ def collect_calls(block_node, target_name) end if defined?(::Prism) - # Prism visitor that finds the block at a specific source line. class BlockAtLineFinder < Prism::Visitor attr_reader :found_block @@ -371,8 +331,6 @@ def visit_call_node(node) end end - # Prism visitor that collects references from call nodes. - # Handles both direct calls (dispatch(Foo)) and chained calls (dispatch(Foo).at(...)). class CallCollector < Prism::Visitor attr_reader :refs @@ -391,9 +349,6 @@ def visit_call_node(node) private - # Walk receiver chain to find the target call. - # dispatch(Foo) => direct match - # dispatch(Foo).to(id).at(time) => receiver chain def find_target_call(node) return node if node.name == @target_name @@ -419,8 +374,6 @@ def extract_first_arg(call_node) @refs << ref if ref end - # Extract Reactor[:symbol] bracket-accessor pattern. - # Returns [:const_index, { receiver: "Reactor", index: "symbol" }] or nil. def extract_const_index(call_node) return unless call_node.name == :[] return unless call_node.arguments&.arguments&.size == 1 diff --git a/lib/sourced/types.rb b/lib/sourced/types.rb index 2d6a41cf..b3b681bd 100644 --- a/lib/sourced/types.rb +++ b/lib/sourced/types.rb @@ -29,17 +29,15 @@ module Types # AutoUUID.parse("test-uuid") # => "test-uuid" AutoUUID = UUID::V4.default { SecureRandom.uuid } - # Turn "Foo::Bar::FooBar" into "foo_bar" - TrailingModuleName = String.transform(::String) { |v| v.split('::').last } + # Turn "Foo::Bar::FooBar" into "foo.bar.foo_bar" ModulesToDots = String.transform(::String) { |v| v.gsub('::', '.') } Underscore = String.build(::String) { |v| v - .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') # Handle sequences like "HTTPResponse" -> "HTTP_Response" - .gsub(/([a-z\d])([A-Z])/, '\1_\2') # Handle transitions from lowercase to uppercase - .gsub(/-/, '_') # Replace hyphens with underscores - .downcase # Convert to lowercase + .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') + .gsub(/([a-z\d])([A-Z])/, '\1_\2') + .gsub(/-/, '_') + .downcase } - ModuleToMethodName = TrailingModuleName >> Underscore ModuleToMessageType = ModulesToDots >> Underscore end end diff --git a/lib/sourced/unit.rb b/lib/sourced/unit.rb deleted file mode 100644 index f5b41fb5..00000000 --- a/lib/sourced/unit.rb +++ /dev/null @@ -1,336 +0,0 @@ -# frozen_string_literal: true - -require 'sourced/injector' - -module Sourced - # Executes a group of reactors synchronously within a single transaction, - # using breadth-first traversal of the message graph. - # - # A Unit wires multiple reactors (actors, projectors) together so that - # the full command -> event -> reaction -> command chain runs in one - # backend transaction, giving all-or-nothing semantics. Messages - # produced by one reactor are immediately routed to any other reactor - # in the unit that handles them. - # - # @example Basic usage with two actors - # unit = Sourced::Unit.new(OrderActor, PaymentActor, backend: backend) - # results = unit.handle(PlaceOrder.new(stream_id: 'order-1', payload: { amount: 100 })) - # - # results.events_for(OrderActor) # => [OrderPlaced, ...] - # results.events_for(PaymentActor) # => [PaymentCharged, ...] - # - # @example Skip persisting commands (events-only storage) - # unit = Sourced::Unit.new(OrderActor, backend: backend, persist_commands: false) - # unit.handle(cmd) # only events are written to the store - # - # @example Limit BFS depth to detect runaway loops - # unit = Sourced::Unit.new(LoopyActor, backend: backend, max_iterations: 10) - # unit.handle(cmd) # raises Sourced::Unit::InfiniteLoopError if chain exceeds 10 steps - class Unit - # Default cap on BFS iterations before raising {InfiniteLoopError}. - DEFAULT_MAX_ITERATIONS = 100 - - # Raised when the BFS message loop exceeds {#max_iterations}. - InfiniteLoopError = Class.new(Sourced::Error) - - # @param reactors [Array] reactor classes (actors, projectors) to wire together - # @param backend [#transaction, #append_to_stream, #append_next_to_stream] storage backend - # @param logger [#info, #debug] logger instance - # @param max_iterations [Integer] safety cap on BFS iterations - # @param persist_commands [Boolean] when false, only events are written to the store - def initialize(*reactors, backend: Sourced.config.backend, logger: Sourced.config.logger, - max_iterations: DEFAULT_MAX_ITERATIONS, persist_commands: true) - @reactors = reactors - @backend = backend - @logger = logger - @max_iterations = max_iterations - @persist_commands = persist_commands - @routing_table = build_routing_table - @kargs_for_handle = resolve_kargs - ensure_consumer_groups! - freeze - end - - # Run the full message chain for +initial_message+ inside a single transaction. - # - # Persists the initial message (subject to +persist_commands+), then - # breadth-first routes every produced message to matching reactors until - # no more messages are enqueued. Finally, ACKs all handled messages so - # background workers won't re-process them. - # - # This method does not mutate instance state, so a single Unit can be - # reused across many calls (e.g. one Unit per route, built at boot time). - # - # @param initial_message [Sourced::Message] the command or event that kicks off the chain - # @return [Results] per-reactor instances and produced events - # @raise [InfiniteLoopError] if BFS iterations exceed +max_iterations+ - # - # @example Prepare once, call per request - # UsersUnit = Sourced::Unit.new(UsersActor, UsersProjection, backend: backend) - # - # post '/users' do - # results = UsersUnit.handle(CreateUser.new(stream_id: id, payload: params)) - # results.events_for(UsersActor) - # end - def handle(initial_message) - results = Results.new - queue = [initial_message] - iteration = 0 - - @backend.transaction do - if should_persist?(initial_message) - @backend.append_next_to_stream(initial_message.stream_id, [initial_message]) - end - - while (message = queue.shift) - iteration += 1 - raise InfiniteLoopError, "Exceeded #{@max_iterations} iterations" if iteration > @max_iterations - - handlers = @routing_table[message.class] || [] - handlers.each do |reactor_class| - new_messages = handle_for_reactor(reactor_class, message, results) - queue.concat(new_messages) - end - end - - ack_all(results) - end - - results - end - - private - - # Whether +message+ should be written to the event store. - # Commands (and subclasses) are only persisted when - # +@persist_commands+ is true. All other messages - # (events, raw messages) are always persisted. - # - # @param message [Sourced::Message] - # @return [Boolean] - def should_persist?(message) - return true if @persist_commands - - !message.is_a?(Sourced::Command) - end - - # Instantiate a reactor, replay its history, handle the message, - # and process resulting actions. Returns messages that should - # continue the BFS traversal. - # - # @param reactor_class [Class] the reactor class to handle the message - # @param message [Sourced::Message] the message to handle - # @param results [Results] accumulator for instances and events - # @return [Array] new messages to enqueue for BFS - def handle_for_reactor(reactor_class, message, results) - instance = reactor_class.new(id: reactor_class.identity_from(message)) - - kargs = build_handle_args(reactor_class, message.stream_id) - actions = instance.handle(message, **kargs) - - new_messages, produced_events = process_actions(message, actions) - - persisted = should_persist?(message) - results.record(reactor_class, instance, produced_events, message, persisted:) - - new_messages - end - - # Build keyword arguments for the reactor's #handle method - # based on which parameters it declares (replaying, history, logger). - # History is only loaded from the backend when the reactor declares it. - # - # @param reactor_class [Class] - # @param stream_id [String] stream to load history from, if needed - # @return [Hash] - def build_handle_args(reactor_class, stream_id) - @kargs_for_handle[reactor_class].each_with_object({}) do |name, hash| - case name - when :replaying - hash[name] = false - when :history - hash[name] = @backend.read_stream(stream_id) - when :logger - hash[name] = @logger - end - end - end - - # Execute a list of actions produced by a reactor. - # - # {Actions::AppendAfter} and {Actions::AppendNext} are handled explicitly - # because the Unit needs their correlated messages for BFS traversal and - # produced-events tracking. {Actions::AppendNext} also respects - # +persist_commands+. {Actions::Schedule} and {Actions::Sync} delegate - # to {Actions::Schedule#execute} / {Actions::Sync#execute}. - # - # @param source_message [Sourced::Message] message that triggered the actions - # @param actions [Array, Object] action(s) returned by the reactor - # @return [Array(Array, Array)] - # a two-element array: [new_messages_for_bfs, produced_events] - def process_actions(source_message, actions) - new_messages = [] - produced_events = [] - actions = [actions] unless actions.is_a?(Array) - actions = actions.compact - - actions.each do |action| - case action - when Actions::AppendAfter - messages = action.execute(@backend, source_message) - produced_events.concat(messages) - new_messages.concat(messages.select { |m| handled_by_unit?(m) }) - - when Actions::AppendNext - messages = action.messages.map { |m| source_message.correlate(m) } - messages.group_by(&:stream_id).each do |stream_id, stream_messages| - if should_persist?(stream_messages.first) - @backend.append_next_to_stream(stream_id, stream_messages) - end - end - new_messages.concat(messages.select { |m| handled_by_unit?(m) }) - - when Actions::Schedule, Actions::Sync - action.execute(@backend, source_message) - - when Actions::OK, :ok - # no-op - end - end - - [new_messages, produced_events] - end - - # Whether the message type is handled by any reactor in this unit. - # - # @param message [Sourced::Message] - # @return [Boolean] - def handled_by_unit?(message) - @routing_table.key?(message.class) - end - - # ACK every message that was handled during this unit of work - # so background workers won't re-process them. - # - # @param results [Results] - def ack_all(results) - results.each_ack do |group_id, message_id| - @backend.ack_on(group_id, message_id) - end - end - - # Register consumer groups for all reactors in this unit. - # Called once during initialization so that {#handle} has no setup overhead. - def ensure_consumer_groups! - @reactors.each do |reactor| - @backend.register_consumer_group(reactor.consumer_info.group_id) - end - end - - # Build a lookup table mapping message classes to the reactor classes - # that handle them. - # - # @return [Hash{Class => Array}] - def build_routing_table - table = {} - @reactors.each do |reactor| - reactor.handled_messages.each do |msg_class| - table[msg_class] ||= [] - table[msg_class] << reactor - end - end - table - end - - # Pre-resolve keyword argument names for each reactor's instance #handle method. - # Uses instance_method to analyze the signature since class-level handle - # may not exist for all reactor types (handle_batch is the class-level interface). - # - # @return [Hash{Class => Array}] - def resolve_kargs - @reactors.each_with_object({}) do |reactor, hash| - hash[reactor] = reactor.instance_method(:handle).parameters - .select { |type, _| Injector::KEYS.include?(type) } - .map { |_, name| name } - end - end - - # Collects reactor instances, produced events, and ACK pairs - # during a {Unit#handle} run. - # - # @example Inspecting results after handle - # results = unit.handle(cmd) - # - # # Get a hash of { instance => [events] } for a reactor class - # results[MyActor].each do |instance, events| - # puts "#{instance.id}: #{events.map(&:class)}" - # end - # - # # Get a flat list of events for a reactor class - # results.events_for(MyActor) # => [MyEvent, ...] - class Results - def initialize - @data = {} - @acks = [] - end - - # Record a reactor invocation and the events it produced. - # Only records an ACK when the handled message was persisted to the store. - # - # @param reactor_class [Class] - # @param instance [Object] the reactor instance - # @param produced_events [Array] - # @param handled_message [Sourced::Message] the message that was handled (for ACK tracking) - # @param persisted [Boolean] whether handled_message was written to the store - def record(reactor_class, instance, produced_events, handled_message, persisted: true) - key = [reactor_class, instance.id] - entry = (@data[key] ||= { instance: nil, events: [] }) - entry[:instance] = instance - entry[:events].concat(produced_events) - - @acks << [reactor_class.consumer_info.group_id, handled_message.id] if persisted - end - - # Get a hash of reactor instances and their produced events. - # - # @param reactor_class [Class] - # @return [Hash{Object => Array}] instance => events - # - # @example - # results[MyActor].each do |instance, events| - # puts instance.state - # end - def [](reactor_class) - result = {} - @data.each do |(klass, _stream_id), entry| - next unless klass == reactor_class - - result[entry[:instance]] = entry[:events] - end - result - end - - # Get a flat list of all events produced by a reactor class. - # - # @param reactor_class [Class] - # @return [Array] - # - # @example - # results.events_for(MyActor) # => [OrderPlaced, PaymentCharged] - def events_for(reactor_class) - self[reactor_class].values.flatten - end - - # Iterate over all ACK pairs (group_id, message_id). - # - # @yield [group_id, message_id] - # @yieldparam group_id [String] - # @yieldparam message_id [String] - def each_ack(&block) - @acks.each do |group_id, message_id| - block.call(group_id, message_id) - end - end - end - end -end diff --git a/lib/sourced/worker.rb b/lib/sourced/worker.rb index 1a09754a..4d0dbdda 100644 --- a/lib/sourced/worker.rb +++ b/lib/sourced/worker.rb @@ -1,69 +1,51 @@ # frozen_string_literal: true -require 'console' -require 'sourced/router' - module Sourced - # Processes messages from a {WorkQueue} in a signal-driven drain loop. - # - # Instead of polling all reactors on a timer, workers block on the queue - # waiting for signals (from {Notifier} or {CatchUpPoller}), then drain all - # available work for the signaled reactor before blocking again. - # - # The drain loop is bounded by +max_drain_rounds+ to prevent a single - # busy reactor from monopolizing a worker. When the cap is hit, the reactor - # is re-enqueued so another worker (or this one) can continue later. - # - # Multiple workers can drain the same reactor concurrently — each will - # claim a different stream via the backend's +SKIP LOCKED+ mechanism. + # Processes reactors from a {WorkQueue} in a signal-driven drain loop. + # Calls {Sourced::Router#handle_next_for} for each signaled reactor. # # @example Signal-driven mode (production) # queue = Sourced::WorkQueue.new(max_per_reactor: 4) - # worker = Sourced::Worker.new(work_queue: queue, name: 'worker-0') + # worker = Sourced::Worker.new(work_queue: queue, router: router, name: 'worker-0') # worker.run # blocks, processing reactors popped from queue # # @example Single-tick for testing - # worker = Sourced::Worker.new(work_queue: queue, name: 'test') - # worker.tick(MyReactor) # => true if a message was processed + # worker = Sourced::Worker.new(work_queue: queue, router: router, name: 'test') + # worker.tick(MyReactor) # => true if messages were processed class Worker - # @!attribute [r] name - # @return [String] Unique identifier for this worker instance + # @return [String] unique identifier for this worker instance attr_reader :name # @param work_queue [WorkQueue] queue to receive reactor signals from - # @param router [Router] router for dispatching events + # @param router [Sourced::Router] router for dispatching messages # @param name [String] unique name for this worker - # @param batch_size [Integer] messages per backend fetch + # @param batch_size [Integer] max messages per claim # @param max_drain_rounds [Integer] max consecutive drain iterations per reactor pickup # @param logger [Object] logger instance def initialize( work_queue:, - router: Sourced::Router, + router:, name: SecureRandom.hex(4), - batch_size: 1, + batch_size: 50, max_drain_rounds: 10, logger: Sourced.config.logger ) @work_queue = work_queue - @logger = logger - @running = false - @name = [Process.pid, name].join('-') @router = router + @name = [Process.pid, name].join('-') @batch_size = batch_size @max_drain_rounds = max_drain_rounds + @logger = logger + @running = false end # Signal the worker to stop after the current drain completes. - # # @return [void] def stop @running = false end # Main run loop. Blocks on the {WorkQueue} waiting for reactor signals. - # For each reactor popped, calls {#drain} to process available messages, - # then blocks again. Exits when a +nil+ shutdown sentinel is received. - # # @return [void] def run @running = true @@ -75,21 +57,20 @@ def run drain(reactor) end - @logger.info "Worker #{name}: stopped" + @logger.info "Sourced::Worker #{name}: stopped" end # Drain available messages for a reactor in a bounded loop. # - # Processes up to +max_drain_rounds+ batches. If all rounds are consumed - # (suggesting more work is available), re-enqueues the reactor into the - # {WorkQueue} for continued processing by this or another worker. + # Processes up to +max_drain_rounds+ batches. If all rounds are consumed, + # re-enqueues the reactor for fair scheduling across all reactors. # # @param reactor [Class] reactor class to drain messages for # @return [void] def drain(reactor) rounds = 0 while @running && rounds < @max_drain_rounds - found = @router.handle_next_event_for_reactor(reactor, name, batch_size: @batch_size) + found = @router.handle_next_for(reactor, worker_id: name, batch_size: @batch_size) break unless found rounds += 1 @@ -98,18 +79,12 @@ def drain(reactor) @work_queue.push(reactor) if @running && rounds >= @max_drain_rounds end - # Process one tick of work for a specific reactor. - # Convenience method for testing. + # Process one tick of work for a specific reactor. Convenience for testing. # - # @param reactor [Class] Specific reactor to process - # @return [Boolean] true if an event was processed, false otherwise + # @param reactor [Class] reactor class to process + # @return [Boolean] true if messages were processed, false otherwise def tick(reactor) - @router.handle_next_event_for_reactor(reactor, name, batch_size: @batch_size) + @router.handle_next_for(reactor, worker_id: name, batch_size: @batch_size) end - - private - - attr_reader :logger - end end diff --git a/spec/actions_spec.rb b/spec/actions_spec.rb deleted file mode 100644 index a850e7a1..00000000 --- a/spec/actions_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::Actions do - describe Sourced::Actions::AppendNext do - specify '#==' do - msg1 = Sourced::Message.new(stream_id: 'one') - msg2 = Sourced::Message.new(stream_id: 'two') - action1 = Sourced::Actions::AppendNext.new([msg1]) - action2 = Sourced::Actions::AppendNext.new([msg1]) - action3 = Sourced::Actions::AppendNext.new([msg2]) - expect(action1).to eq(action2) - expect(action1).not_to eq(action3) - end - - describe '#execute' do - it 'correlates messages and appends via backend.append_next_to_stream' do - backend = Sourced::Backends::TestBackend.new - source = Sourced::Message.new(stream_id: 'stream-1', seq: 1) - msg1 = Sourced::Message.new(stream_id: 'stream-1') - msg2 = Sourced::Message.new(stream_id: 'stream-2') - action = Sourced::Actions::AppendNext.new([msg1, msg2]) - - correlated = action.execute(backend, source) - - expect(correlated.size).to eq(2) - expect(correlated.map(&:correlation_id)).to all(eq(source.correlation_id)) - expect(correlated.map(&:causation_id)).to all(eq(source.id)) - expect(backend.read_stream('stream-1').size).to eq(1) - expect(backend.read_stream('stream-2').size).to eq(1) - end - end - end - - describe Sourced::Actions::AppendAfter do - describe '#execute' do - it 'correlates messages and appends via backend.append_to_stream' do - backend = Sourced::Backends::TestBackend.new - source = Sourced::Message.new(stream_id: 'stream-1', seq: 1) - msg1 = Sourced::Message.new(stream_id: 'stream-1', seq: 1) - msg2 = Sourced::Message.new(stream_id: 'stream-1', seq: 2) - action = Sourced::Actions::AppendAfter.new('stream-1', [msg1, msg2]) - - correlated = action.execute(backend, source) - - expect(correlated.size).to eq(2) - expect(correlated.map(&:correlation_id)).to all(eq(source.correlation_id)) - expect(correlated.map(&:causation_id)).to all(eq(source.id)) - expect(backend.read_stream('stream-1').size).to eq(2) - end - end - end - - describe Sourced::Actions::Schedule do - describe '#execute' do - it 'correlates messages and schedules via backend.schedule_messages' do - backend = Sourced::Backends::TestBackend.new - source = Sourced::Message.new(stream_id: 'stream-1', seq: 1) - future_time = Time.now + 3600 - msg = Sourced::Message.new(stream_id: 'stream-1') - action = Sourced::Actions::Schedule.new([msg], at: future_time) - - correlated = action.execute(backend, source) - - expect(correlated.size).to eq(1) - expect(correlated.first.correlation_id).to eq(source.correlation_id) - expect(correlated.first.causation_id).to eq(source.id) - end - end - end - - describe Sourced::Actions::Sync do - describe '#execute' do - it 'calls the work block and returns nil' do - called = false - action = Sourced::Actions::Sync.new(-> { called = true }) - source = Sourced::Message.new(stream_id: 'stream-1') - backend = Sourced::Backends::TestBackend.new - - result = action.execute(backend, source) - - expect(called).to be true - expect(result).to be_nil - end - end - end -end diff --git a/spec/actor_spec.rb b/spec/actor_spec.rb deleted file mode 100644 index 3a9f3ff5..00000000 --- a/spec/actor_spec.rb +++ /dev/null @@ -1,267 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -module TestDomain - TodoList = Struct.new(:archive_status, :seq, :id, :status, :items) - - AddItem = Sourced::Message.define('actor.todos.add') do - attribute :name, String - end - - Notify = Sourced::Message.define('actor.todos.notify') do - attribute :item_count, Integer - end - - ListStarted = Sourced::Message.define('actor.todos.started') - ArchiveRequested = Sourced::Message.define('actor.todos.archive_requested') - ConfirmArchive = Sourced::Message.define('actor.todos.archive_confirm') - ArchiveConfirmed = Sourced::Message.define('actor.todos.archive_confirmed') - - class Tracer - attr_reader :calls - - def initialize - @calls = [] - end - end - - SyncTracer = Tracer.new - - class TodoListActor < Sourced::Actor - state do |id| - TodoList.new(nil, 0, id, :new, []) - end - - command AddItem do |list, cmd| - event ListStarted if list.status == :new - event :item_added, name: cmd.payload.name - end - - command :add_one, name: String do |_list, cmd| - event :item_added, name: cmd.payload.name - end - - event ListStarted do |list, _event| - list.status = :active - end - - # Test that this block is returned after #decide - # wrapperd as a Sourced::Actions::Sync - # to be run within the same transaction as the append action - sync do |command:, events:, state:| - SyncTracer.calls << [command, events, state] - end - - event :item_added, name: String do |list, event| - list.items << event.payload - end - - reaction ListStarted do |list, event| - dispatch(Notify, item_count: list.items.size).to('different-stream') - dispatch(Notify, item_count: list.items.size).at(Time.now + 30) - end - - reaction :item_added do |list, event| - dispatch(Notify, item_count: list.items.size) - end - end -end - -RSpec.describe Sourced::Actor do - describe '.handled_messages' do - it 'returns commands and events to react to' do - expect(TestDomain::TodoListActor.handled_messages).to match_array([ - TestDomain::AddItem, - TestDomain::TodoListActor::AddOne, - TestDomain::ListStarted, - TestDomain::TodoListActor::ItemAdded, - ]) - end - end - - describe '.command' do - it 'raises if the message is also reacted to' do - klass = Class.new(described_class) - klass.reaction TestDomain::ListStarted do |_| - end - expect { - klass.command TestDomain::ListStarted do |_, _| - end - }.to raise_error(Sourced::Actor::DualMessageRegistrationError) - end - end - - describe '.reaction' do - it 'raises if the message is also registered as a command' do - klass = Class.new(described_class) - klass.command TestDomain::ListStarted do |_, _| - end - expect { - klass.reaction TestDomain::ListStarted do |_| - end - }.to raise_error(Sourced::Actor::DualMessageRegistrationError) - end - - it 'supports multiple events as symbols' do - klass = Class.new(described_class) do - event(:e1) - event(:e2) - reaction :e1, :e2 do |_, _| - end - end - - expect(klass.handled_messages).to match_array([ - klass::E1, - klass::E2, - ]) - end - end - - describe '#evolve' do - it 'evolves internal state' do - actor = TestDomain::TodoListActor.new - e1 = TestDomain::ListStarted.parse(stream_id: actor.id, seq: 1) - e2 = TestDomain::TodoListActor::ItemAdded.parse( - stream_id: actor.id, - seq: 2, - payload: { name: 'Shoes' } - ) - state = actor.evolve([e1, e2]) - expect(state).to eq(actor.state) - expect(actor.state.status).to eq(:active) - expect(actor.seq).to eq(2) - end - end - - describe '#decide' do - it 'returns events with the right sequence, updates state' do - actor = TestDomain::TodoListActor.new - cmd = TestDomain::AddItem.parse( - stream_id: actor.id, - payload: { name: 'Shoes' } - ) - events = actor.decide(cmd) - expect(events.map(&:class)).to eq([TestDomain::ListStarted, TestDomain::TodoListActor::ItemAdded]) - expect(events.map(&:seq)).to eq([1, 2]) - expect(actor.seq).to eq(2) - expect(actor.state.items.size).to eq(1) - cmd2 = cmd.with(seq: 1) - events = actor.decide(cmd2) - expect(actor.seq).to eq(3) - expect(events.map(&:class)).to eq([TestDomain::TodoListActor::ItemAdded]) - expect(events.map(&:seq)).to eq([3]) - expect(actor.state.items.size).to eq(2) - end - end - - describe '#react' do - it 'reacts to events and return commands' do - now = Time.now - Timecop.freeze(now) do - actor = TestDomain::TodoListActor.new - event = TestDomain::ListStarted.parse(stream_id: actor.id) - commands = actor.react(event) - expect(commands.map(&:class)).to eq([TestDomain::Notify, TestDomain::Notify]) - expect(commands.first.metadata[:producer]).to eq('TestDomain::TodoListActor') - expect(commands.map(&:created_at)).to eq([now, now + 30]) - expect(commands.map(&:stream_id)).to eq(['different-stream', actor.id]) - end - end - end - - describe '.handle' do - context 'with a command to decide on' do - let(:cmd) do - TestDomain::AddItem.parse( - stream_id: Sourced.new_stream_id, - seq: 1, - metadata: { foo: 'bar' }, - payload: { name: 'Shoes' } - ) - end - - it 'returns an array with Sourced::Actions::AppendAfter Sourced::Actions::Sync actions' do - result = TestDomain::TodoListActor.handle(cmd, history: [cmd]) - expect(result.map(&:class)).to eq([Sourced::Actions::AppendAfter, Sourced::Actions::Sync]) - end - - specify 'the AppendAfter action contains messages to append' do - result = TestDomain::TodoListActor.handle(cmd, history: [cmd]) - append_action = result[0] - expect(append_action.stream_id).to eq(cmd.stream_id) - # two new events, seq 2, and 3 - expect(append_action.messages.map(&:seq)).to eq([2, 3]) - append_action.messages[0].tap do |msg| - expect(msg).to be_a(TestDomain::ListStarted) - expect(msg.stream_id).to eq(cmd.stream_id) - expect(msg.metadata[:foo]).to eq('bar') - end - append_action.messages[1].tap do |msg| - expect(msg).to be_a(TestDomain::TodoListActor::ItemAdded) - expect(msg.stream_id).to eq(cmd.stream_id) - expect(msg.metadata[:foo]).to eq('bar') - end - end - - specify 'the Sync action contains a side-effect to run' do - result = TestDomain::TodoListActor.handle(cmd, history: [cmd]) - append_action = result[0] - sync_action = result[1] - - expect(TestDomain::SyncTracer.calls).to eq([]) - sync_action.call - expect(TestDomain::SyncTracer.calls.size).to eq(1) - TestDomain::SyncTracer.calls.first.tap do |(command, events, state)| - expect(command).to eq(cmd) - expect(events).to eq(append_action.messages) - expect(state).to be_a(TestDomain::TodoList) - expect(state.items.size).to eq(1) - end - end - end - - context 'with an event to react to' do - let(:stream_id) { Sourced.new_stream_id } - - let(:history) do - [ - TestDomain::AddItem.parse(stream_id:, seq: 1, payload: { name: 'test' }), - TestDomain::ListStarted.parse(stream_id:, seq: 2), - TestDomain::TodoListActor::ItemAdded.parse(stream_id:, seq: 3, payload: { name: 'test' }) - ] - end - - it 'returns new commands to append' do - result = TestDomain::TodoListActor.handle(history.last, history:) - expect(result).to be_a(Array) - expect(result.first).to be_a(Sourced::Actions::AppendNext) - expect(result.first.messages.map(&:stream_id)).to eq([stream_id]) - expect(result.first.messages.map(&:class)).to eq([TestDomain::Notify]) - expect(result.first.messages.first.payload.item_count).to eq(1) - end - - it 'returns multiple commands to append or schedule' do - now = Time.now - Timecop.freeze(now) do - result = TestDomain::TodoListActor.handle(history[1], history:) - expect(result).to be_a(Array) - expect(result.map(&:class)).to eq [Sourced::Actions::AppendNext, Sourced::Actions::Schedule] - expect(result[0].messages.size).to eq(1) - result[0].messages[0].tap do |msg| - expect(msg).to be_a TestDomain::Notify - expect(msg.stream_id).to eq('different-stream') - expect(msg.payload.item_count).to eq(1) - end - expect(result[1].messages.size).to eq(1) - expect(result[1].at).to eq(now + 30) - result[1].messages[0].tap do |msg| - expect(msg).to be_a TestDomain::Notify - expect(msg.stream_id).to eq(stream_id) - expect(msg.payload.item_count).to eq(1) - end - end - end - end - end -end diff --git a/spec/async_executor_spec.rb b/spec/async_executor_spec.rb deleted file mode 100644 index d1a9efa1..00000000 --- a/spec/async_executor_spec.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/async_executor' - -RSpec.describe Sourced::AsyncExecutor, type: :executor do - subject(:executor) { described_class.new } - - it_behaves_like 'an executor' -end diff --git a/spec/backends/concurrent_projectors_spec.rb b/spec/backends/concurrent_projectors_spec.rb deleted file mode 100644 index a042d593..00000000 --- a/spec/backends/concurrent_projectors_spec.rb +++ /dev/null @@ -1,115 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/backends/pg_backend' - -module ConcurrencyExamples - SomethingHappened = Sourced::Message.define('concurrent.something_happened') do - attribute :number, Integer - end - - class Store - attr_reader :data, :queue, :trace - - def initialize - @mutex = Mutex.new - @queue = Queue.new - @data = {} - end - - def get(stream_id) - @data[stream_id] - end - - def set(stream_id, state) - @mutex.synchronize do - @data[stream_id] = state - @queue << 1 - end - end - end - - STORE = Store.new - - class Projector < Sourced::Projector::StateStored - state do |id| - STORE.get(id) || {id:, seq: 0, seqs: []} - end - - sync do |state:, events:, replaying:| - STORE.set(state[:id], state) - end - - event ConcurrencyExamples::SomethingHappened do |state, event| - state[:seq] = event.seq - state[:seqs] << event.seq - end - end -end - -RSpec.describe 'Processing events concurrently', type: :backend do - subject(:backend) { Sourced::Backends::PGBackend.new(db) } - - let(:db) do - # Sequel.sqlite - Sequel.postgres('sourced_test') - end - - let(:router) { Sourced::Router.new(backend:) } - - before do - backend.clear! - backend.uninstall - backend.install - - router.register ConcurrencyExamples::Projector - - stream1_events = 100.times.map do |i| - seq = i + 1 - ConcurrencyExamples::SomethingHappened.parse(stream_id: 'stream1', seq:, payload: { number: seq }) - end - - stream2_events = 120.times.map do |i| - seq = i + 1 - ConcurrencyExamples::SomethingHappened.parse(stream_id: 'stream2', seq:, payload: { number: seq }) - end - - all_events = (stream2_events + stream1_events).flatten.compact - all_events.each do |event| - backend.append_to_stream(event.stream_id, [event]) - end - end - - specify 'consumes streams concurrently, maintaining per-stream event ordering, consuming all available events for each stream' do - work_queue = Sourced::WorkQueue.new(max_per_reactor: 3, queue: Queue.new) - workers = 3.times.map do |i| - Sourced::Worker.new(work_queue:, name: "worker-#{i}", router:) - end - - threads = workers.map do |worker| - Thread.new do - worker.run - end - end - - # Push the reactor to wake up workers - 3.times { work_queue.push(ConcurrencyExamples::Projector) } - - count = 0 - while count < 220 - ConcurrencyExamples::STORE.queue.pop - count += 1 - end - - workers.each(&:stop) - work_queue.close(workers.size) - threads.each(&:join) - - duplicates = ConcurrencyExamples::STORE.data['stream1'][:seqs].group_by(&:itself).select {|k,v| v.size > 1 } - expect(ConcurrencyExamples::STORE.data['stream1'][:seq]).to eq(100) - expect(duplicates).to be_empty - expect(ConcurrencyExamples::STORE.data['stream1'][:seqs]).to eq((1..100).to_a) - expect(ConcurrencyExamples::STORE.data['stream2'][:seq]).to eq(120) - expect(ConcurrencyExamples::STORE.data['stream2'][:seqs]).to eq((1..120).to_a) - end -end diff --git a/spec/backends/pg_backend_spec.rb b/spec/backends/pg_backend_spec.rb deleted file mode 100644 index 92aa7b8c..00000000 --- a/spec/backends/pg_backend_spec.rb +++ /dev/null @@ -1,130 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/backends/pg_backend' - -RSpec.describe 'Sourced::Backends::PGBackend', type: :backend do - before(:all) do - @db = Sequel.postgres('sourced_test') - @backend = Sourced::Backends::PGBackend.new(@db) - @backend.setup!(Sourced.config) - @backend.uninstall if @backend.installed? - @backend.install - end - - after(:all) do - @db&.disconnect - end - - subject(:backend) { @backend } - let(:db) { @db } - - before do - backend.clear! - end - - it_behaves_like 'a backend' - - describe '#update_schedule!' do - it 'blocks concurrent workers from selecting the same messages' do - now = Time.now - 10 - cmd1 = BackendExamples::Tests::DoSomething.parse(stream_id: 'as1', payload: { account_id: 1 }) - cmd2 = BackendExamples::Tests::DoSomething.parse(stream_id: 'as1', payload: { account_id: 1 }) - backend.schedule_messages([cmd1, cmd2], at: now) - Sourced.config.executor.start do |t| - 2.times.each do - t.spawn do - backend.update_schedule! - end - end - end - - expect(backend.read_stream('as1').map(&:id)).to eq([cmd1, cmd2].map(&:id)) - end - end - - describe '#ack_on' do - it 'raises exception if concurrently processed by the same group' do - evt1 = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 1 }) - backend.append_to_stream('s1', [evt1, evt2]) - - expect do - Sourced.config.executor.start do |t| - t.spawn do - backend.ack_on('group1', evt1.id) { sleep 0.01 } - end - t.spawn do - backend.ack_on('group1', evt2.id) { true } - end - end - end.to raise_error(Sourced::ConcurrentAckError) - end - end - - describe 'worker heartbeats and stale claim reaping' do - it 'records heartbeats and releases stale claims' do - # Prepare a consumer group and a stream with one event - backend.register_consumer_group('group_hb') - - evt = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's-hb', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s-hb', [evt]) - - # First record a heartbeat for a worker that will become stale - stale_time = Time.now - 3600 - backend.worker_heartbeat(['dead-worker-1'], at: stale_time) - - # Insert a stale claimed offset for the dead worker - group_fk = db[:sourced_consumer_groups].where(group_id: 'group_hb').get(:id) - stream_fk = db[:sourced_streams].where(stream_id: 's-hb').get(:id) - - off_id = db[:sourced_offsets].insert( - group_id: group_fk, - stream_id: stream_fk, - global_seq: 0, - created_at: stale_time, - claimed: true, - claimed_at: stale_time, - claimed_by: 'dead-worker-1' - ) - - # Heartbeat two workers - backend.worker_heartbeat(['live-worker-1', 'live-worker-2']) - expect(db[:sourced_workers].where(id: 'live-worker-1').count).to eq(1) - - # Reap stale claims - released = backend.release_stale_claims(ttl_seconds: 60) - expect(released).to be >= 1 - - row = db[:sourced_offsets].where(id: off_id).first - expect(row[:claimed]).to eq(false) - expect(row[:claimed_by]).to be_nil - expect(row[:claimed_at]).to be_nil - - # Ensure live worker claims are not reaped - # Create a fresh claimed offset for a live worker on a different stream - evt2 = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's-hb-2', seq: 1, payload: { account_id: 2 }) - backend.append_to_stream('s-hb-2', [evt2]) - stream2_fk = db[:sourced_streams].where(stream_id: 's-hb-2').get(:id) - - fresh_time = Time.now - off2_id = db[:sourced_offsets].insert( - group_id: group_fk, - stream_id: stream2_fk, - global_seq: 0, - created_at: fresh_time, - claimed: true, - claimed_at: fresh_time, - claimed_by: 'live-worker-1' - ) - - backend.worker_heartbeat(['live-worker-1']) - not_released = backend.release_stale_claims(ttl_seconds: 60) - expect(not_released).to be_a(Integer) - - row2 = db[:sourced_offsets].where(id: off2_id).first - expect(row2[:claimed]).to eq(true) - expect(row2[:claimed_by]).to eq('live-worker-1') - end - end -end diff --git a/spec/backends/pg_notifier_spec.rb b/spec/backends/pg_notifier_spec.rb deleted file mode 100644 index 6fba55dd..00000000 --- a/spec/backends/pg_notifier_spec.rb +++ /dev/null @@ -1,134 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/backends/sequel_backend' - -RSpec.describe Sourced::Backends::SequelBackend::PGNotifier do - # LISTEN holds a connection, NOTIFY must go through a different one. - before(:all) do - @listen_db = Sequel.postgres('sourced_test') - @notify_db = Sequel.postgres('sourced_test') - end - - after(:all) do - @listen_db&.disconnect - @notify_db&.disconnect - end - - let(:listen_db) { @listen_db } - let(:notify_db) { @notify_db } - - describe '#notify_new_messages' do - it 'delivers type notifications to subscribers' do - listener = described_class.new(db: listen_db) - sender = described_class.new(db: notify_db) - - received = [] - listener.subscribe(->(event, value) { received << [event, value] }) - - Sourced.config.executor.start do |t| - t.spawn { listener.start } - - t.spawn do - sleep 0.01 - sender.notify_new_messages(['orders.created', 'orders.shipped']) - sleep 0.01 - listener.stop - end - end - - expect(received.size).to eq(1) - expect(received.first).to eq(['messages_appended', 'orders.created,orders.shipped']) - end - - it 'deduplicates types' do - listener = described_class.new(db: listen_db) - sender = described_class.new(db: notify_db) - - received = [] - listener.subscribe(->(event, value) { received << [event, value] }) - - Sourced.config.executor.start do |t| - t.spawn { listener.start } - - t.spawn do - sleep 0.01 - sender.notify_new_messages(['orders.created', 'orders.created', 'orders.shipped']) - sleep 0.01 - listener.stop - end - end - - expect(received.size).to eq(1) - expect(received.first).to eq(['messages_appended', 'orders.created,orders.shipped']) - end - end - - describe '#notify_reactor_resumed' do - it 'delivers reactor notifications to subscribers' do - listener = described_class.new(db: listen_db) - sender = described_class.new(db: notify_db) - - received = [] - listener.subscribe(->(event, value) { received << [event, value] }) - - Sourced.config.executor.start do |t| - t.spawn { listener.start } - - t.spawn do - sleep 0.01 - sender.notify_reactor_resumed('OrderReactor') - sleep 0.01 - listener.stop - end - end - - expect(received).to eq([['reactor_resumed', 'OrderReactor']]) - end - end - - describe 'multiplexing' do - it 'routes type and reactor notifications correctly over the same channel' do - listener = described_class.new(db: listen_db) - sender = described_class.new(db: notify_db) - - received = [] - listener.subscribe(->(event, value) { received << [event, value] }) - - Sourced.config.executor.start do |t| - t.spawn { listener.start } - - t.spawn do - sleep 0.01 - sender.notify_new_messages(['orders.created']) - sleep 0.01 - sender.notify_reactor_resumed('ShipReactor') - sleep 0.01 - listener.stop - end - end - - expect(received).to eq([ - ['messages_appended', 'orders.created'], - ['reactor_resumed', 'ShipReactor'] - ]) - end - end - - describe '#stop' do - it 'breaks out of the listen loop' do - notifier = described_class.new(db: listen_db) - - Sourced.config.executor.start do |t| - t.spawn { notifier.start } - - t.spawn do - sleep 0.01 - notifier.stop - end - end - - # If we get here, start returned — stop worked - end - end -end diff --git a/spec/backends/sqlite_backend_spec.rb b/spec/backends/sqlite_backend_spec.rb deleted file mode 100644 index 57ee9a66..00000000 --- a/spec/backends/sqlite_backend_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/backends/sqlite_backend' - -RSpec.describe 'Sourced::Backends::SQLiteBackend', type: :backend do - before(:all) do - @db = Sequel.sqlite - @backend = Sourced::Backends::SQLiteBackend.new(@db) - @backend.install - end - - subject(:backend) { @backend } - let(:db) { @db } - - before do - backend.clear! - end - - it_behaves_like 'a backend' - - describe '#update_schedule!' do - it 'blocks concurrent workers from selecting the same messages' do - now = Time.now - 10 - cmd1 = BackendExamples::Tests::DoSomething.parse(stream_id: 'as1', payload: { account_id: 1 }) - cmd2 = BackendExamples::Tests::DoSomething.parse(stream_id: 'as1', payload: { account_id: 1 }) - backend.schedule_messages([cmd1, cmd2], at: now) - Sourced.config.executor.start do |t| - 2.times.each do - t.spawn do - backend.update_schedule! - end - end - end - - expect(backend.read_stream('as1').map(&:id)).to eq([cmd1, cmd2].map(&:id)) - end - end -end diff --git a/spec/backends/test_backend_spec.rb b/spec/backends/test_backend_spec.rb deleted file mode 100644 index 7fc401bb..00000000 --- a/spec/backends/test_backend_spec.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/backends/test_backend' - -RSpec.describe Sourced::Backends::TestBackend, type: :backend do - subject(:backend) { described_class.new } - - it_behaves_like 'a backend' - - describe 'housekeeping interfaces' do - it 'exposes worker_heartbeat and release_stale_claims' do - expect(backend.worker_heartbeat([])).to eq(0) - expect(backend.worker_heartbeat(['w1', 'w2'])).to eq(2) - expect(backend.release_stale_claims(ttl_seconds: 10)).to eq(0) - end - end -end diff --git a/spec/catchup_poller_spec.rb b/spec/catchup_poller_spec.rb deleted file mode 100644 index 82d10532..00000000 --- a/spec/catchup_poller_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::CatchUpPoller do - let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } - let(:logger) { instance_double('Logger', info: nil, error: nil) } - let(:reactor1) { double('Reactor1') } - let(:reactors) { [reactor1] } - - describe '#run' do - it 'pushes all reactors on startup and periodically' do - poller = described_class.new( - work_queue: work_queue, - reactors: reactors, - interval: 0.01, - logger: logger - ) - - t = Thread.new { poller.run } - sleep 0.05 - poller.stop - t.join(1) - - # Should have pushed reactor1 at least once (immediate catch-up) - popped = work_queue.pop - expect(popped).to eq(reactor1) - end - end - - describe '#stop' do - it 'stops the poller loop' do - poller = described_class.new( - work_queue: work_queue, - reactors: reactors, - interval: 0.01, - logger: logger - ) - - t = Thread.new { poller.run } - sleep 0.05 - poller.stop - t.join(1) - - expect(logger).to have_received(:info).with('CatchUpPoller: stopped') - end - end -end diff --git a/spec/command_context_spec.rb b/spec/command_context_spec.rb index a012c2d3..9fbae83d 100644 --- a/spec/command_context_spec.rb +++ b/spec/command_context_spec.rb @@ -1,63 +1,242 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' -module ContextTest - Add = Sourced::Command.define('ctest.add') do +module CccContextTest + Add = Sourced::Command.define('ccc_ctest.add') do attribute :value, Integer end - Added = Sourced::Event.define('ctest.added') + Remove = Sourced::Command.define('ccc_ctest.remove') do + attribute :value, Integer + end + + Added = Sourced::Event.define('ccc_ctest.added') end RSpec.describe Sourced::CommandContext do describe '#build' do - it 'builds command with stream_id and metadata' do - ctx = described_class.new(stream_id: '123', metadata: { user_id: 10 }) - cmd = ctx.build(type: 'ctest.add', payload: { value: 1 }) - expect(cmd).to be_a(ContextTest::Add) - expect(cmd.stream_id).to eq('123') + it 'builds command from type string with metadata' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) + expect(cmd).to be_a(CccContextTest::Add) expect(cmd.payload.value).to eq(1) expect(cmd.metadata[:user_id]).to eq(10) end - it 'allows overriding stream_id' do - ctx = described_class.new(stream_id: '123', metadata: { user_id: 10 }) - cmd = ctx.build(stream_id: 'aaa', type: 'ctest.add', payload: { value: 1 }) - expect(cmd.stream_id).to eq('aaa') - end - it 'can take a command class' do ctx = described_class.new(metadata: { user_id: 10 }) - cmd = ctx.build(ContextTest::Add, stream_id: '123', payload: { value: 1 }) - expect(cmd).to be_a(ContextTest::Add) - expect(cmd.stream_id).to eq('123') + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd).to be_a(CccContextTest::Add) expect(cmd.payload.value).to eq(1) expect(cmd.metadata[:user_id]).to eq(10) - expect(cmd.valid?).to eq(true) end - it 'symbolizes attributes' do - ctx = described_class.new(stream_id: '123', metadata: { user_id: 10 }) - cmd = ctx.build('type' => 'ctest.add', 'payload' => { 'value' => 1 }) - expect(cmd).to be_a(ContextTest::Add) - expect(cmd.stream_id).to eq('123') + it 'symbolizes string keys' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build('type' => 'ccc_ctest.add', 'payload' => { 'value' => 1 }) + expect(cmd).to be_a(CccContextTest::Add) expect(cmd.payload.value).to eq(1) expect(cmd.metadata[:user_id]).to eq(10) end - it 'raises an exception if command type does not exist' do - ctx = described_class.new(stream_id: '123', metadata: { user_id: 10 }) + it 'raises UnknownMessageError for unknown types' do + ctx = described_class.new(metadata: { user_id: 10 }) expect do ctx.build('type' => 'nope', 'payload' => { 'value' => 1 }) end.to raise_error(Sourced::UnknownMessageError) end - it 'raises an exception if command type does not exist in scope class' do - ctx = described_class.new(stream_id: '123', metadata: { user_id: 10 }) + it 'raises UnknownMessageError for event types when scoped to Command' do + ctx = described_class.new(metadata: { user_id: 10 }) expect do - ctx.build('type' => 'ctest.added', 'payload' => { 'value' => 1 }) + ctx.build('type' => 'ccc_ctest.added', 'payload' => {}) end.to raise_error(Sourced::UnknownMessageError) end + + it 'allows scoping to a custom command subclass' do + custom_scope = Class.new(Sourced::Command) + custom_cmd = custom_scope.define('ccc_ctest.custom') do + attribute :name, String + end + + ctx = described_class.new(metadata: { user_id: 10 }, scope: custom_scope) + cmd = ctx.build(type: 'ccc_ctest.custom', payload: { name: 'hello' }) + expect(cmd).to be_a(custom_cmd) + expect(cmd.payload.name).to eq('hello') + expect(cmd.metadata[:user_id]).to eq(10) + end + end + + describe 'callback hooks' do + it 'runs on block for matching command type' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.payload.value).to eq(11) + end + + it 'registers on block for multiple command types' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add, CccContextTest::Remove) { |_app, cmd| cmd.with_metadata(tagged: true) } + + ctx = klass.new + add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) + expect(add_cmd.metadata[:tagged]).to eq(true) + expect(remove_cmd.metadata[:tagged]).to eq(true) + end + + it 'accumulates multiple on blocks for the same command type' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add, CccContextTest::Remove) { |_app, cmd| cmd.with_metadata(first: true) } + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_metadata(second: true) } + + ctx = klass.new + # Add gets both blocks + add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(add_cmd.metadata[:first]).to eq(true) + expect(add_cmd.metadata[:second]).to eq(true) + + # Remove only gets the first block + remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) + expect(remove_cmd.metadata[:first]).to eq(true) + expect(remove_cmd.metadata).not_to have_key(:second) + end + + it 'does not run on block for non-matching command type' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Remove, payload: { value: 1 }) + expect(cmd.payload.value).to eq(1) + end + + it 'passes app scope to on block' do + app = double('app', session_id: 'abc') + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |a, cmd| cmd.with_metadata(session_id: a.session_id) } + + ctx = klass.new(app: app) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:session_id]).to eq('abc') + end + + it 'runs any block for all commands' do + klass = Class.new(described_class) + klass.any { |_app, cmd| cmd.with_metadata(source: 'web') } + + ctx = klass.new + add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) + expect(add_cmd.metadata[:source]).to eq('web') + expect(remove_cmd.metadata[:source]).to eq('web') + end + + it 'runs on before any (pipeline order)' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_metadata(step: 'on') } + klass.any { |_app, cmd| cmd.with_metadata(step: "#{cmd.metadata[:step]}_any") } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:step]).to eq('on_any') + end + + it 'runs multiple any blocks in order' do + klass = Class.new(described_class) + klass.any { |_app, cmd| cmd.with_metadata(steps: ['first']) } + klass.any { |_app, cmd| cmd.with_metadata(steps: cmd.metadata[:steps] + ['second']) } + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:steps]).to eq(%w[first second]) + end + + it 'passes through unchanged when no hooks registered' do + ctx = described_class.new(metadata: { user_id: 10 }) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.payload.value).to eq(1) + expect(cmd.metadata[:user_id]).to eq(10) + end + + it 'app defaults to nil' do + klass = Class.new(described_class) + received_app = :not_set + klass.any { |a, cmd| received_app = a; cmd } + + ctx = klass.new + ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(received_app).to be_nil + end + + it 'subclass inherits parent blocks' do + parent = Class.new(described_class) + parent.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 100) } + parent.any { |_app, cmd| cmd.with_metadata(inherited: true) } + + child = Class.new(parent) + + ctx = child.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.payload.value).to eq(101) + expect(cmd.metadata[:inherited]).to eq(true) + end + + it 'subclass blocks do not affect parent' do + parent = Class.new(described_class) + child = Class.new(parent) + child.any { |_app, cmd| cmd.with_metadata(child_only: true) } + + ctx = parent.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata).not_to have_key(:child_only) + end + + it 'works with build from type string + on block' do + klass = Class.new(described_class) + klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 5) } + + ctx = klass.new + cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) + expect(cmd.payload.value).to eq(6) + end + + it 'runs on blocks in the context of the instance' do + klass = Class.new(described_class) do + on(CccContextTest::Add) { |app, cmd| cmd.with_metadata(user_id: build_user_id(app)) } + + private + + def build_user_id(app) + "user-#{app.session_id}" + end + end + + app = double('app', session_id: '42') + ctx = klass.new(app: app) + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:user_id]).to eq('user-42') + end + + it 'runs any blocks in the context of the instance' do + klass = Class.new(described_class) do + any { |app, cmd| cmd.with_metadata(source: request_source) } + + private + + def request_source + 'web' + end + end + + ctx = klass.new + cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) + expect(cmd.metadata[:source]).to eq('web') + end end end diff --git a/spec/command_methods_spec.rb b/spec/command_methods_spec.rb deleted file mode 100644 index 0b0ce3b1..00000000 --- a/spec/command_methods_spec.rb +++ /dev/null @@ -1,103 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/command_methods' - -module CMDMethodsTest - class Actor < Sourced::Actor - include Sourced::CommandMethods - - UpdateAge = Sourced::Message.define('cmdtest.update_age') do - attribute :age, Integer - end - - command :start, name: String do |_, cmd| - event :started, cmd.payload - end - - event :started, name: String - - command UpdateAge do |_, cmd| - event :age_updated, cmd.payload - end - - event :age_updated, age: Integer - end -end - -RSpec.describe Sourced::CommandMethods do - describe 'in-memory version (#start, #update_age)' do - it 'creates method for symbolised commands' do - actor = CMDMethodsTest::Actor.new(id: 'aa') - cmd, new_events = actor.start(name: 'Joe') - expect(cmd.valid?).to be(true) - expect(actor.seq).to eq(1) - expect(new_events).to match_sourced_messages([ - CMDMethodsTest::Actor::Started.build('aa', name: 'Joe') - ]) - end - - it 'creates method for command class' do - actor = CMDMethodsTest::Actor.new(id: 'aa') - cmd, new_events = actor.update_age(age: 10) - expect(cmd.valid?).to be(true) - expect(new_events).to match_sourced_messages([ - CMDMethodsTest::Actor::AgeUpdated.build('aa', age: 10) - ]) - end - - it 'returns invalid command on invalid arguments' do - actor = CMDMethodsTest::Actor.new(id: 'aa') - cmd, new_events = actor.start(name: 20) - expect(cmd.valid?).to be(false) - expect(actor.seq).to eq(0) - expect(new_events).to eq([]) - end - end - - describe 'durable version that saves messages to backend (#start!, #update_age!)' do - before do - Sourced.config.backend.clear! - end - - it 'appends messages to backend' do - actor = CMDMethodsTest::Actor.new(id: 'aa') - cmd, new_events = actor.start!(name: 'Joe') - expect(cmd.valid?).to be(true) - expect(actor.seq).to eq(1) - events = Sourced.config.backend.read_stream(actor.id) - expect(events).to eq(new_events) - expect(new_events).to match_sourced_messages([ - CMDMethodsTest::Actor::Started.build('aa', name: 'Joe') - ]) - end - - it 'works when command is a class' do - actor = CMDMethodsTest::Actor.new(id: 'aa') - cmd, new_events = actor.update_age!(age: 20) - expect(cmd.valid?).to be(true) - expect(actor.seq).to eq(1) - events = Sourced.config.backend.read_stream(actor.id) - expect(events).to eq(new_events) - expect(new_events).to match_sourced_messages([ - CMDMethodsTest::Actor::AgeUpdated.build('aa', age: 20) - ]) - end - - it 'validates command' do - actor = CMDMethodsTest::Actor.new(id: 'aa') - cmd, new_events = actor.update_age!(age: 'nope') - expect(cmd.valid?).to be(false) - expect(actor.seq).to eq(0) - expect(new_events).to eq([]) - end - - it 'raises an exception if backend fails to append' do - allow(Sourced.config.backend).to receive(:append_to_stream).and_return(false) - actor = CMDMethodsTest::Actor.new(id: 'aa') - expect { - actor.update_age!(age: 20) - }.to raise_error(Sourced::CommandMethods::FailedToAppendMessagesError) - end - end -end diff --git a/spec/configuration_spec.rb b/spec/configuration_spec.rb index 78e7e8eb..4812230d 100644 --- a/spec/configuration_spec.rb +++ b/spec/configuration_spec.rb @@ -1,139 +1,276 @@ # frozen_string_literal: true +require 'spec_helper' +require 'sourced' require 'sequel' RSpec.describe Sourced::Configuration do - subject(:config) { described_class.new } + after { Sourced.reset! } - it 'has a test backend by default' do - expect(config.backend).to be_a(Sourced::Backends::TestBackend) - end + describe 'Sourced.config' do + it 'returns a Configuration with sensible defaults' do + config = Sourced.config + expect(config).to be_a(described_class) + expect(config.worker_count).to eq(2) + expect(config.batch_size).to eq(50) + expect(config.catchup_interval).to eq(5) + expect(config.max_drain_rounds).to eq(10) + expect(config.claim_ttl_seconds).to eq(120) + expect(config.housekeeping_interval).to eq(30) + expect(config.logger).to eq(Sourced.config.logger) + end - it 'has a default #error_strategy' do - expect(config.error_strategy).to be_a(Sourced::ErrorStrategy) + it 'returns the same instance on repeated calls' do + expect(Sourced.config).to be(Sourced.config) + end end - specify '#error_strategy=' do - st = Sourced::ErrorStrategy.new - config.error_strategy = st - expect(config.error_strategy).to eq(st) + describe 'Sourced.configure' do + it 'yields the config and freezes it after setup' do + Sourced.configure do |c| + c.worker_count = 4 + c.batch_size = 100 + end + + expect(Sourced.config.worker_count).to eq(4) + expect(Sourced.config.batch_size).to eq(100) + expect(Sourced.config).to be_frozen + end + + it 'calls setup! which creates store and router' do + Sourced.configure {} + + expect(Sourced.config.store).to be_a(Sourced::Store) + expect(Sourced.config.router).to be_a(Sourced::Router) + end end - describe '#error_strategy(&block)' do - it 'configures the error strategy with a block' do - config.error_strategy do |s| - s.retry(times: 30, after: 50) + describe 'Sourced.register' do + let(:reactor_class) do + Class.new(Sourced::Projector::StateStored) do + def self.name = 'TestConfigReactor' + + consumer_group 'test-config-reactor' + partition_by :thing_id + + state { |_| {} } end + end - expect(config.error_strategy).to be_a(Sourced::ErrorStrategy) - expect(config.error_strategy.max_retries).to eq(30) - expect(config.error_strategy.retry_after).to eq(50) + it 'triggers setup and delegates to router.register' do + Sourced.register(reactor_class) + + expect(Sourced.router.reactors).to include(reactor_class) end end - describe '#backend=' do - it 'can configure backend with a Sequel SQLite database' do - config.backend = Sequel.sqlite - expect(config.backend).to be_a(Sourced::Backends::SQLiteBackend) - end - - it 'uses PGBackend for Postgres databases' do - config.backend = Sequel.postgres('sourced_test') - expect(config.backend).to be_a(Sourced::Backends::PGBackend) - end - - it 'accepts anything with the Backend interface' do - backend = Struct.new( - :installed?, - :reserve_next_for_reactor, - :append_to_stream, - :read_correlation_batch, - :read_stream, - :updating_consumer_group, - :register_consumer_group, - :start_consumer_group, - :stop_consumer_group, - :reset_consumer_group, - :stats, - :transaction - ) + describe 'Sourced.store' do + it 'triggers setup and returns the store' do + store = Sourced.store + expect(store).to be_a(Sourced::Store) + expect(store.installed?).to be true + end + end + + describe 'Sourced.router' do + it 'triggers setup and returns the router' do + router = Sourced.router + expect(router).to be_a(Sourced::Router) + expect(router.store).to be(Sourced.store) + end + end + + describe 'Sourced.setup!' do + let(:reactor_class) do + Class.new(Sourced::Projector::StateStored) do + def self.name = 'SetupTestReactor' + + consumer_group 'setup-test-reactor' + partition_by :thing_id + + state { |_| {} } + end + end + + it 'replays the configure block on a fresh Configuration' do + call_count = 0 + Sourced.configure do |c| + call_count += 1 + c.worker_count = 8 + end + + expect(call_count).to eq(1) + original_config = Sourced.config + + Sourced.setup! + + expect(call_count).to eq(2) + expect(Sourced.config).not_to be(original_config) + expect(Sourced.config.worker_count).to eq(8) + expect(Sourced.config).to be_frozen + end + + it 'creates a new store connection on each call' do + Sourced.configure {} + store1 = Sourced.config.store - config.backend = backend.new(nil, nil, nil, nil, nil, nil) + Sourced.setup! + store2 = Sourced.config.store - expect(config.backend).to be_a(backend) + expect(store2).not_to be(store1) end - it 'fails loudly if the backend does not implement the Backend interface' do - expect { config.backend = Object.new }.to raise_error(Plumb::ParseError) + it 'works without a configure block' do + Sourced.setup! + + expect(Sourced.config.store).to be_a(Sourced::Store) + expect(Sourced.config.router).to be_a(Sourced::Router) + expect(Sourced.config).to be_frozen end end - describe '#pubsub' do - it 'defaults to PubSub::Test' do - expect(config.pubsub).to be_a(Sourced::PubSub::Test) + describe 'Sourced.reset!' do + it 'clears the singleton config' do + original = Sourced.config + Sourced.reset! + expect(Sourced.config).not_to be(original) + end + + it 'clears the stored configure block' do + Sourced.configure do |c| + c.worker_count = 8 + end + + Sourced.reset! + Sourced.setup! + + expect(Sourced.config.worker_count).to eq(2) end + end + + describe '#store=' do + it 'accepts a Sourced::Store instance directly' do + db = Sequel.sqlite + store = Sourced::Store.new(db) - it 'auto-sets PubSub::PG when backend is a Postgres database' do - config.backend = Sequel.postgres('sourced_test') - expect(config.pubsub).to be_a(Sourced::PubSub::PG) + config = described_class.new + config.store = store + expect(config.store).to be(store) end - it 'keeps current pubsub when backend is SQLite' do - config.backend = Sequel.sqlite - expect(config.pubsub).to be_a(Sourced::PubSub::Test) + it 'wraps a Sequel::SQLite::Database in a Store' do + db = Sequel.sqlite + + config = described_class.new + config.store = db + expect(config.store).to be_a(Sourced::Store) + expect(config.store.db).to be(db) end - it 'can be overridden with pubsub=' do - custom = Struct.new(:subscribe, :publish).new - config.pubsub = custom - expect(config.pubsub).to eq(custom) + it 'accepts any object implementing StoreInterface' do + fake_store = double('CustomStore', + installed?: true, install!: nil, append: nil, read: nil, + read_partition: nil, claim_next: nil, ack: nil, release: nil, + register_consumer_group: nil, worker_heartbeat: nil, + release_stale_claims: nil, notifier: nil + ) + + config = described_class.new + config.store = fake_store + expect(config.store).to be(fake_store) end - it 'fails loudly if the pubsub does not implement the PubSub interface' do - expect { config.pubsub = Object.new }.to raise_error(Plumb::ParseError) + it 'raises for objects not implementing StoreInterface' do + config = described_class.new + expect { config.store = Object.new }.to raise_error(Plumb::ParseError) end end - specify '#executor' do - expect(config.executor).to be_a(Sourced::AsyncExecutor) + describe '#error_strategy' do + it 'returns a default ErrorStrategy' do + config = described_class.new + expect(config.error_strategy).to be_a(Sourced::ErrorStrategy) + end + + it 'can be overridden with a custom callable' do + custom = ->(_e, _m, _g) {} + config = described_class.new + config.error_strategy = custom + expect(config.error_strategy).to be(custom) + end + + it 'raises if assigned a non-callable' do + config = described_class.new + expect { config.error_strategy = 'not callable' }.to raise_error(ArgumentError) + end end - describe 'subscribers' do - it 'triggers subscribers on #setup!' do - executor_class = nil - config.subscribe do |c| - executor_class = c.executor.class - end - config.executor = :thread + describe '#setup!' do + it 'is idempotent' do + config = described_class.new config.setup! - expect(executor_class).to eq(Sourced::ThreadExecutor) + store1 = config.store + router1 = config.router + config.setup! + expect(config.store).to be(store1) + expect(config.router).to be(router1) end - end - describe '#executor=()' do - specify ':async' do - config.executor = :async - expect(config.executor).to be_a(Sourced::AsyncExecutor) + it 'defaults to in-memory SQLite store when none configured' do + config = described_class.new + config.setup! + expect(config.store).to be_a(Sourced::Store) + expect(config.store.installed?).to be true + end + + it 'uses configured store when set' do + db = Sequel.sqlite + store = Sourced::Store.new(db) + store.install! + + config = described_class.new + config.store = store + config.setup! + expect(config.store).to be(store) end + end + + describe 'Sourced.load with global store' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::Store.new(db) } + + let(:decider_class) do + Class.new(Sourced::Decider) do + def self.name = 'ConfigLoadDecider' - specify ':thread' do - config.executor = :thread - expect(config.executor).to be_a(Sourced::ThreadExecutor) + partition_by :thing_id + consumer_group 'config-load-decider' + + state { |_| { count: 0 } } + end end - specify 'any #start interface' do - custom = Class.new do - def start; end + before do + store.install! + Sourced.configure do |c| + c.store = store end + end - config.executor = custom.new - expect(config.executor).to be_a(custom) + it 'uses global store when store: not provided' do + instance, read_result = Sourced.load(decider_class, thing_id: 'abc') + expect(instance.state[:count]).to eq(0) + expect(read_result.messages).to be_empty end - specify 'an invalid executor' do - expect { - config.executor = Object.new - }.to raise_error(ArgumentError) + it 'uses override store when store: provided' do + other_db = Sequel.sqlite + other_store = Sourced::Store.new(other_db) + other_store.install! + + instance, read_result = Sourced.load(decider_class, store: other_store, thing_id: 'abc') + expect(instance.state[:count]).to eq(0) + expect(read_result.messages).to be_empty end end end diff --git a/spec/consumer_spec.rb b/spec/consumer_spec.rb deleted file mode 100644 index 502818bc..00000000 --- a/spec/consumer_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -module TestConsumer - class TestConsumer - extend Sourced::Consumer - end -end - -RSpec.describe Sourced::Consumer do - describe '#group_id' do - it 'is class name by default' do - expect(TestConsumer::TestConsumer.consumer_info.group_id).to eq('TestConsumer::TestConsumer') - end - - it 'can be set' do - klass = Class.new do - extend Sourced::Consumer - - consumer do |info| - info.group_id = 'my-group' - end - end - - expect(klass.consumer_info.group_id).to eq('my-group') - end - end - - describe '#start_from' do - specify 'default is nil' do - expect(TestConsumer::TestConsumer.consumer_info.start_from.call).to be_nil - end - - it 'can be set to a proc that returns a Time' do - klass = Class.new do - extend Sourced::Consumer - - consumer do |info| - info.group_id = 'my-group' - info.start_from = -> { Time.new(2020, 1, 1) } - end - end - - expect(klass.consumer_info.start_from.call).to be_a(Time) - end - - it 'can be set to an :now which is a 5 second time window' do - klass = Class.new do - extend Sourced::Consumer - - consumer do |info| - info.group_id = 'my-group' - info.start_from = :now - end - end - - now = Time.now - Timecop.freeze(now) do - expect(klass.consumer_info.start_from.call).to eq(now - 5) - end - end - end - - describe '.on_exception' do - it 'fails the consumer group by default' do - group = double('group', error_context: {}, fail: true) - exception = StandardError.new('test error') - message = { id: 1 } - TestConsumer::TestConsumer.on_exception(exception, message, group) - expect(group).to have_received(:fail).with(exception:) - end - end -end diff --git a/spec/sourced/ccc/decider_spec.rb b/spec/decider_spec.rb similarity index 52% rename from spec/sourced/ccc/decider_spec.rb rename to spec/decider_spec.rb index 07570cb9..06c1e2ae 100644 --- a/spec/sourced/ccc/decider_spec.rb +++ b/spec/decider_spec.rb @@ -1,72 +1,72 @@ # frozen_string_literal: true require 'spec_helper' -require 'sourced/ccc' +require 'sourced' require 'sequel' -module CCCDeciderTestMessages - DeviceRegistered = Sourced::CCC::Message.define('decider_test.device.registered') do +module DeciderTestMessages + DeviceRegistered = Sourced::Message.define('decider_test.device.registered') do attribute :device_id, String attribute :name, String end - DeviceBound = Sourced::CCC::Message.define('decider_test.device.bound') do + DeviceBound = Sourced::Message.define('decider_test.device.bound') do attribute :device_id, String attribute :asset_id, String end - BindDevice = Sourced::CCC::Message.define('decider_test.bind_device') do + BindDevice = Sourced::Message.define('decider_test.bind_device') do attribute :device_id, String attribute :asset_id, String end - NotifyBound = Sourced::CCC::Message.define('decider_test.notify_bound') do + NotifyBound = Sourced::Message.define('decider_test.notify_bound') do attribute :device_id, String end - DelayedNotifyBound = Sourced::CCC::Message.define('decider_test.delayed_notify_bound') do + DelayedNotifyBound = Sourced::Message.define('decider_test.delayed_notify_bound') do attribute :device_id, String end - SymbolicBound = Sourced::CCC::Message.define('decider_test.symbolic_bound') do + SymbolicBound = Sourced::Message.define('decider_test.symbolic_bound') do attribute :device_id, String attribute :asset_id, String end - BindDeviceWithSymbol = Sourced::CCC::Message.define('decider_test.bind_device_with_symbol') do + BindDeviceWithSymbol = Sourced::Message.define('decider_test.bind_device_with_symbol') do attribute :device_id, String attribute :asset_id, String end end -class TestDeviceDecider < Sourced::CCC::Decider +class TestDeviceDecider < Sourced::Decider partition_by :device_id consumer_group 'device-decider-test' state { |_| { exists: false, bound: false } } - evolve CCCDeciderTestMessages::DeviceRegistered do |state, _evt| + evolve DeciderTestMessages::DeviceRegistered do |state, _evt| state[:exists] = true end - evolve CCCDeciderTestMessages::DeviceBound do |state, _evt| + evolve DeciderTestMessages::DeviceBound do |state, _evt| state[:bound] = true end - command CCCDeciderTestMessages::BindDevice do |state, cmd| + command DeciderTestMessages::BindDevice do |state, cmd| raise 'Not found' unless state[:exists] raise 'Already bound' if state[:bound] - event CCCDeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + event DeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id end - command CCCDeciderTestMessages::BindDeviceWithSymbol do |state, cmd| + command DeciderTestMessages::BindDeviceWithSymbol do |state, cmd| raise 'Not found' unless state[:exists] raise 'Already bound' if state[:bound] event :decider_test_symbolic_bound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id end - reaction CCCDeciderTestMessages::DeviceBound do |_state, evt| - CCCDeciderTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) + reaction DeciderTestMessages::DeviceBound do |_state, evt| + DeciderTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) end after_sync do |state:, messages:, events:| @@ -74,47 +74,47 @@ class TestDeviceDecider < Sourced::CCC::Decider end end -class TestDelayedReactionDecider < Sourced::CCC::Decider +class TestDelayedReactionDecider < Sourced::Decider partition_by :device_id consumer_group 'device-delayed-decider-test' state { |_| { exists: false, bound: false } } - evolve CCCDeciderTestMessages::DeviceRegistered do |state, _evt| + evolve DeciderTestMessages::DeviceRegistered do |state, _evt| state[:exists] = true end - evolve CCCDeciderTestMessages::DeviceBound do |state, _evt| + evolve DeciderTestMessages::DeviceBound do |state, _evt| state[:bound] = true end - command CCCDeciderTestMessages::BindDevice do |state, cmd| + command DeciderTestMessages::BindDevice do |state, cmd| raise 'Not found' unless state[:exists] raise 'Already bound' if state[:bound] - event CCCDeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + event DeciderTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id end - reaction CCCDeciderTestMessages::DeviceBound do |_state, evt| - dispatch(CCCDeciderTestMessages::DelayedNotifyBound, device_id: evt.payload.device_id) + reaction DeciderTestMessages::DeviceBound do |_state, evt| + dispatch(DeciderTestMessages::DelayedNotifyBound, device_id: evt.payload.device_id) .at(Time.now + 10) end end -RSpec.describe Sourced::CCC::Decider do +RSpec.describe Sourced::Decider do describe '.command' do it 'registers handler and #decide runs it' do - expect(TestDeviceDecider.handled_commands).to include(CCCDeciderTestMessages::BindDevice) - expect(TestDeviceDecider.handled_commands).to include(CCCDeciderTestMessages::BindDeviceWithSymbol) + expect(TestDeviceDecider.handled_commands).to include(DeciderTestMessages::BindDevice) + expect(TestDeviceDecider.handled_commands).to include(DeciderTestMessages::BindDeviceWithSymbol) instance = TestDeviceDecider.new instance.instance_variable_set(:@state, { exists: true, bound: false }) events = instance.decide( - CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) expect(events.size).to eq(1) - expect(events.first).to be_a(CCCDeciderTestMessages::DeviceBound) + expect(events.first).to be_a(DeciderTestMessages::DeviceBound) end end @@ -124,7 +124,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider instance.instance_variable_set(:@state, { exists: true, bound: false }) events = instance.decide( - CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) expect(events.size).to eq(1) @@ -136,17 +136,17 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider instance.instance_variable_set(:@state, { exists: true, bound: false }) events = instance.decide( - CCCDeciderTestMessages::BindDeviceWithSymbol.new(payload: { device_id: 'd1', asset_id: 'a1' }) + DeciderTestMessages::BindDeviceWithSymbol.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) expect(events.size).to eq(1) - expect(events.first).to be_a(CCCDeciderTestMessages::SymbolicBound) + expect(events.first).to be_a(DeciderTestMessages::SymbolicBound) end end describe '.handle_claim' do let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } + let(:store) { Sourced::Store.new(db) } before do store.install! @@ -154,17 +154,17 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider it 'evolves from history, decides commands, returns action pairs' do # Set up history - reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) store.append(reg) - history_msgs = [Sourced::CCC::PositionedMessage.new(reg, 1)] - guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 1) - history = Sourced::CCC::ReadResult.new(messages: history_msgs, guard: guard) + history_msgs = [Sourced::PositionedMessage.new(reg, 1)] + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 1) + history = Sourced::ReadResult.new(messages: history_msgs, guard: guard) - cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - cmd_positioned = Sourced::CCC::PositionedMessage.new(cmd, 2) + cmd = DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + cmd_positioned = Sourced::PositionedMessage.new(cmd, 2) - claim = Sourced::CCC::ClaimResult.new( + claim = Sourced::ClaimResult.new( offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', partition_value: { 'device_id' => 'd1' }, messages: [cmd_positioned], replaying: false, guard: guard @@ -180,32 +180,32 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider expect(source_msg).to eq(cmd_positioned) # Actions: Append(events with guard), Append(reactions), possibly sync - append_actions = Array(actions).select { |a| a.is_a?(Sourced::CCC::Actions::Append) } + append_actions = Array(actions).select { |a| a.is_a?(Sourced::Actions::Append) } expect(append_actions.size).to be >= 1 # First append has the events with guard event_append = append_actions.first - expect(event_append.messages.first).to be_a(CCCDeciderTestMessages::DeviceBound) + expect(event_append.messages.first).to be_a(DeciderTestMessages::DeviceBound) expect(event_append.guard).to eq(guard) # Second append has the reactions (no guard) if append_actions.size > 1 reaction_append = append_actions[1] - expect(reaction_append.messages.first).to be_a(CCCDeciderTestMessages::NotifyBound) + expect(reaction_append.messages.first).to be_a(DeciderTestMessages::NotifyBound) expect(reaction_append.guard).to be_nil end end it 'includes after_sync actions in action pairs' do - reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - history_msgs = [Sourced::CCC::PositionedMessage.new(reg, 1)] - guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 1) - history = Sourced::CCC::ReadResult.new(messages: history_msgs, guard: guard) + reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + history_msgs = [Sourced::PositionedMessage.new(reg, 1)] + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 1) + history = Sourced::ReadResult.new(messages: history_msgs, guard: guard) - cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - cmd_positioned = Sourced::CCC::PositionedMessage.new(cmd, 2) + cmd = DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + cmd_positioned = Sourced::PositionedMessage.new(cmd, 2) - claim = Sourced::CCC::ClaimResult.new( + claim = Sourced::ClaimResult.new( offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', partition_value: { 'device_id' => 'd1' }, messages: [cmd_positioned], replaying: false, guard: guard @@ -214,18 +214,18 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider pairs = TestDeviceDecider.handle_claim(claim, history: history) actions = Array(pairs.first.first) - after_sync_action = actions.find { |a| a.is_a?(Sourced::CCC::Actions::AfterSync) } + after_sync_action = actions.find { |a| a.is_a?(Sourced::Actions::AfterSync) } expect(after_sync_action).not_to be_nil end it 'returns [OK, msg] for non-command messages' do - reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - reg_positioned = Sourced::CCC::PositionedMessage.new(reg, 1) + reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + reg_positioned = Sourced::PositionedMessage.new(reg, 1) - guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 0) - history = Sourced::CCC::ReadResult.new(messages: [], guard: guard) + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 0) + history = Sourced::ReadResult.new(messages: [], guard: guard) - claim = Sourced::CCC::ClaimResult.new( + claim = Sourced::ClaimResult.new( offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', partition_value: { 'device_id' => 'd1' }, messages: [reg_positioned], replaying: false, guard: guard @@ -235,41 +235,41 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider expect(pairs.size).to eq(1) actions, source_msg = pairs.first - expect(actions).to eq(Sourced::CCC::Actions::OK) + expect(actions).to eq(Sourced::Actions::OK) expect(source_msg).to eq(reg_positioned) end it 'returns schedule actions for delayed reaction dispatches' do - reg = CCCDeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - history_msgs = [Sourced::CCC::PositionedMessage.new(reg, 1)] - guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 1) - history = Sourced::CCC::ReadResult.new(messages: history_msgs, guard: guard) + reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + history_msgs = [Sourced::PositionedMessage.new(reg, 1)] + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 1) + history = Sourced::ReadResult.new(messages: history_msgs, guard: guard) - cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - claim = Sourced::CCC::ClaimResult.new( + cmd = DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + claim = Sourced::ClaimResult.new( offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', partition_value: { 'device_id' => 'd1' }, - messages: [Sourced::CCC::PositionedMessage.new(cmd, 2)], + messages: [Sourced::PositionedMessage.new(cmd, 2)], replaying: false, guard: guard ) pairs = TestDelayedReactionDecider.handle_claim(claim, history: history) actions = pairs.first.first - schedule_action = Array(actions).find { |action| action.is_a?(Sourced::CCC::Actions::Schedule) } + schedule_action = Array(actions).find { |action| action.is_a?(Sourced::Actions::Schedule) } expect(schedule_action).not_to be_nil - expect(schedule_action.messages.first).to be_a(CCCDeciderTestMessages::DelayedNotifyBound) + expect(schedule_action.messages.first).to be_a(DeciderTestMessages::DelayedNotifyBound) end it 'invariant violation propagates as error' do - guard = Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 0) - history = Sourced::CCC::ReadResult.new(messages: [], guard: guard) + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 0) + history = Sourced::ReadResult.new(messages: [], guard: guard) - cmd = CCCDeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - cmd_positioned = Sourced::CCC::PositionedMessage.new(cmd, 1) + cmd = DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + cmd_positioned = Sourced::PositionedMessage.new(cmd, 1) - claim = Sourced::CCC::ClaimResult.new( + claim = Sourced::ClaimResult.new( offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', partition_value: { 'device_id' => 'd1' }, messages: [cmd_positioned], replaying: false, guard: guard @@ -285,9 +285,9 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider describe '.handled_messages' do it 'includes commands and react types but not evolve types' do msgs = TestDeviceDecider.handled_messages - expect(msgs).to include(CCCDeciderTestMessages::BindDevice) - expect(msgs).to include(CCCDeciderTestMessages::DeviceBound) # reaction - expect(msgs).not_to include(CCCDeciderTestMessages::DeviceRegistered) # evolve only + expect(msgs).to include(DeciderTestMessages::BindDevice) + expect(msgs).to include(DeciderTestMessages::DeviceBound) # reaction + expect(msgs).not_to include(DeciderTestMessages::DeviceRegistered) # evolve only end end @@ -306,7 +306,7 @@ class TestDelayedReactionDecider < Sourced::CCC::Decider describe 'inheritance' do it 'subclass inherits command handlers' do subclass = Class.new(TestDeviceDecider) - expect(subclass.handled_commands).to include(CCCDeciderTestMessages::BindDevice) + expect(subclass.handled_commands).to include(DeciderTestMessages::BindDevice) end end end diff --git a/spec/dispatcher_spec.rb b/spec/dispatcher_spec.rb index f7aadcf5..eb3b493b 100644 --- a/spec/dispatcher_spec.rb +++ b/spec/dispatcher_spec.rb @@ -1,31 +1,283 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' +require 'sequel' + +module DispatcherTestMessages + DeviceRegistered = Sourced::Message.define('dispatch_test.device.registered') do + attribute :device_id, String + attribute :name, String + end + + DeviceBound = Sourced::Message.define('dispatch_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDevice = Sourced::Message.define('dispatch_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end + + DelayedNotify = Sourced::Message.define('dispatch_test.delayed_notify') do + attribute :device_id, String + end +end + +class DispatchTestDecider < Sourced::Decider + partition_by :device_id + consumer_group 'dispatch-test-decider' + + state { |_| { exists: false, bound: false } } + + evolve DispatcherTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + evolve DispatcherTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command DispatcherTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event DispatcherTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end + + reaction DispatcherTestMessages::DeviceBound do |_state, evt| + dispatch(DispatcherTestMessages::DelayedNotify, device_id: evt.payload.device_id) + .at(Time.now + 2) + end +end + +class DispatchTestProjector < Sourced::Projector::StateStored + partition_by :device_id + consumer_group 'dispatch-test-projector' + + state { |_| { devices: [] } } + + evolve DispatcherTestMessages::DeviceRegistered do |state, evt| + state[:devices] << evt.payload.name + end + + evolve DispatcherTestMessages::DeviceBound do |state, _evt| + # noop + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + end +end RSpec.describe Sourced::Dispatcher do - let(:reactor1) { double('Reactor1', handled_messages: [double(type: 'event1')], consumer_info: double(group_id: 'Reactor1')) } - let(:reactor2) { double('Reactor2', handled_messages: [double(type: 'event2')], consumer_info: double(group_id: 'Reactor2')) } - let(:reactors) { Set.new([reactor1, reactor2]) } - let(:backend_notifier) { Sourced::InlineNotifier.new } - let(:backend) { double('Backend', notifier: backend_notifier) } - let(:router) { instance_double(Sourced::Router, async_reactors: reactors, backend: backend) } + let(:db) { Sequel.sqlite } + let(:notifier) { Sourced::InlineNotifier.new } + let(:store) { Sourced::Store.new(db, notifier: notifier) } + let(:router) { Sourced::Router.new(store: store) } let(:logger) { instance_double('Logger', info: nil, warn: nil, debug: nil) } - let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } - subject(:dispatcher) do - described_class.new( - router: router, - worker_count: 2, - batch_size: 1, - max_drain_rounds: 10, - catchup_interval: 5, - work_queue: work_queue, - logger: logger - ) + before do + store.install! + router.register(DispatchTestDecider) + router.register(DispatchTestProjector) + end + + describe 'Store notifications' do + it 'append triggers notify_new_messages' do + expect(notifier).to receive(:notify_new_messages).with(['dispatch_test.device.registered']) + + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + end + + it 'start_consumer_group triggers notify_reactor_resumed' do + store.stop_consumer_group('dispatch-test-decider') + + expect(notifier).to receive(:notify_reactor_resumed).with('dispatch-test-decider') + + store.start_consumer_group('dispatch-test-decider') + end + + it 'empty append does not notify' do + expect(notifier).not_to receive(:notify_new_messages) + + store.append([]) + end + end + + describe 'batch_size' do + it 'claim_next with batch_size limits returned messages' do + # Append 5 messages for the same partition + 5.times do |i| + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: "Sensor #{i}" }) + ) + end + + claim = store.claim_next( + 'dispatch-test-projector', + partition_by: ['device_id'], + handled_types: DispatchTestProjector.handled_messages.map(&:type), + worker_id: 'w1', + batch_size: 2 + ) + + expect(claim).not_to be_nil + expect(claim.messages.size).to eq(2) + end + + it 'claim_next without batch_size returns all messages' do + 5.times do |i| + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: "Sensor #{i}" }) + ) + end + + claim = store.claim_next( + 'dispatch-test-projector', + partition_by: ['device_id'], + handled_types: DispatchTestProjector.handled_messages.map(&:type), + worker_id: 'w1' + ) + + expect(claim).not_to be_nil + expect(claim.messages.size).to eq(5) + end + end + + describe Sourced::Dispatcher::NotificationQueuer do + let(:queuer) do + described_class.new( + work_queue: work_queue, + reactors: [DispatchTestDecider, DispatchTestProjector] + ) + end + + it 'maps message types to interested reactors' do + # DeviceRegistered is handled by projector (via evolve), + # BindDevice is handled by decider (via command) + queuer.call('messages_appended', 'dispatch_test.device.registered,dispatch_test.bind_device') + + popped = [] + popped << work_queue.pop + popped << work_queue.pop + + expect(popped).to contain_exactly(DispatchTestDecider, DispatchTestProjector) + end + + it 'maps group_id to reactor for reactor_resumed' do + queuer.call('reactor_resumed', 'dispatch-test-decider') + + popped = work_queue.pop + expect(popped).to eq(DispatchTestDecider) + end + + it 'ignores unknown message types' do + queuer.call('messages_appended', 'unknown.type') + + # Queue should be empty — push a sentinel to avoid blocking + work_queue.push(nil) + expect(work_queue.pop).to be_nil + end + + it 'ignores unknown group_ids' do + queuer.call('reactor_resumed', 'unknown-group') + + work_queue.push(nil) + expect(work_queue.pop).to be_nil + end + end + + describe Sourced::Worker do + let(:worker) do + described_class.new( + work_queue: work_queue, + router: router, + name: 'test-worker', + batch_size: 50, + max_drain_rounds: 10, + logger: logger + ) + end + + describe '#tick' do + it 'processes one claim for a reactor' do + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + result = worker.tick(DispatchTestProjector) + expect(result).to be true + end + + it 'returns false when no work available' do + result = worker.tick(DispatchTestDecider) + expect(result).to be false + end + end + + describe '#drain' do + it 'processes until no more work for reactor' do + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor A' }) + ) + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor B' }) + ) + + worker.instance_variable_set(:@running, true) + worker.drain(DispatchTestProjector) + + # Both partitions should have been processed — no more work + result = worker.tick(DispatchTestProjector) + expect(result).to be false + end + + it 're-enqueues reactor when max_drain_rounds reached' do + # Create a worker with max_drain_rounds: 1 + bounded_worker = described_class.new( + work_queue: work_queue, + router: router, + name: 'bounded', + batch_size: 50, + max_drain_rounds: 1, + logger: logger + ) + + # Append messages for 2 partitions + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'A' }) + ) + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'B' }) + ) + + bounded_worker.instance_variable_set(:@running, true) + bounded_worker.drain(DispatchTestProjector) + + # Should have re-enqueued — pop it back + popped = work_queue.pop + expect(popped).to eq(DispatchTestProjector) + end + end end - describe '#workers' do + describe 'Dispatcher wiring' do + subject(:dispatcher) do + described_class.new( + router: router, + worker_count: 2, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 5, + work_queue: work_queue, + logger: logger + ) + end + it 'creates the requested number of workers' do expect(dispatcher.workers.size).to eq(2) end @@ -35,34 +287,89 @@ expect(names).to include(match(/worker-0$/)) expect(names).to include(match(/worker-1$/)) end - end - describe '#spawn_into' do - it 'spawns via #spawn when task responds to spawn (executor Task)' do + it 'spawns via #spawn when task responds to spawn' do task = double('Task') - # 1 notifier + 1 catchup_poller + 2 workers = 4 spawns - expect(task).to receive(:spawn).exactly(4).times + # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns + expect(task).to receive(:spawn).exactly(6).times dispatcher.spawn_into(task) end - it 'spawns via #async when task does not respond to spawn (Async::Task)' do + it 'spawns via #async when task does not respond to spawn' do task = Object.new def task.async; end - # 1 notifier + 1 catchup_poller + 2 workers = 4 spawns - expect(task).to receive(:async).exactly(4).times + expect(task).to receive(:async).exactly(6).times dispatcher.spawn_into(task) end - end - describe '#stop' do - it 'stops all components' do + it '#stop stops all components' do dispatcher.stop - # Workers should be stopped (running = false) dispatcher.workers.each do |w| - # Workers are stopped — verify by checking they won't loop in run expect(w.instance_variable_get(:@running)).to eq(false) end end + + it 'creates zero workers when worker_count is 0' do + d = described_class.new( + router: router, + worker_count: 0, + logger: logger + ) + expect(d.workers).to be_empty + end + end + + describe 'Integration: append → notify → queue → worker' do + it 'InlineNotifier fires synchronously through the full pipeline' do + # Build dispatcher which subscribes NotificationQueuer to the store's notifier + dispatcher = described_class.new( + router: router, + worker_count: 1, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 60, # long interval — we test synchronous path only + work_queue: work_queue, + logger: logger + ) + + # Append triggers notifier → NotificationQueuer → WorkQueue + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + # Pop from queue — should have the reactors that handle this type + popped = work_queue.pop + expect([DispatchTestDecider, DispatchTestProjector]).to include(popped) + + # Worker processes the message + worker = dispatcher.workers.first + result = worker.tick(popped) + expect(result).to be true + + dispatcher.stop + end + end + + describe 'scheduled message promotion' do + it 'promotes delayed reactions into the main log when due' do + store.append( + DispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + DispatcherTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + expect(router.handle_next_for(DispatchTestDecider)).to be true + expect(db[:sourced_scheduled_messages].count).to eq(1) + + Timecop.freeze(Time.now + 3) do + expect(store.update_schedule!).to eq(1) + end + + conds = DispatcherTestMessages::DelayedNotify.to_conditions(device_id: 'd1') + result = store.read(conds) + expect(result.messages.map(&:class)).to include(DispatcherTestMessages::DelayedNotify) + end end end diff --git a/spec/durable_workflow_spec.rb b/spec/durable_workflow_spec.rb index c0dd4b31..1fdd1572 100644 --- a/spec/durable_workflow_spec.rb +++ b/spec/durable_workflow_spec.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true require 'spec_helper' -require 'sourced/durable_workflow' +require 'sourced' +require 'sourced/store' +require 'sourced/testing/rspec' +require 'sequel' module DurableTests FilledStringArray = Sourced::Types::Array[String].with(size: 1..) @@ -11,7 +14,7 @@ def self.resolve = '11.111.111' end class Geolocator - def self.locate(ip) = 'London, UK' + def self.locate(_ip) = 'London, UK' end class Task < Sourced::DurableWorkflow @@ -73,8 +76,8 @@ def execute durable def iterate index = context[:index] @numbers[index..].each do |n| - # Simulate a failure on the first run raise 'oopsie!' if n == 3 && index == 0 + context[:index] += 1 context[:results] << n * 2 end @@ -84,7 +87,7 @@ def execute class WithDelay < Sourced::DurableWorkflow def execute name = get_name - wait 10 # seconds + wait 10 notify(name) end @@ -92,35 +95,45 @@ def execute 'Joe' end - durable def notify(name) + durable def notify(_name) true end end end RSpec.describe Sourced::DurableWorkflow do - let(:stream_id) { 'durable-test-1' } + include Sourced::Testing::RSpec + + let(:workflow_id) { 'durable-test-1' } let(:name) { 'Joe' } + def make_started(klass, args: [], workflow_id: 'durable-test-1') + klass::WorkflowStarted.new(payload: { workflow_id:, args: }) + end + context 'with happy path' do it 'starts and produces new messages until completing workflow' do - started = DurableTests::Task::WorkflowStarted.parse(stream_id:, payload: { args: [name] }) + started = make_started(DurableTests::Task, args: [name]) history = [started] until history.last.is_a?(DurableTests::Task::WorkflowComplete) next_action = DurableTests::Task.handle(history.last, history:) - expect(next_action).to be_a Sourced::Actions::AppendAfter + expect(next_action).to be_a Sourced::Actions::Append history += next_action.messages end - assert_messages(history, [ - [DurableTests::Task::WorkflowStarted, stream_id, args: [name]], - [DurableTests::Task::StepStarted, stream_id, key: Sourced::Types::Any, step_name: :get_ip, args: []], - [DurableTests::Task::StepComplete, stream_id, key: Sourced::Types::Any, step_name: :get_ip, output: '11.111.111'], - [DurableTests::Task::StepStarted, stream_id, key: Sourced::Types::Any, step_name: :geolocate, args: ['11.111.111']], - [DurableTests::Task::StepComplete, stream_id, key: Sourced::Types::Any, step_name: :geolocate, output: 'London, UK'], - [DurableTests::Task::WorkflowComplete, stream_id, output: 'Hello Joe, your IP is 11.111.111 and its location is London, UK'] + expect(history.map(&:class)).to eq([ + DurableTests::Task::WorkflowStarted, + DurableTests::Task::StepStarted, + DurableTests::Task::StepComplete, + DurableTests::Task::StepStarted, + DurableTests::Task::StepComplete, + DurableTests::Task::WorkflowComplete ]) + + last = history.last + expect(last.payload.output).to eq('Hello Joe, your IP is 11.111.111 and its location is London, UK') + expect(last.payload.workflow_id).to eq(workflow_id) end end @@ -128,88 +141,78 @@ def execute it 'produces StepFailed message' do expect(DurableTests::IPResolver).to receive(:resolve).and_raise('Network Error!') - started = DurableTests::Task::WorkflowStarted.parse(stream_id:, payload: { args: [name] }) + started = make_started(DurableTests::Task, args: [name]) history = [started] until history.last.is_a?(DurableTests::Task::StepFailed) next_action = DurableTests::Task.handle(history.last, history:) - expect(next_action).to be_a Sourced::Actions::AppendAfter + expect(next_action).to be_a Sourced::Actions::Append history += next_action.messages end - assert_messages(history, [ - [DurableTests::Task::WorkflowStarted, stream_id, args: [name]], - [DurableTests::Task::StepStarted, stream_id, step_name: :get_ip, args: []], - [DurableTests::Task::StepFailed, stream_id, step_name: :get_ip, error_class: 'RuntimeError', error_message: '#', backtrace: DurableTests::FilledStringArray], + expect(history.map(&:class)).to eq([ + DurableTests::Task::WorkflowStarted, + DurableTests::Task::StepStarted, + DurableTests::Task::StepFailed ]) + expect(history.last.payload.error_class).to eq('RuntimeError') + expect(DurableTests::FilledStringArray).to be === history.last.payload.backtrace end end context 'with previously successful step' do it 'does not invoke step again, using cached result instead' do - history = build_history([ - [DurableTests::Task::WorkflowStarted, stream_id, args: [name] ], - [DurableTests::Task::StepStarted, stream_id, key: Sourced::DurableWorkflow.step_key(:get_ip, []), step_name: :get_ip, args: [] ], - [DurableTests::Task::StepComplete, stream_id, key: Sourced::DurableWorkflow.step_key(:get_ip, []), step_name: :get_ip, output: '11.111.111' ], - [DurableTests::Task::StepStarted, stream_id, key: Sourced::DurableWorkflow.step_key(:geolocate, ['11.111.111']), step_name: :geolocate, args: ['11.111.111'] ], - ]) + get_ip_key = Sourced::DurableWorkflow.step_key(:get_ip, []) + geolocate_key = Sourced::DurableWorkflow.step_key(:geolocate, ['11.111.111']) - # IPResolver's output is already in history expect(DurableTests::IPResolver).not_to receive(:resolve) - # Geolocator hasn't been invoked yet - expect(DurableTests::Geolocator).to receive(:locate).with('11.111.111').and_return 'Santiago, Chile' - - # Handle last message and produce new messages until workflow completes. - until history.last.is_a?(DurableTests::Task::WorkflowComplete) - next_action = DurableTests::Task.handle(history.last, history:) - expect(next_action).to be_a Sourced::Actions::AppendAfter - history += next_action.messages - end - - # Load current task state from history - task = DurableTests::Task.from(history) - expect(task.status).to eq(:complete) - expect(task.output).to eq('Hello Joe, your IP is 11.111.111 and its location is Santiago, Chile') + expect(DurableTests::Geolocator).to receive(:locate).with('11.111.111').and_return('Santiago, Chile') + + with_reactor(DurableTests::Task, workflow_id: workflow_id) + .given(DurableTests::Task::WorkflowStarted, workflow_id:, args: [name]) + .given(DurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) + .given(DurableTests::Task::StepComplete, workflow_id:, key: get_ip_key, step_name: :get_ip, output: '11.111.111') + .when(DurableTests::Task::StepStarted, workflow_id:, key: geolocate_key, step_name: :geolocate, args: ['11.111.111']) + .then( + DurableTests::Task::StepComplete.new(payload: { + workflow_id:, key: geolocate_key, step_name: :geolocate, output: 'Santiago, Chile' + }) + ) end end context 'when workflow is finally failed' do it 'does not try again' do - history = build_history([ - [DurableTests::Task::WorkflowStarted, stream_id, args: [name]], - [DurableTests::Task::StepStarted, stream_id, key: Sourced::DurableWorkflow.step_key(:get_ip, []), step_name: :get_ip, args: []], - [DurableTests::Task::StepFailed, stream_id, key: Sourced::DurableWorkflow.step_key(:get_ip, []), step_name: :get_ip, error_class: 'NewtworkError', error_message: 'foo', backtrace: []], - [DurableTests::Task::WorkflowFailed, stream_id, nil], - ]) - - step_started = DurableTests::Task::StepStarted.parse(stream_id:, payload: { key: Sourced::DurableWorkflow.step_key(:get_ip, []), step_name: :get_ip, args: [] }) - - next_action = DurableTests::Task.handle(step_started, history:) - expect(next_action).to eq(Sourced::Actions::OK) + get_ip_key = Sourced::DurableWorkflow.step_key(:get_ip, []) + + with_reactor(DurableTests::Task, workflow_id: workflow_id) + .given(DurableTests::Task::WorkflowStarted, workflow_id:, args: [name]) + .given(DurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) + .given(DurableTests::Task::StepFailed, workflow_id:, key: get_ip_key, step_name: :get_ip, error_class: 'NetworkError', error_message: 'foo', backtrace: []) + .given(DurableTests::Task::WorkflowFailed, workflow_id:) + .when(DurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) + .then(Sourced::Testing::RSpec::NONE) end end context 'with a different workflow handling irrelevant messages' do it 'blows up' do - started = DurableTests::AnotherTask::WorkflowStarted.parse(stream_id:, payload: { args: [name] }) - history = [started] - - expect { - DurableTests::Task.handle(history.last, history:) - }.to raise_error(Sourced::DurableWorkflow::UnknownMessageError) + with_reactor(DurableTests::Task, workflow_id: workflow_id) + .when(DurableTests::AnotherTask::WorkflowStarted, workflow_id:, args: [name]) + .then(Sourced::DurableWorkflow::UnknownMessageError) end end describe 'caching method calls by signature' do it 'only invokes methods with the same arguments once per workflow' do - started = DurableTests::MultiArgTask::WorkflowStarted.parse(stream_id:, payload: { args: [] }) + started = make_started(DurableTests::MultiArgTask) history = [started] allow(DurableTests::Doubler).to receive(:double).and_call_original until history.last.is_a?(DurableTests::MultiArgTask::WorkflowComplete) next_action = DurableTests::MultiArgTask.handle(history.last, history:) - expect(next_action).to be_a Sourced::Actions::AppendAfter + expect(next_action).to be_a Sourced::Actions::Append history += next_action.messages end @@ -221,10 +224,9 @@ def execute describe 'limited retries' do it 'retries the configured number of times until it fails the workflow' do - started = DurableTests::Retryable::WorkflowStarted.parse(stream_id:, payload: { args: [] }) + started = make_started(DurableTests::Retryable) history = [started] - # until history.last.is_a?(DurableTests::MultiArgTask::WorkflowFailed) 6.times do next_action = DurableTests::Retryable.handle(history.last, history:) history += next_action.messages if next_action.respond_to?(:messages) @@ -246,7 +248,7 @@ def execute context 'with context preserved across failures' do it 'tracks context changes in event history' do - started = DurableTests::WithContext::WorkflowStarted.parse(stream_id:, seq: 1, payload: {}) + started = make_started(DurableTests::WithContext) history = [started] until history.last.is_a?(DurableTests::WithContext::WorkflowComplete) @@ -257,16 +259,20 @@ def execute task = DurableTests::WithContext.from(history) expect(task.output).to eq([2, 4, 6, 8, 10]) - assert_messages(history, [ - [DurableTests::WithContext::WorkflowStarted, stream_id, args: []], - [DurableTests::WithContext::StepStarted, stream_id, step_name: :iterate, args: []], - [DurableTests::WithContext::StepFailed, stream_id, step_name: :iterate, error_class: 'RuntimeError', error_message: '#', backtrace: DurableTests::FilledStringArray], - [DurableTests::WithContext::ContextUpdated, stream_id, context: { index: 2, results: [2, 4] }], - [DurableTests::WithContext::StepStarted, stream_id, step_name: :iterate, args: []], - [DurableTests::WithContext::StepComplete, stream_id, step_name: :iterate, output: [3, 4, 5]], - [DurableTests::WithContext::ContextUpdated, stream_id, context: { index: 5, results: [2, 4, 6, 8, 10] }], - [DurableTests::WithContext::WorkflowComplete, stream_id, output: [2, 4, 6, 8, 10]], + expect(history.map(&:class)).to eq([ + DurableTests::WithContext::WorkflowStarted, + DurableTests::WithContext::StepStarted, + DurableTests::WithContext::StepFailed, + DurableTests::WithContext::ContextUpdated, + DurableTests::WithContext::StepStarted, + DurableTests::WithContext::StepComplete, + DurableTests::WithContext::ContextUpdated, + DurableTests::WithContext::WorkflowComplete ]) + + ctx_events = history.select { |m| m.is_a?(DurableTests::WithContext::ContextUpdated) } + expect(ctx_events[0].payload.context).to eq(index: 2, results: [2, 4]) + expect(ctx_events[1].payload.context).to eq(index: 5, results: [2, 4, 6, 8, 10]) end end @@ -275,7 +281,7 @@ def execute now = Time.now Timecop.freeze(now) do - started = DurableTests::WithDelay::WorkflowStarted.parse(stream_id:, seq: 1, payload: {}) + started = make_started(DurableTests::WithDelay) history = [started] next_action = DurableTests::WithDelay.handle(history.last, history:) @@ -290,14 +296,12 @@ def execute expect(history.last).to be_a(DurableTests::WithDelay::WaitStarted) expect(history.last.payload.at).to eq(now + 10) - # Now handle the WaitStarted to produce a scheduled event next_action = DurableTests::WithDelay.handle(history.last, history:) expect(next_action).to be_a(Sourced::Actions::Schedule) expect(next_action.at).to eq(now + 10) expect(next_action.messages.first).to be_a(DurableTests::WithDelay::WaitEnded) - expect(next_action.messages.first.stream_id).to eq(stream_id) + expect(next_action.messages.first.payload.workflow_id).to eq('durable-test-1') - # Now handle the scheduled event history << next_action.messages.first until history.last.is_a?(DurableTests::WithDelay::WorkflowComplete) @@ -319,33 +323,30 @@ def execute end end + describe 'end-to-end via store + router' do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::Store.new(db) } + let(:router) { Sourced::Router.new(store:) } - private - - # assert_messages( - # messages, - # [ - # [SomeMessage, some_stream, some_payload] - # ] - # ) - def assert_messages(messages, expected_message_tuples) - expect(messages.size).to eq(expected_message_tuples.size) - - messages.each.with_index do |m, idx| - e = expected_message_tuples[idx] - expect(m).to be_a(e[0]) - expect(m.stream_id).to eq(e[1]) - payload = m.payload.to_h - e[2].each do |k, v| - expect(v).to be === payload[k] - end if e[2] - end - end + before { store.install! } + + it 'drains two concurrent workflows to completion' do + router.register(DurableTests::Task) + + wf1_id = "wf-#{SecureRandom.uuid}" + wf2_id = "wf-#{SecureRandom.uuid}" + store.append([DurableTests::Task::WorkflowStarted.new(payload: { workflow_id: wf1_id, args: ['Alice'] })]) + store.append([DurableTests::Task::WorkflowStarted.new(payload: { workflow_id: wf2_id, args: ['Bob'] })]) - def build_history(message_tuples) - message_tuples.map do |(message_class, stream_id, payload)| - message_class.parse(stream_id:, payload:) + router.drain + + wf1, = Sourced.load(DurableTests::Task, store:, workflow_id: wf1_id) + wf2, = Sourced.load(DurableTests::Task, store:, workflow_id: wf2_id) + + expect(wf1.status).to eq(:complete) + expect(wf1.output).to include('Alice') + expect(wf2.status).to eq(:complete) + expect(wf2.output).to include('Bob') end end end - diff --git a/spec/error_strategy_spec.rb b/spec/error_strategy_spec.rb deleted file mode 100644 index 58968c70..00000000 --- a/spec/error_strategy_spec.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::ErrorStrategy do - let(:group_class) do - Struct.new(:status, :retry_at, :error_context) do - def retry(later, ctx = {}) - self.retry_at = later - self.error_context.merge!(ctx) - self - end - - def stop(message: nil) - self.status = :stopped - self.error_context[:message] = message if message - self - end - - def fail(exception: nil) - self.status = :failed - if exception - self.error_context[:exception_class] = exception.class.to_s - self.error_context[:exception_message] = exception.message - end - self - end - end - end - let(:group) { group_class.new(:active, nil, {}) } - let(:exception) { StandardError.new } - let(:message) { Sourced::Message.new } - - before do - allow(group).to receive(:retry).and_call_original - allow(group).to receive(:stop).and_call_original - allow(group).to receive(:fail).and_call_original - end - - it 'fails the group immediatly by default' do - strategy = described_class.new - strategy.call(exception, message, group) - expect(group).to have_received(:fail).with(exception:) - end - - it 'can be configured with retries' do - now = Time.new(2020, 1, 1).utc - - retries = [] - fail_call = nil - - strategy = described_class.new do |s| - s.retry(times: 3, after: 5, backoff: ->(retry_after, retry_count) { retry_after * retry_count }) - - s.on_retry do |n, exception, message, later| - retries << [n, exception, message, later] - end - - s.on_fail do |exception, message| - fail_call = [exception, message] - end - end - - Timecop.freeze(now) do - strategy.call(exception, message, group) - strategy.call(exception, message, group) - strategy.call(exception, message, group) - - expect(fail_call).to be(nil) - - strategy.call(exception, message, group) - - expect(retries).to eq([ - [1, exception, message, now + 5], - [2, exception, message, now + 10], - [3, exception, message, now + 15] - ]) - - expect(fail_call).to eq([exception, message]) - - expect(group.retry_at).to eq(now + 15) - expect(group.status).to eq(:failed) - expect(group.error_context[:retry_count]).to eq(4) - end - end -end diff --git a/spec/evolve_spec.rb b/spec/evolve_spec.rb index 6439804e..7b470016 100644 --- a/spec/evolve_spec.rb +++ b/spec/evolve_spec.rb @@ -1,115 +1,102 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' -module EvolveTest - class Reactor - include Sourced::Evolve - - Event1 = Sourced::Message.define('evolvetest.reactor.event1') - Event2 = Sourced::Message.define('evolvetest.reactor.event2') - Event3 = Sourced::Message.define('evolvetest.reactor.event3') - - state do |_id| - [] - end - - event Event1 do |state, event| - state << event - end - - event Event2 do |state, event| - state << event - end +module EvolveTestMessages + ItemAdded = Sourced::Message.define('evolve_test.item.added') do + attribute :item_id, String + attribute :name, String end - class ChildReactor < Reactor - event Event3 + ItemRemoved = Sourced::Message.define('evolve_test.item.removed') do + attribute :item_id, String end - class Noop - include Sourced::Evolve + Unhandled = Sourced::Message.define('evolve_test.unhandled') do + attribute :foo, String + end +end - state do |_id| - [] - end +RSpec.describe Sourced::Evolve do + let(:evolver_class) do + Class.new do + include Sourced::Evolve - event Reactor::Event1 - end + def initialize(partition_values = {}) + @partition_values = partition_values + end - class EvolveAll - include Sourced::Evolve + state do |partition_values| + { items: [], partition_values: partition_values } + end - state do |_id| - [] - end + evolve EvolveTestMessages::ItemAdded do |state, msg| + state[:items] << { id: msg.payload.item_id, name: msg.payload.name } + end - evolve_all Reactor do |state, event| - state << event + evolve EvolveTestMessages::ItemRemoved do |state, msg| + state[:items].reject! { |i| i[:id] == msg.payload.item_id } + end end end - class WithBefore < EvolveAll - before_evolve do |state, event| - state << event.seq + describe '.state' do + it 'initializes state with partition values hash' do + instance = evolver_class.new(key1: 'val1', key2: 'val2') + expect(instance.state[:partition_values]).to eq({ key1: 'val1', key2: 'val2' }) end end -end -RSpec.describe Sourced::Evolve do describe '#evolve' do - it 'evolves instance' do - evt1 = EvolveTest::Reactor::Event1.new(stream_id: '1', seq: 1) - evt2 = EvolveTest::Reactor::Event2.new(stream_id: '1', seq: 2) - state = EvolveTest::Reactor.new.evolve([evt1, evt2]) - expect(state).to eq([evt1, evt2]) - end + it 'applies registered handlers in order' do + instance = evolver_class.new + messages = [ + EvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }), + EvolveTestMessages::ItemAdded.new(payload: { item_id: 'i2', name: 'Banana' }), + EvolveTestMessages::ItemRemoved.new(payload: { item_id: 'i1' }) + ] + + instance.evolve(messages) - it 'accepts single message' do - evt1 = EvolveTest::Reactor::Event1.new(stream_id: '1', seq: 1) - instance = EvolveTest::Reactor.new - instance.evolve(evt1) - expect(instance.state).to eq([evt1]) + expect(instance.state[:items]).to eq([{ id: 'i2', name: 'Banana' }]) end - end - specify '.handled_messages_for_evolve' do - expect(EvolveTest::Reactor.handled_messages_for_evolve).to eq([ - EvolveTest::Reactor::Event1, - EvolveTest::Reactor::Event2 - ]) - - expect(EvolveTest::ChildReactor.handled_messages_for_evolve).to eq([ - EvolveTest::Reactor::Event1, - EvolveTest::Reactor::Event2, - EvolveTest::Reactor::Event3, - ]) - end + it 'skips unregistered message types' do + instance = evolver_class.new + messages = [ + EvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }), + EvolveTestMessages::Unhandled.new(payload: { foo: 'bar' }) + ] - specify '.evolve handlers without a block' do - expect(EvolveTest::Noop.handled_messages_for_evolve).to eq([EvolveTest::Reactor::Event1]) + instance.evolve(messages) - evt1 = EvolveTest::Reactor::Event1.new(stream_id: '1', seq: 1) - new_state = EvolveTest::Noop.new.evolve([evt1]) - expect(new_state).to eq([]) + expect(instance.state[:items]).to eq([{ id: 'i1', name: 'Apple' }]) + end end - specify '.evolve_all' do - evt1 = EvolveTest::Reactor::Event1.new(stream_id: '1', seq: 1) - evt2 = EvolveTest::Reactor::Event2.new(stream_id: '1', seq: 2) - evolver = EvolveTest::EvolveAll.new - expect(evolver.state).to eq([]) - new_state = evolver.evolve([evt1, evt2]) - expect(new_state).to eq([evt1, evt2]) - expect(evolver.state).to eq([evt1, evt2]) + describe '.handled_messages_for_evolve' do + it 'tracks registered classes' do + expect(evolver_class.handled_messages_for_evolve).to contain_exactly( + EvolveTestMessages::ItemAdded, + EvolveTestMessages::ItemRemoved + ) + end end - specify '.before_evolve' do - evt1 = EvolveTest::Reactor::Event1.new(stream_id: '1', seq: 1) - evt2 = EvolveTest::Reactor::Event2.new(stream_id: '1', seq: 2) - # evt3 is not handled by the reactor - evt3 = EvolveTest::Reactor::Event3.new(stream_id: '1', seq: 3) - new_state = EvolveTest::WithBefore.new.evolve([evt1, evt2, evt3]) - expect(new_state).to eq([1, evt1, 2, evt2]) + describe 'inheritance' do + it 'subclass inherits evolve handlers' do + subclass = Class.new(evolver_class) + expect(subclass.handled_messages_for_evolve).to contain_exactly( + EvolveTestMessages::ItemAdded, + EvolveTestMessages::ItemRemoved + ) + + instance = subclass.new + instance.evolve([ + EvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }) + ]) + expect(instance.state[:items]).to eq([{ id: 'i1', name: 'Apple' }]) + end end end diff --git a/spec/sourced/ccc/handle_spec.rb b/spec/handle_spec.rb similarity index 59% rename from spec/sourced/ccc/handle_spec.rb rename to spec/handle_spec.rb index b20529d3..5aa8c8eb 100644 --- a/spec/sourced/ccc/handle_spec.rb +++ b/spec/handle_spec.rb @@ -1,82 +1,82 @@ # frozen_string_literal: true require 'spec_helper' -require 'sourced/ccc' +require 'sourced' require 'sequel' -module CCCHandleTestMessages - CreateDevice = Sourced::CCC::Command.define('handle_test.create_device') do +module HandleTestMessages + CreateDevice = Sourced::Command.define('handle_test.create_device') do attribute :device_id, String attribute :name, Sourced::Types::String.present end - DeviceCreated = Sourced::CCC::Event.define('handle_test.device_created') do + DeviceCreated = Sourced::Event.define('handle_test.device_created') do attribute :device_id, String attribute :name, String end - ActivateDevice = Sourced::CCC::Command.define('handle_test.activate_device') do + ActivateDevice = Sourced::Command.define('handle_test.activate_device') do attribute :device_id, String end - DeviceActivated = Sourced::CCC::Event.define('handle_test.device_activated') do + DeviceActivated = Sourced::Event.define('handle_test.device_activated') do attribute :device_id, String end end -class HandleTestDecider < Sourced::CCC::Decider +class HandleTestDecider < Sourced::Decider partition_by :device_id consumer_group 'handle-test-decider' state { |_| { exists: false, active: false } } - evolve CCCHandleTestMessages::DeviceCreated do |state, _evt| + evolve HandleTestMessages::DeviceCreated do |state, _evt| state[:exists] = true end - evolve CCCHandleTestMessages::DeviceActivated do |state, _evt| + evolve HandleTestMessages::DeviceActivated do |state, _evt| state[:active] = true end - command CCCHandleTestMessages::CreateDevice do |state, cmd| + command HandleTestMessages::CreateDevice do |state, cmd| raise 'Already exists' if state[:exists] - event CCCHandleTestMessages::DeviceCreated, device_id: cmd.payload.device_id, name: cmd.payload.name + event HandleTestMessages::DeviceCreated, device_id: cmd.payload.device_id, name: cmd.payload.name end - command CCCHandleTestMessages::ActivateDevice do |state, cmd| + command HandleTestMessages::ActivateDevice do |state, cmd| raise 'Not found' unless state[:exists] raise 'Already active' if state[:active] - event CCCHandleTestMessages::DeviceActivated, device_id: cmd.payload.device_id + event HandleTestMessages::DeviceActivated, device_id: cmd.payload.device_id end end -RSpec.describe 'Sourced::CCC.handle!' do +RSpec.describe 'Sourced.handle!' do let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } + let(:store) { Sourced::Store.new(db) } before { store.install! } describe 'valid command, no prior history' do it 'returns command, reactor, and events' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 'Sensor' } ) - result = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + result = Sourced.handle!(HandleTestDecider, cmd, store: store) - expect(result).to be_a(Sourced::CCC::HandleResult) + expect(result).to be_a(Sourced::HandleResult) expect(result.command).to eq(cmd) expect(result.reactor).to be_a(HandleTestDecider) expect(result.events.size).to eq(1) - expect(result.events.first).to be_a(CCCHandleTestMessages::DeviceCreated) + expect(result.events.first).to be_a(HandleTestMessages::DeviceCreated) end it 'supports array destructuring' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 'Sensor' } ) - cmd_out, reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + cmd_out, reactor, events = Sourced.handle!(HandleTestDecider, cmd, store: store) expect(cmd_out).to eq(cmd) expect(reactor).to be_a(HandleTestDecider) @@ -84,21 +84,21 @@ class HandleTestDecider < Sourced::CCC::Decider end it 'evolves reactor state' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 'Sensor' } ) - _cmd, reactor, _events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + _cmd, reactor, _events = Sourced.handle!(HandleTestDecider, cmd, store: store) expect(reactor.state[:exists]).to be true end it 'appends command and correlated events to the store' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 'Sensor' } ) - _cmd, _reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + _cmd, _reactor, events = Sourced.handle!(HandleTestDecider, cmd, store: store) # Both command and event should be in the store all = store.db[:sourced_messages].order(:position).all @@ -108,11 +108,11 @@ class HandleTestDecider < Sourced::CCC::Decider end it 'correlates events with the command' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 'Sensor' } ) - _cmd, _reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + _cmd, _reactor, events = Sourced.handle!(HandleTestDecider, cmd, store: store) expect(events.first.causation_id).to eq(cmd.id) expect(events.first.correlation_id).to eq(cmd.correlation_id) @@ -122,22 +122,22 @@ class HandleTestDecider < Sourced::CCC::Decider describe 'valid command with prior history' do before do # Create device first - Sourced::CCC.handle!( + Sourced.handle!( HandleTestDecider, - CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + HandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), store: store ) end it 'loads history and evolves before deciding' do - cmd = CCCHandleTestMessages::ActivateDevice.new( + cmd = HandleTestMessages::ActivateDevice.new( payload: { device_id: 'd1' } ) - _cmd, reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + _cmd, reactor, events = Sourced.handle!(HandleTestDecider, cmd, store: store) expect(events.size).to eq(1) - expect(events.first).to be_a(CCCHandleTestMessages::DeviceActivated) + expect(events.first).to be_a(HandleTestMessages::DeviceActivated) expect(reactor.state[:exists]).to be true expect(reactor.state[:active]).to be true end @@ -145,11 +145,11 @@ class HandleTestDecider < Sourced::CCC::Decider describe 'invalid command' do it 'returns immediately without appending' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 123 } # name should be a present string ) - cmd_out, reactor, events = Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + cmd_out, reactor, events = Sourced.handle!(HandleTestDecider, cmd, store: store) expect(cmd_out.valid?).to be false expect(reactor).to be_a(HandleTestDecider) @@ -162,12 +162,12 @@ class HandleTestDecider < Sourced::CCC::Decider describe 'domain invariant violation' do it 'raises the domain error' do - cmd = CCCHandleTestMessages::ActivateDevice.new( + cmd = HandleTestMessages::ActivateDevice.new( payload: { device_id: 'd1' } ) expect { - Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + Sourced.handle!(HandleTestDecider, cmd, store: store) }.to raise_error(RuntimeError, 'Not found') end end @@ -175,24 +175,24 @@ class HandleTestDecider < Sourced::CCC::Decider describe 'optimistic concurrency' do it 'guard prevents concurrent writes within the same partition' do # Create the device first so there is history (and thus guard conditions) - Sourced::CCC.handle!( + Sourced.handle!( HandleTestDecider, - CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + HandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), store: store ) # Simulate a concurrent write to the same partition between load and append # by directly appending an event after the first handle! store.append( - CCCHandleTestMessages::DeviceActivated.new(payload: { device_id: 'd1' }) + HandleTestMessages::DeviceActivated.new(payload: { device_id: 'd1' }) ) # A second handle! that also needs to write to the same partition # should detect the concurrent write. Since the decider also sees the # activated state, it raises an invariant error first. - cmd = CCCHandleTestMessages::ActivateDevice.new(payload: { device_id: 'd1' }) + cmd = HandleTestMessages::ActivateDevice.new(payload: { device_id: 'd1' }) expect { - Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + Sourced.handle!(HandleTestDecider, cmd, store: store) }.to raise_error(RuntimeError, 'Already active') end @@ -201,35 +201,35 @@ class HandleTestDecider < Sourced::CCC::Decider # guard from load is used when appending allow(store).to receive(:append).and_call_original - Sourced::CCC.handle!( + Sourced.handle!( HandleTestDecider, - CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + HandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), store: store ) expect(store).to have_received(:append).with( anything, - guard: an_instance_of(Sourced::CCC::ConsistencyGuard) + guard: an_instance_of(Sourced::ConsistencyGuard) ) end end describe 'offset advancement for registered reactors' do - let(:router) { Sourced::CCC::Router.new(store: store) } + let(:router) { Sourced::Router.new(store: store) } before do router.register(HandleTestDecider) - allow(Sourced::CCC).to receive(:config).and_return( - instance_double(Sourced::CCC::Configuration, router: router) + allow(Sourced).to receive(:config).and_return( + instance_double(Sourced::Configuration, router: router) ) end it 'advances offsets so background workers skip handled commands' do - cmd = CCCHandleTestMessages::CreateDevice.new( + cmd = HandleTestMessages::CreateDevice.new( payload: { device_id: 'd1', name: 'Sensor' } ) - Sourced::CCC.handle!(HandleTestDecider, cmd, store: store) + Sourced.handle!(HandleTestDecider, cmd, store: store) # Background worker should find no work for this partition handled = router.handle_next_for(HandleTestDecider, worker_id: 'test-worker') @@ -237,15 +237,15 @@ class HandleTestDecider < Sourced::CCC::Decider end it 'advances offsets after multiple commands on same partition' do - Sourced::CCC.handle!( + Sourced.handle!( HandleTestDecider, - CCCHandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), + HandleTestMessages::CreateDevice.new(payload: { device_id: 'd1', name: 'Sensor' }), store: store ) - Sourced::CCC.handle!( + Sourced.handle!( HandleTestDecider, - CCCHandleTestMessages::ActivateDevice.new(payload: { device_id: 'd1' }), + HandleTestMessages::ActivateDevice.new(payload: { device_id: 'd1' }), store: store ) diff --git a/spec/handler_spec.rb b/spec/handler_spec.rb deleted file mode 100644 index 0d5be4c6..00000000 --- a/spec/handler_spec.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -module HandlerTests - class MyHandler - include Sourced::Handler - - Event = Sourced::Message.define('handlertest.event') do - attribute :value - end - - on :start, name: String do |event| - [event.follow(Event, value: event.payload.name)] - end - - on :stop do |event, history:| - [event.follow(Event, value: history.size)] - end - - on :foo, :bar do |event, history:| - [event.follow(Event, value: event.class.name)] - end - end -end - -RSpec.describe Sourced::Handler do - it 'implements the Reactor interface' do - expect(Sourced::ReactorInterface === HandlerTests::MyHandler).to be(true) - end - - specify '.handle' do - msg = HandlerTests::MyHandler::Start.build('aa', name: 'Joe') - result = HandlerTests::MyHandler.handle(msg) - expect(result.first).to be_a(Sourced::Actions::AppendNext) - expect(result.first.messages.first.payload.value).to eq('Joe') - - msg2 = HandlerTests::MyHandler::Stop.build('aa') - result = HandlerTests::MyHandler.handle(msg2, history: [msg2]) - expect(result.first).to be_a(Sourced::Actions::AppendNext) - expect(result.first.messages.first.payload.value).to eq(1) - end - - specify '.on with multiple messages' do - msg = HandlerTests::MyHandler::Foo.build('aa') - result = HandlerTests::MyHandler.handle(msg) - expect(result.first).to be_a(Sourced::Actions::AppendNext) - expect(result.first.messages.first.payload.value).to eq('HandlerTests::MyHandler::Foo') - - msg = HandlerTests::MyHandler::Bar.build('aa') - result = HandlerTests::MyHandler.handle(msg) - expect(result.first).to be_a(Sourced::Actions::AppendNext) - expect(result.first.messages.first.payload.value).to eq('HandlerTests::MyHandler::Bar') - end -end diff --git a/spec/injector_spec.rb b/spec/injector_spec.rb deleted file mode 100644 index b4f09f5d..00000000 --- a/spec/injector_spec.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/injector' - -RSpec.describe Sourced::Injector do - describe '.resolve_args' do - context 'with class method' do - let(:target) do - Class.new do - def self.handle(backend:, logger:, clock: 'default-clock') - [backend, logger, clock] - end - end - end - - it 'returns a list of argument names for an object and method' do - args = described_class.resolve_args(target, :handle) - expect(args).to eq(%i[backend logger clock]) - end - end - - context 'with method having only required keyword arguments' do - let(:target) do - Class.new do - def self.handle(replaying:, history:) - [replaying, history] - end - end - end - - it 'returns only required keyword argument names' do - args = described_class.resolve_args(target, :handle) - expect(args).to eq(%i[replaying history]) - end - end - - context 'with method having only optional keyword arguments' do - let(:target) do - Class.new do - def self.handle(backend: nil, logger: 'default') - [backend, logger] - end - end - end - - it 'returns optional keyword argument names' do - args = described_class.resolve_args(target, :handle) - expect(args).to eq(%i[backend logger]) - end - end - - context 'with method having no keyword arguments' do - let(:target) do - Class.new do - def self.handle(event) - event - end - end - end - - it 'returns empty array' do - args = described_class.resolve_args(target, :handle) - expect(args).to eq([]) - end - end - - context 'with method having mixed positional and keyword arguments' do - let(:target) do - Class.new do - def self.handle(event, stream_id, replaying:, history: nil) - [event, stream_id, replaying, history] - end - end - end - - it 'returns only keyword argument names' do - args = described_class.resolve_args(target, :handle) - expect(args).to eq(%i[replaying history]) - end - end - - context 'with method having keyword splat arguments' do - let(:target) do - Class.new do - def self.handle(event, replaying:, **kwargs) - [event, replaying, kwargs] - end - end - end - - it 'ignores keyword splat and returns explicit keywords' do - args = described_class.resolve_args(target, :handle) - expect(args).to eq(%i[replaying]) - end - end - - context 'with instance method via initialize' do - let(:target) do - Class.new do - def initialize(backend:, logger: nil) - @backend = backend - @logger = logger - end - end - end - - it 'resolves constructor arguments' do - args = described_class.resolve_args(target, :new) - expect(args).to eq(%i[backend logger]) - end - end - - context 'with proc argument' do - let(:proc_target) do - proc { |event, replaying:, history: nil| [event, replaying, history] } - end - - it 'resolves proc parameters' do - args = described_class.resolve_args(proc_target) - expect(args).to eq(%i[replaying history]) - end - end - end -end diff --git a/spec/load_actor_spec.rb b/spec/load_actor_spec.rb deleted file mode 100644 index e5c868d3..00000000 --- a/spec/load_actor_spec.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe 'Sourced.load' do - let(:stream_id) { 'abc' } - - let(:actor_class) do - Class.new(Sourced::Actor) do - state do |id| - { id:, name: nil, age: 0 } - end - - event :start, name: String do |state, event| - state[:name] = event.payload.name - end - - event :update_age, age: Integer do |state, event| - state[:age] = event.payload.age - end - end - end - - before do - Sourced.config.backend.clear! - end - - describe '.load' do - it 'loads history and evolves actor' do - actor = actor_class.new(id: stream_id) - - e1 = actor_class[:start].parse(stream_id:, seq: 1, payload: { name: 'Joe' }) - e2 = actor_class[:update_age].parse(stream_id:, seq: 2, payload: { age: 40 }) - - Sourced.config.backend.append_to_stream(stream_id, [e1, e2]) - actor, events = Sourced.load(actor) - expect(actor.seq).to eq(2) - expect(actor.state[:age]).to eq(40) - expect(events.map(&:id)).to eq([e1.id, e2.id]) - end - - it 'load from class and stream_id' do - e1 = actor_class[:start].parse(stream_id:, seq: 1, payload: { name: 'Joe' }) - e2 = actor_class[:update_age].parse(stream_id:, seq: 2, payload: { age: 40 }) - - Sourced.config.backend.append_to_stream(stream_id, [e1, e2]) - - actor, events = Sourced.load(actor_class, stream_id) - expect(actor).to be_a(actor_class) - expect(actor.seq).to eq(2) - expect(actor.state[:age]).to eq(40) - expect(events.map(&:id)).to eq([e1.id, e2.id]) - end - - it 'catches up to latest history' do - actor = actor_class.new(id: stream_id) - - e1 = actor_class[:start].parse(stream_id:, seq: 1, payload: { name: 'Joe' }) - e2 = actor_class[:update_age].parse(stream_id:, seq: 2, payload: { age: 40 }) - - Sourced.config.backend.append_to_stream(stream_id, [e1]) - actor, events = Sourced.load(actor) - expect(actor.seq).to eq(1) - expect(events.map(&:id)).to eq([e1.id]) - - Sourced.config.backend.append_to_stream(stream_id, [e2]) - actor, events = Sourced.load(actor) - expect(actor.seq).to eq(2) - expect(events.map(&:id)).to eq([e2.id]) - end - - it 'loads events up to a given sequence' do - actor = actor_class.new(id: stream_id) - - e1 = actor_class[:start].parse(stream_id:, seq: 1, payload: { name: 'Joe' }) - e2 = actor_class[:update_age].parse(stream_id:, seq: 2, payload: { age: 40 }) - - Sourced.config.backend.append_to_stream(stream_id, [e1, e2]) - - actor, events = Sourced.load(actor, upto: 1) - expect(actor.seq).to eq(1) - expect(events.map(&:id)).to eq([e1.id]) - end - end - - describe '.history_for' do - it 'loads events for an #id interface' do - actor = actor_class.new(id: stream_id) - - e1 = actor_class[:start].parse(stream_id:, seq: 1, payload: { name: 'Joe' }) - e2 = actor_class[:update_age].parse(stream_id:, seq: 2, payload: { age: 40 }) - - Sourced.config.backend.append_to_stream(stream_id, [e1, e2]) - - history = Sourced.history_for(actor) - expect(history.map(&:class)).to eq([actor_class[:start], actor_class[:update_age]]) - expect(history.map(&:id)).to eq([e1.id, e2.id]) - - history = Sourced.history_for(actor, upto: 1) - expect(history.map(&:class)).to eq([actor_class[:start]]) - expect(history.map(&:id)).to eq([e1.id]) - end - end -end diff --git a/spec/sourced/ccc/load_spec.rb b/spec/load_spec.rb similarity index 73% rename from spec/sourced/ccc/load_spec.rb rename to spec/load_spec.rb index d0806635..6bd1302c 100644 --- a/spec/sourced/ccc/load_spec.rb +++ b/spec/load_spec.rb @@ -1,43 +1,43 @@ # frozen_string_literal: true require 'spec_helper' -require 'sourced/ccc' +require 'sourced' require 'sequel' -module CCCLoadTestMessages - CourseCreated = Sourced::CCC::Message.define('load_test.course.created') do +module LoadTestMessages + CourseCreated = Sourced::Message.define('load_test.course.created') do attribute :course_id, String attribute :title, String end - StudentEnrolled = Sourced::CCC::Message.define('load_test.student.enrolled') do + StudentEnrolled = Sourced::Message.define('load_test.student.enrolled') do attribute :course_id, String attribute :student_id, String end - AssignmentSubmitted = Sourced::CCC::Message.define('load_test.assignment.submitted') do + AssignmentSubmitted = Sourced::Message.define('load_test.assignment.submitted') do attribute :course_id, String attribute :student_id, String attribute :grade, String end end -class LoadTestDecider < Sourced::CCC::Decider +class LoadTestDecider < Sourced::Decider partition_by :course_id, :student_id consumer_group 'load-test-decider' state { |_| { enrolled: false, grades: [] } } - evolve CCCLoadTestMessages::StudentEnrolled do |state, _evt| + evolve LoadTestMessages::StudentEnrolled do |state, _evt| state[:enrolled] = true end - evolve CCCLoadTestMessages::AssignmentSubmitted do |state, evt| + evolve LoadTestMessages::AssignmentSubmitted do |state, evt| state[:grades] << evt.payload.grade end end -class LoadTestProjector < Sourced::CCC::Projector::StateStored +class LoadTestProjector < Sourced::Projector::StateStored partition_by :course_id consumer_group 'load-test-projector' @@ -45,52 +45,52 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored { course_id: partition_values[:course_id], title: nil, student_count: 0 } end - evolve CCCLoadTestMessages::CourseCreated do |state, evt| + evolve LoadTestMessages::CourseCreated do |state, evt| state[:title] = evt.payload.title end - evolve CCCLoadTestMessages::StudentEnrolled do |state, _evt| + evolve LoadTestMessages::StudentEnrolled do |state, _evt| state[:student_count] += 1 end end -RSpec.describe 'Sourced::CCC.load' do +RSpec.describe 'Sourced.load' do let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } + let(:store) { Sourced::Store.new(db) } before { store.install! } describe 'loading a Decider' do before do store.append([ - CCCLoadTestMessages::StudentEnrolled.new( + LoadTestMessages::StudentEnrolled.new( payload: { course_id: 'algebra', student_id: 'joe' } ), - CCCLoadTestMessages::AssignmentSubmitted.new( + LoadTestMessages::AssignmentSubmitted.new( payload: { course_id: 'algebra', student_id: 'joe', grade: 'A' } ), - CCCLoadTestMessages::AssignmentSubmitted.new( + LoadTestMessages::AssignmentSubmitted.new( payload: { course_id: 'algebra', student_id: 'joe', grade: 'B' } ), # Different partition — should not be loaded - CCCLoadTestMessages::StudentEnrolled.new( + LoadTestMessages::StudentEnrolled.new( payload: { course_id: 'algebra', student_id: 'jane' } ) ]) end it 'returns an evolved instance and a ReadResult' do - instance, read_result = Sourced::CCC.load( + instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) expect(instance).to be_a(LoadTestDecider) - expect(read_result).to be_a(Sourced::CCC::ReadResult) + expect(read_result).to be_a(Sourced::ReadResult) end it 'evolves state from matching messages' do - instance, _read_result = Sourced::CCC.load( + instance, _read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -100,7 +100,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored end it 'sets partition_values on the instance' do - instance, _read_result = Sourced::CCC.load( + instance, _read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -109,7 +109,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored end it 'read_result contains the messages used for evolution' do - _instance, read_result = Sourced::CCC.load( + _instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -120,23 +120,23 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored end it 'read_result contains a guard for subsequent appends' do - _instance, read_result = Sourced::CCC.load( + _instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) - expect(read_result.guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(read_result.guard).to be_a(Sourced::ConsistencyGuard) expect(read_result.guard.last_position).to be > 0 end it 'guard can be used for optimistic concurrency on append' do - instance, read_result = Sourced::CCC.load( + instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) # Append with guard succeeds when no conflicts - new_msg = CCCLoadTestMessages::AssignmentSubmitted.new( + new_msg = LoadTestMessages::AssignmentSubmitted.new( payload: { course_id: 'algebra', student_id: 'joe', grade: 'C' } ) expect { @@ -144,7 +144,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored }.not_to raise_error # Subsequent append with same guard fails (conflict) - another = CCCLoadTestMessages::AssignmentSubmitted.new( + another = LoadTestMessages::AssignmentSubmitted.new( payload: { course_id: 'algebra', student_id: 'joe', grade: 'D' } ) expect { @@ -153,7 +153,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored end it 'excludes messages from other partitions (AND filtering at SQL level)' do - instance, read_result = Sourced::CCC.load( + instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) @@ -174,17 +174,17 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored end it 'guard detects conflicts from concurrent writes to the partition' do - _instance, read_result = Sourced::CCC.load( + _instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'algebra', student_id: 'joe' ) # Concurrent write to the same partition - store.append(CCCLoadTestMessages::AssignmentSubmitted.new( + store.append(LoadTestMessages::AssignmentSubmitted.new( payload: { course_id: 'algebra', student_id: 'joe', grade: 'X' } )) - new_msg = CCCLoadTestMessages::AssignmentSubmitted.new( + new_msg = LoadTestMessages::AssignmentSubmitted.new( payload: { course_id: 'algebra', student_id: 'joe', grade: 'C' } ) expect { @@ -196,24 +196,24 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored describe 'loading a Projector' do before do store.append([ - CCCLoadTestMessages::CourseCreated.new( + LoadTestMessages::CourseCreated.new( payload: { course_id: 'algebra', title: 'Algebra 101' } ), - CCCLoadTestMessages::StudentEnrolled.new( + LoadTestMessages::StudentEnrolled.new( payload: { course_id: 'algebra', student_id: 'joe' } ), - CCCLoadTestMessages::StudentEnrolled.new( + LoadTestMessages::StudentEnrolled.new( payload: { course_id: 'algebra', student_id: 'jane' } ), # Different course — should not be loaded - CCCLoadTestMessages::CourseCreated.new( + LoadTestMessages::CourseCreated.new( payload: { course_id: 'physics', title: 'Physics 201' } ) ]) end it 'evolves projector state from matching messages' do - instance, _read_result = Sourced::CCC.load( + instance, _read_result = Sourced.load( LoadTestProjector, store: store, course_id: 'algebra' ) @@ -223,7 +223,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored end it 'passes partition values to state initializer' do - instance, _read_result = Sourced::CCC.load( + instance, _read_result = Sourced.load( LoadTestProjector, store: store, course_id: 'algebra' ) @@ -234,7 +234,7 @@ class LoadTestProjector < Sourced::CCC::Projector::StateStored describe 'empty history' do it 'returns instance with initial state when no matching messages' do - instance, read_result = Sourced::CCC.load( + instance, read_result = Sourced.load( LoadTestDecider, store: store, course_id: 'nonexistent', student_id: 'nobody' ) diff --git a/spec/message_spec.rb b/spec/message_spec.rb index 1af3ae97..f20be66a 100644 --- a/spec/message_spec.rb +++ b/spec/message_spec.rb @@ -1,172 +1,477 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' module TestMessages - Command = Class.new(Sourced::Message) + DeviceRegistered = Sourced::Message.define('device.registered') do + attribute :device_id, String + attribute :name, String + end + + AssetRegistered = Sourced::Message.define('asset.registered') do + attribute :asset_id, String + attribute :label, String + end - Add = Command.define('test.add') do - attribute :value, Integer + SystemUpdated = Sourced::Message.define('system.updated') do + attribute :version, String end - Added = Sourced::Message.define('test.added') do - attribute :value, Integer + OptionalFields = Sourced::Message.define('test.optional_fields') do + attribute? :required_field, String + attribute? :optional_field, String end end RSpec.describe Sourced::Message do - it 'has Payload as standalone schemas' do - payload = TestMessages::Add::Payload.new(value: 'aaa') - expect(payload.valid?).to be false - expect(payload.errors[:value]).not_to be(nil) + describe '.define' do + it 'creates a subclass with a type string' do + expect(TestMessages::DeviceRegistered.type).to eq('device.registered') + end + + it 'creates a typed payload' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.payload.device_id).to eq('dev-1') + expect(msg.payload.name).to eq('Sensor A') + end + + it 'auto-generates an id' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.id).not_to be_nil + expect(msg.id).to match(/\A[0-9a-f-]{36}\z/) + end + + it 'sets created_at automatically' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.created_at).to be_a(Time) + end + + it 'sets type on the instance' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.type).to eq('device.registered') + end + + it 'defaults metadata to empty hash' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.metadata).to eq({}) + end + + it 'accepts metadata' do + msg = TestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + expect(msg.metadata[:user_id]).to eq(42) + end + + it 'defaults causation_id and correlation_id to id' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.causation_id).to eq(msg.id) + expect(msg.correlation_id).to eq(msg.id) + end + + it 'accepts explicit causation_id and correlation_id' do + msg = TestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + causation_id: 'cause-1', + correlation_id: 'corr-1' + ) + expect(msg.causation_id).to eq('cause-1') + expect(msg.correlation_id).to eq('corr-1') + end + end + + describe '.from' do + it 'instantiates the correct subclass from a hash' do + msg = Sourced::Message.from(type: 'device.registered', payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg).to be_a(TestMessages::DeviceRegistered) + expect(msg.payload.device_id).to eq('dev-1') + end + + it 'raises UnknownMessageError for unknown types' do + expect { + Sourced::Message.from(type: 'unknown.type', payload: {}) + }.to raise_error(Sourced::UnknownMessageError, /Unknown message type: unknown.type/) + end end - it 'requires a stream_id' do - msg = TestMessages::Add.new(payload: { value: 1 }) - expect(msg.valid?).to be false - expect(msg.errors[:stream_id]).not_to be(nil) + describe '.registry' do + it 'stores defined message types' do + expect(Sourced::Message.registry['device.registered']).to eq(TestMessages::DeviceRegistered) + expect(Sourced::Message.registry['asset.registered']).to eq(TestMessages::AssetRegistered) + end - msg = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - expect(msg.valid?).to be true end - it 'validates payload' do - msg = TestMessages::Add.new(stream_id: '123', payload: { value: 'aaa' }) - expect(msg.valid?).to be false - expect(msg.errors[:payload][:value]).not_to be(nil) + describe '#extracted_keys' do + it 'extracts all top-level payload attributes as string pairs' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + keys = msg.extracted_keys + expect(keys).to contain_exactly( + ['device_id', 'dev-1'], + ['name', 'Sensor A'] + ) + end + + it 'skips nil values' do + msg = TestMessages::OptionalFields.new(payload: { required_field: 'present', optional_field: nil }) + keys = msg.extracted_keys + expect(keys).to eq([['required_field', 'present']]) + end + + it 'converts values to strings' do + msg = TestMessages::SystemUpdated.new(payload: { version: 'v2.0.5' }) + keys = msg.extracted_keys + expect(keys).to eq([['version', 'v2.0.5']]) + end + + it 'returns empty array for messages without payload attributes' do + # Message base class with no payload definition + bare = Sourced::Message.define('test.bare') + msg = bare.new + expect(msg.extracted_keys).to eq([]) + end end - it 'defines Payload#fetch and Payload#[]' do - msg = TestMessages::Add.new(stream_id: '123', payload: { value: 'aaa' }) - expect(msg.payload[:value]).to eq('aaa') - expect(msg.payload.fetch(:value)).to eq('aaa') + describe '.payload_attribute_names' do + it 'returns attribute names for a defined message class' do + expect(TestMessages::DeviceRegistered.payload_attribute_names).to eq([:device_id, :name]) + end - msg = TestMessages::Add.new(stream_id: '123') - expect(msg.payload[:value]).to be(nil) - expect(msg.payload.fetch(:value)).to be(nil) - expect do - msg.payload.fetch(:nope) - end.to raise_error(KeyError) + it 'returns empty array for a bare message class' do + bare = Sourced::Message.define('test.payload_attrs.bare') + expect(bare.payload_attribute_names).to eq([]) + end end - it 'initializes an empty payload if the class defines one' do - msg = TestMessages::Add.new - expect(msg.payload).not_to be(nil) + describe '.to_conditions' do + it 'returns one condition with only attributes the message class has' do + conditions = TestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', asset_id: 'asset-1') + expect(conditions.size).to eq(1) + expect(conditions.first.message_type).to eq('device.registered') + expect(conditions.first.attrs).to eq({ device_id: 'dev-1' }) + end + + it 'includes all matching attributes in one condition' do + conditions = TestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', name: 'Sensor A') + expect(conditions.size).to eq(1) + expect(conditions.first.attrs).to eq({ device_id: 'dev-1', name: 'Sensor A' }) + end + + it 'returns empty array when no attributes match' do + conditions = TestMessages::DeviceRegistered.to_conditions(course_name: 'Algebra') + expect(conditions).to eq([]) + end end - it 'sets #type' do - msg = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - expect(msg.type).to eq('test.add') + describe '#correlate' do + it 'sets causation_id to source message id' do + source = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + target = TestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) + + correlated = source.correlate(target) + expect(correlated.causation_id).to eq(source.id) + end + + it 'propagates correlation_id from source' do + source = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + target = TestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) + + correlated = source.correlate(target) + expect(correlated.correlation_id).to eq(source.correlation_id) + end + + it 'preserves correlation_id through a chain' do + first = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + second = first.correlate(TestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' })) + third = second.correlate(TestMessages::SystemUpdated.new(payload: { version: 'v1' })) + + expect(third.causation_id).to eq(second.id) + expect(third.correlation_id).to eq(first.id) + end + + it 'merges metadata from both messages' do + source = TestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + target = TestMessages::AssetRegistered.new( + payload: { asset_id: 'asset-1', label: 'Label' }, + metadata: { request_id: 'req-1' } + ) + + correlated = source.correlate(target) + expect(correlated.metadata).to eq({ user_id: 42, request_id: 'req-1' }) + end + + it 'returns a new instance without mutating the original' do + source = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + target = TestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) + + correlated = source.correlate(target) + expect(correlated).not_to equal(target) + expect(correlated).to be_a(TestMessages::AssetRegistered) + expect(target.causation_id).to eq(target.id) # original unchanged + end end - it 'sets #causation_id and #correlation_id' do - msg = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - expect(msg.id).not_to be(nil) - expect(msg.causation_id).to eq(msg.id) - expect(msg.correlation_id).to eq(msg.id) + describe '#with_metadata' do + it 'merges new metadata into existing metadata' do + msg = TestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + updated = msg.with_metadata(request_id: 'req-1') + expect(updated.metadata).to eq({ user_id: 42, request_id: 'req-1' }) + end + + it 'returns self when given empty hash' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + expect(msg.with_metadata({})).to equal(msg) + end + + it 'does not mutate the original message' do + msg = TestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + updated = msg.with_metadata(request_id: 'req-1') + expect(msg.metadata).to eq({ user_id: 42 }) + expect(updated).not_to equal(msg) + end end - describe '.build' do - it 'builds instance with stream_id and payload' do - msg = TestMessages::Add.build('aaa', value: 2) - expect(msg).to be_a(TestMessages::Add) - expect(msg.stream_id).to eq('aaa') - expect(msg.payload.value).to eq(2) + describe '#with_payload' do + it 'merges new attributes into existing payload' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + updated = msg.with_payload(name: 'Sensor B') + expect(updated.payload.device_id).to eq('dev-1') + expect(updated.payload.name).to eq('Sensor B') + end + + it 'preserves id and other attributes' do + msg = TestMessages::DeviceRegistered.new( + payload: { device_id: 'dev-1', name: 'Sensor A' }, + metadata: { user_id: 42 } + ) + updated = msg.with_payload(name: 'Sensor B') + expect(updated.id).to eq(msg.id) + expect(updated.metadata).to eq({ user_id: 42 }) + expect(updated.type).to eq('device.registered') + end + + it 'returns a new instance without mutating the original' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + updated = msg.with_payload(name: 'Sensor B') + expect(updated).not_to equal(msg) + expect(msg.payload.name).to eq('Sensor A') + end + + it 'works with empty hash (returns equivalent copy)' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + updated = msg.with_payload({}) + expect(updated.payload.device_id).to eq('dev-1') + expect(updated.payload.name).to eq('Sensor A') + expect(updated).not_to equal(msg) end end - describe '.from' do - it 'creates a message from a hash' do - msg = Sourced::Message.from(stream_id: '123', type: 'test.add', payload: { value: 1 }) - expect(msg).to be_a(TestMessages::Add) - expect(msg.valid?).to be(true) + describe '#at' do + it 'returns new message with updated created_at' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + future = msg.created_at + 3600 + updated = msg.at(future) + expect(updated.created_at).to eq(future) + expect(updated).not_to equal(msg) + end + + it 'raises PastMessageDateError when given a past time' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + past = msg.created_at - 3600 + expect { msg.at(past) }.to raise_error(Sourced::PastMessageDateError) + end + + it 'does not mutate the original message' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + original_time = msg.created_at + msg.at(msg.created_at + 3600) + expect(msg.created_at).to eq(original_time) end + end - it 'raises a known exception if no type found' do - expect do - Sourced::Message.from(stream_id: '123', type: 'test.unknown', payload: { value: 1 }) - end.to raise_error(Sourced::UnknownMessageError, 'Unknown event type: test.unknown') + describe 'Registry' do + describe '#keys' do + it 'returns array of registered type strings' do + keys = Sourced::Message.registry.keys + expect(keys).to include('device.registered', 'asset.registered', 'system.updated') + end end - it 'scopes message registries by sub-class' do - msg = TestMessages::Command.from(stream_id: '123', type: 'test.add', payload: { value: 1 }) - expect(msg).to be_a(TestMessages::Add) + describe '#all' do + let!(:test_cmd) { Sourced::Command.define('test.reg_all_cmd') { attribute :name, String } } + let!(:test_evt) { Sourced::Event.define('test.reg_all_evt') { attribute :name, String } } + + it 'returns an Enumerator when no block given' do + expect(Sourced::Message.registry.all).to be_a(Enumerator) + end + + it 'includes classes from subclass registries' do + all = Sourced::Message.registry.all.to_a + expect(all).to include(test_cmd) + expect(all).to include(test_evt) + end - expect do - TestMessages::Command.from(stream_id: '123', type: 'test.added', payload: { value: 1 }) - end.to raise_error(Sourced::UnknownMessageError, 'Unknown event type: test.added') + it 'includes classes registered directly on Message' do + all = Sourced::Message.registry.all.to_a + expect(all).to include(TestMessages::DeviceRegistered) + end + + it 'yields each class when block given' do + yielded = [] + Sourced::Message.registry.all { |c| yielded << c } + expect(yielded).to include(test_cmd, test_evt) + end + + it 'scoped to a subclass registry only includes that branch' do + cmd_all = Sourced::Command.registry.all.to_a + expect(cmd_all).to include(test_cmd) + expect(cmd_all).not_to include(test_evt) + end end end - describe '#follow' do - it 'creates a new message with causation_id and correlation_id' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - added = add.follow(TestMessages::Added, value: 2) - expect(added.causation_id).to eq(add.id) - expect(added.correlation_id).to eq(add.id) + describe 'Payload' do + let(:msg) { TestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) } + + describe '#[]' do + it 'returns attribute value by symbol key' do + expect(msg.payload[:device_id]).to eq('dev-1') + expect(msg.payload[:name]).to eq('Sensor A') + end end - it 'copies payload attributes' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - added = add.follow(TestMessages::Added, add.payload) - expect(added.payload.value).to eq(1) + describe '#fetch' do + it 'returns attribute value for existing key' do + expect(msg.payload.fetch(:device_id)).to eq('dev-1') + end + + it 'raises KeyError for missing key' do + expect { msg.payload.fetch(:missing) }.to raise_error(KeyError) + end + + it 'supports default value' do + expect(msg.payload.fetch(:missing, 'default')).to eq('default') + end + + it 'supports block fallback' do + expect(msg.payload.fetch(:missing) { 'from_block' }).to eq('from_block') + end end + end - it 'copies metadata' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }, metadata: { user_id: 10 }) - added = add.follow(TestMessages::Added, add.payload) - expect(added.metadata[:user_id]).to eq(10) + describe '#initialize default payload' do + it 'creates message without explicit payload arg' do + bare = Sourced::Message.define('test.init_default') + msg = bare.new + expect(msg.payload).to be_nil + expect(msg.id).not_to be_nil + expect(msg.type).to eq('test.init_default') end end - describe '#follow_with_seq' do - it 'creates a new message with custom seq, causation_id and correlation_id' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - added = add.follow_with_seq(TestMessages::Added, 2, value: 2) - expect(added.seq).to eq(2) - expect(added.causation_id).to eq(add.id) - expect(added.correlation_id).to eq(add.id) + describe '#to_message' do + it 'returns self — identity implementation of the to_message contract' do + msg = TestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + expect(msg.to_message).to equal(msg) end end - describe '#follow_with_stream_id' do - it 'creates a new message with custom stream_id, causation_id and correlation_id' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - added = add.follow_with_stream_id(TestMessages::Added, 'foo', value: 2) - expect(added.stream_id).to eq('foo') - expect(added.causation_id).to eq(add.id) - expect(added.correlation_id).to eq(add.id) + describe '.===' do + let(:msg) { TestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) } + + it 'matches unwrapped instances like Module#===' do + expect(TestMessages::DeviceRegistered === msg).to be true + expect(TestMessages::AssetRegistered === msg).to be false + end + + it 'matches messages wrapped in a to_message-aware delegator' do + wrapper = Class.new(SimpleDelegator) do + def to_message = __getobj__ + end.new(msg) + + expect(TestMessages::DeviceRegistered === wrapper).to be true + expect(TestMessages::AssetRegistered === wrapper).to be false + end + + it 'makes case/when transparent across wrapped and unwrapped messages' do + classify = ->(m) do + case m + when TestMessages::DeviceRegistered then :device + when TestMessages::AssetRegistered then :asset + else :unknown + end + end + + wrapper = Sourced::PositionedMessage.new(msg, 1) + expect(classify.call(msg)).to eq(:device) + expect(classify.call(wrapper)).to eq(:device) + end + + it 'returns false for arbitrary non-message objects without looping' do + expect(TestMessages::DeviceRegistered === 'string').to be false + expect(TestMessages::DeviceRegistered === 42).to be false + expect(TestMessages::DeviceRegistered === Object.new).to be false end end - describe '#at' do - it 'creates a message with a created_at date in the future' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - delayed = add.at(add.created_at + 10) - expect(delayed.created_at).to eq(add.created_at + 10) + describe Sourced::ConsistencyGuard do + it 'is a Data struct with conditions and last_position' do + conditions = [Sourced::QueryCondition.new(message_type: 'device.registered', attrs: { device_id: 'dev-1' })] + guard = Sourced::ConsistencyGuard.new(conditions: conditions, last_position: 42) + expect(guard.conditions).to eq(conditions) + expect(guard.last_position).to eq(42) + end + end + + describe 'Command and Event subclass registries' do + let!(:test_cmd) { Sourced::Command.define('test.do_something') { attribute :name, String } } + let!(:test_evt) { Sourced::Event.define('test.something_happened') { attribute :name, String } } + + it 'registers Command types in Command registry' do + expect(Sourced::Command.registry['test.do_something']).to eq(test_cmd) + end + + it 'registers Event types in Event registry' do + expect(Sourced::Event.registry['test.something_happened']).to eq(test_evt) + end + + it 'does not register Command types in Event registry' do + expect(Sourced::Event.registry['test.do_something']).to be_nil end - it 'does not allow setting a date lower than current' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - expect do - add.at(add.created_at - 10) - end.to raise_error(Sourced::PastMessageDateError) + it 'Message.registry can look up types from subclass registries' do + expect(Sourced::Message.registry['test.do_something']).to eq(test_cmd) + expect(Sourced::Message.registry['test.something_happened']).to eq(test_evt) end end - describe '#to' do - it 'creates a message with a new #stream_id' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - add2 = add.to('222') - expect(add.stream_id).to eq('123') - expect(add2.stream_id).to eq('222') + describe Sourced::QueryCondition do + it 'is a Data struct with message_type and attrs hash' do + cond = Sourced::QueryCondition.new( + message_type: 'device.registered', + attrs: { device_id: 'dev-1' } + ) + expect(cond.message_type).to eq('device.registered') + expect(cond.attrs).to eq({ device_id: 'dev-1' }) end - it 'accepts a #stream_id interface' do - add = TestMessages::Add.new(stream_id: '123', payload: { value: 1 }) - streamable = double('Streamable', stream_id: '222') - add2 = add.to(streamable) - expect(add2.stream_id).to eq('222') + it 'supports multiple attrs for compound conditions' do + cond = Sourced::QueryCondition.new( + message_type: 'seat.selected', + attrs: { showing_id: 'show-1', seat_id: 'C7' } + ) + expect(cond.attrs).to eq({ showing_id: 'show-1', seat_id: 'C7' }) end end end diff --git a/spec/notifier_spec.rb b/spec/notifier_spec.rb deleted file mode 100644 index 3919d5c4..00000000 --- a/spec/notifier_spec.rb +++ /dev/null @@ -1,137 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::Dispatcher::NotificationQueuer do - let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } - - let(:msg_type_a) { double('MsgA', type: 'orders.created') } - let(:msg_type_b) { double('MsgB', type: 'orders.shipped') } - let(:consumer_info1) { double('ConsumerInfo1', group_id: 'Reactor1') } - let(:consumer_info2) { double('ConsumerInfo2', group_id: 'Reactor2') } - let(:reactor1) { double('Reactor1', handled_messages: [msg_type_a], consumer_info: consumer_info1) } - let(:reactor2) { double('Reactor2', handled_messages: [msg_type_a, msg_type_b], consumer_info: consumer_info2) } - - subject(:queuer) do - described_class.new( - work_queue: work_queue, - reactors: [reactor1, reactor2] - ) - end - - describe 'messages_appended' do - it 'pushes matching reactors to work_queue' do - queuer.call('messages_appended', 'orders.created') - - popped = [] - popped << work_queue.pop - popped << work_queue.pop - expect(popped).to contain_exactly(reactor1, reactor2) - end - - it 'handles multiple types and deduplicates reactors' do - queuer.call('messages_appended', 'orders.created,orders.shipped') - - popped = [] - popped << work_queue.pop - popped << work_queue.pop - expect(popped).to contain_exactly(reactor1, reactor2) - end - - it 'strips whitespace from type strings' do - queuer.call('messages_appended', ' orders.created ') - - popped = [] - popped << work_queue.pop - popped << work_queue.pop - expect(popped).to contain_exactly(reactor1, reactor2) - end - - it 'ignores unknown types' do - queuer.call('messages_appended', 'unknown.type') - - work_queue.push(:sentinel) - expect(work_queue.pop).to eq(:sentinel) - end - end - - describe 'reactor_resumed' do - it 'pushes reactor matching the group_id' do - queuer.call('reactor_resumed', 'Reactor1') - - expect(work_queue.pop).to eq(reactor1) - end - - it 'ignores unknown group_ids' do - queuer.call('reactor_resumed', 'Unknown') - - work_queue.push(:sentinel) - expect(work_queue.pop).to eq(:sentinel) - end - end - - describe 'unknown events' do - it 'ignores them' do - queuer.call('something_else', 'data') - - work_queue.push(:sentinel) - expect(work_queue.pop).to eq(:sentinel) - end - end -end - -RSpec.describe Sourced::InlineNotifier do - subject(:notifier) { described_class.new } - - describe '#subscribe / #publish' do - it 'delivers events to subscribers' do - received = [] - notifier.subscribe(->(event, value) { received << [event, value] }) - - notifier.publish('test_event', 'test_value') - expect(received).to eq([['test_event', 'test_value']]) - end - - it 'delivers to multiple subscribers' do - received1 = [] - received2 = [] - notifier.subscribe(->(event, value) { received1 << [event, value] }) - notifier.subscribe(->(event, value) { received2 << [event, value] }) - - notifier.publish('evt', 'val') - expect(received1).to eq([['evt', 'val']]) - expect(received2).to eq([['evt', 'val']]) - end - - it 'does not raise when no subscribers registered' do - expect { notifier.publish('evt', 'val') }.not_to raise_error - end - end - - describe '#notify_new_messages' do - it 'publishes messages_appended with deduped comma-separated types' do - received = [] - notifier.subscribe(->(event, value) { received << [event, value] }) - - notifier.notify_new_messages(['a', 'b', 'a']) - expect(received).to eq([['messages_appended', 'a,b']]) - end - end - - describe '#notify_reactor_resumed' do - it 'publishes reactor_resumed with group_id' do - received = [] - notifier.subscribe(->(event, value) { received << [event, value] }) - - notifier.notify_reactor_resumed('MyReactor') - expect(received).to eq([['reactor_resumed', 'MyReactor']]) - end - end - - describe '#start / #stop' do - it 'are no-ops' do - expect(notifier.start).to be_nil - expect(notifier.stop).to be_nil - end - end -end diff --git a/spec/projector_spec.rb b/spec/projector_spec.rb index 51d7d7f5..ef85cd6e 100644 --- a/spec/projector_spec.rb +++ b/spec/projector_spec.rb @@ -1,208 +1,439 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' -module ProjectorTest - STORE = {} +module ProjectorTestMessages + ItemAdded = Sourced::Message.define('projector_test.item.added') do + attribute :list_id, String + attribute :name, String + end - State = Struct.new(:id, :total) + ItemArchived = Sourced::Message.define('projector_test.item.archived') do + attribute :list_id, String + attribute :name, String + end - Added = Sourced::Event.define('prtest.added') do - attribute :amount, Integer + NotifyArchive = Sourced::Message.define('projector_test.notify_archive') do + attribute :list_id, String end - Probed = Sourced::Event.define('prtest.probed') + DelayedNotifyArchive = Sourced::Message.define('projector_test.delayed_notify_archive') do + attribute :list_id, String + end +end + +class TestItemProjector < Sourced::Projector::StateStored + partition_by :list_id + consumer_group 'item-projector-test' - NextCommand = Sourced::Command.define('prtest.next_command') do - attribute :amount, Integer + state do |(list_id)| + { list_id: list_id, items: [], synced: false } end - NextCommand2 = Sourced::Command.define('prtest.next_command2') do - attribute :amount, Integer + evolve ProjectorTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name end - class StateStored < Sourced::Projector::StateStored - state do |id| - STORE[id] || State.new(id, 0) - end + evolve ProjectorTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end - event Added do |state, event| - state.total += event.payload.amount - end + reaction ProjectorTestMessages::ItemArchived do |_state, msg| + ProjectorTestMessages::NotifyArchive.new(payload: { list_id: msg.payload.list_id }) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + state[:last_replaying] = replaying + end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end +end + +class TestItemESProjector < Sourced::Projector::EventSourced + partition_by :list_id + consumer_group 'item-es-projector-test' + + state do |(list_id)| + { list_id: list_id, items: [], synced: false } + end + + evolve ProjectorTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end - sync do |state:, events:, replaying:| - STORE[state.id] = state + evolve ProjectorTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + reaction ProjectorTestMessages::ItemArchived do |_state, msg| + ProjectorTestMessages::NotifyArchive.new(payload: { list_id: msg.payload.list_id }) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + state[:last_replaying] = replaying + end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end +end + +class TestDelayedItemProjector < Sourced::Projector::StateStored + partition_by :list_id + consumer_group 'delayed-item-projector-test' + + state do |(list_id)| + { list_id: list_id, items: [] } + end + + evolve ProjectorTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + reaction ProjectorTestMessages::ItemArchived do |_state, msg| + dispatch(ProjectorTestMessages::DelayedNotifyArchive, list_id: msg.payload.list_id) + .at(Time.now + 10) + end +end + +RSpec.describe Sourced::Projector do + describe '.handled_messages' do + it 'includes evolve and react types' do + msgs = TestItemProjector.handled_messages + expect(msgs).to include(ProjectorTestMessages::ItemAdded) + expect(msgs).to include(ProjectorTestMessages::ItemArchived) end end - class EventSourced < Sourced::Projector::EventSourced - state do |id| - State.new(id, 0) + describe '.handle_batch (StateStored)' do + it 'evolves from new_messages and includes sync and after_sync actions' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ) + ] + + pairs = TestItemProjector.handle_batch(['L1'], msgs) + + sync_pair = pairs.last + sync_actions, source_msg = sync_pair + expect(source_msg).to eq(msgs.last) + + sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::Actions::Sync) } + expect(sync_action).not_to be_nil + + after_sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::Actions::AfterSync) } + expect(after_sync_action).not_to be_nil end - event Added do |state, event| - state.total += event.payload.amount + it 'runs reactions when not replaying' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + + pairs = TestItemProjector.handle_batch(['L1'], msgs) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + expect(append_actions.size).to eq(1) + expect(append_actions.first.messages.first).to be_a(ProjectorTestMessages::NotifyArchive) end - sync do |state:, events:, replaying:| - STORE[state.id] = [state, events.last.type] + it 'skips reactions when replaying' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + + pairs = TestItemProjector.handle_batch(['L1'], msgs, replaying: true) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + expect(append_actions).to be_empty end end - class StateStoredWithReactions < Sourced::Projector::StateStored - state do |id| - STORE[id] || State.new(id, 0) + describe '.handle_batch (EventSourced)' do + let(:guard) { Sourced::ConsistencyGuard.new(conditions: [], last_position: 5) } + + def make_history(messages) + Sourced::ReadResult.new(messages: messages, guard: guard) end - event Added do |state, event| - state.total += event.payload.amount + it 'evolves from full history, not just new messages' do + history_msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 3 + ) + ] + new_msgs = [history_msgs.last] + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(['L1'], new_msgs, history: history) + + sync_pair = pairs.last + _sync_actions, source_msg = sync_pair + expect(source_msg).to eq(new_msgs.last) end - event Probed # register so that it's handled by .reaction + it 'runs reactions only on new messages, not full history' do + history_msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Old' }), 1 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'New' }), 2 + ) + ] + new_msgs = [history_msgs.last] + history = make_history(history_msgs) - # React to a specific event - reaction Added do |state, event| - if state.total > 20 - dispatch(NextCommand, amount: state.total).to(event) - end - end + pairs = TestItemESProjector.handle_batch(['L1'], new_msgs, history: history) - # React to any event - reaction do |state, event| - if state.total > 10 - dispatch(NextCommand2, amount: state.total) - end + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + expect(append_actions.size).to eq(1) end - sync do |state:, events:, replaying:| - STORE[state.id] = state + it 'skips reactions when replaying' do + history_msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_batch(['L1'], history_msgs, history: history, replaying: true) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + expect(append_actions).to be_empty end end -end -RSpec.describe Sourced::Projector do - before do - ProjectorTest::STORE.clear - end + describe '.handle_claim' do + let(:guard) { Sourced::ConsistencyGuard.new(conditions: [], last_position: 2) } + + def make_claim(messages, replaying: false) + Sourced::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'list_id:L1', + partition_value: { 'list_id' => 'L1' }, + messages: messages, replaying: replaying, guard: guard + ) + end - describe Sourced::Projector::StateStored do - it 'has consumer info' do - expect(ProjectorTest::StateStored.consumer_info.group_id).to eq('ProjectorTest::StateStored') + it 'evolves from claim.messages and includes sync actions' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ) + ] + claim = make_claim(msgs) + + pairs = TestItemProjector.handle_claim(claim) + + # Last pair should contain sync actions + sync_pair = pairs.last + sync_actions, source_msg = sync_pair + expect(source_msg).to eq(msgs.last) + + sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::Actions::Sync) } + expect(sync_action).not_to be_nil end - specify 'with new state' do - e1 = ProjectorTest::Added.parse(stream_id: '111', payload: { amount: 10 }) + it 'runs reactions when not replaying' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: false) - actions = ProjectorTest::StateStored.handle(e1, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) - # Run actions. Normally the backend runs these - run_sync_blocks(actions) + pairs = TestItemProjector.handle_claim(claim) - expect(ProjectorTest::STORE['111'].total).to eq(10) + # Should have reaction pair + sync pair + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + expect(append_actions.size).to eq(1) + expect(append_actions.first.messages.first).to be_a(ProjectorTestMessages::NotifyArchive) end - specify 'with existing state' do - ProjectorTest::STORE['111'] = ProjectorTest::State.new('111', 10) + it 'returns schedule actions for delayed reactions' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: false) - e1 = ProjectorTest::Added.parse(stream_id: '111', payload: { amount: 10 }) + pairs = TestDelayedItemProjector.handle_claim(claim) - actions = ProjectorTest::StateStored.handle(e1, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) - # Run actions. Normally the backend runs these - run_sync_blocks(actions) + schedule_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |action| action.is_a?(Sourced::Actions::Schedule) } - expect(ProjectorTest::STORE['111'].total).to eq(20) + expect(schedule_actions.size).to eq(1) + expect(schedule_actions.first.messages.first).to be_a(ProjectorTestMessages::DelayedNotifyArchive) end - it 'increments @seq' do - e1 = ProjectorTest::Added.parse(stream_id: '222', seq: 1, payload: { amount: 12 }) - e2 = ProjectorTest::Probed.parse(stream_id: '222', seq: 2) - projector = ProjectorTest::StateStored.new(id: '222') - projector.evolve([e1, e2]) - expect(projector.seq).to eq(2) - end - end + it 'skips reactions when replaying' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: true) - describe 'Sourced::Projector::StateStored with reactions' do - it 'reacts to events based on projected state, and returns commands' do - e1 = ProjectorTest::Added.parse(stream_id: '222', payload: { amount: 10 }) - e2 = ProjectorTest::Added.parse(stream_id: '222', payload: { amount: 5 }) - e3 = ProjectorTest::Added.parse(stream_id: '222', payload: { amount: 6 }) + pairs = TestItemProjector.handle_claim(claim) - actions = ProjectorTest::StateStoredWithReactions.handle(e1, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) - run_sync_blocks(actions) + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } - actions = ProjectorTest::StateStoredWithReactions.handle(e1, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) - run_sync_blocks(actions) + expect(append_actions).to be_empty + end - actions = ProjectorTest::StateStoredWithReactions.handle(e2, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync, Sourced::Actions::AppendNext]) - run_sync_blocks(actions) + it 'passes replaying to sync blocks' do + msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(msgs, replaying: true) - actions = ProjectorTest::StateStoredWithReactions.handle(e3, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync, Sourced::Actions::AppendNext]) - run_sync_blocks(actions) - expect(ProjectorTest::STORE['222'].total).to eq(31) - expect(actions.last.messages.map(&:class)).to eq([ProjectorTest::NextCommand]) - expect(actions.last.messages.map(&:stream_id)).to eq(['222']) - expect(actions.last.messages.first.payload.amount).to eq(31) - end + pairs = TestItemProjector.handle_claim(claim) - it 'reacts to wildcard events, if it evolves from them' do - e1 = ProjectorTest::Added.parse(stream_id: '222', payload: { amount: 12 }) - e2 = ProjectorTest::Probed.parse(stream_id: '222') + # Execute the sync action to verify replaying is passed through + sync_pair = pairs.last + sync_actions = Array(sync_pair.first).select { |a| a.is_a?(Sourced::Actions::Sync) } + expect(sync_actions).not_to be_empty - actions = ProjectorTest::StateStoredWithReactions.handle(e1, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) - run_sync_blocks(actions) - actions = ProjectorTest::StateStoredWithReactions.handle(e2, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync, Sourced::Actions::AppendNext]) - expect(actions.last.messages.map(&:class)).to eq([ProjectorTest::NextCommand2]) + # Call the sync to verify it runs + sync_actions.first.call end + end - it 'does not react if replaying' do - e1 = ProjectorTest::Added.parse(stream_id: '222', payload: { amount: 12 }) - e2 = ProjectorTest::Probed.parse(stream_id: '222') + describe 'EventSourced' do + let(:guard) { Sourced::ConsistencyGuard.new(conditions: [], last_position: 5) } - actions = ProjectorTest::StateStoredWithReactions.handle(e1, replaying: false) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) - run_sync_blocks(actions) - actions = ProjectorTest::StateStoredWithReactions.handle(e2, replaying: true) - expect(actions.map(&:class)).to eq([Sourced::Actions::Sync]) + def make_claim(messages, replaying: false) + Sourced::ClaimResult.new( + offset_id: 1, key_pair_ids: [], partition_key: 'list_id:L1', + partition_value: { 'list_id' => 'L1' }, + messages: messages, replaying: replaying, guard: guard + ) end - it 'rejects reactions to events not handled by .event handlers' do - expect { - Class.new(Sourced::Projector::StateStored) do - reaction ProjectorTest::Added do |_state, _event| - end - end - }.to raise_error(ArgumentError) + def make_history(messages) + Sourced::ReadResult.new(messages: messages, guard: guard) + end + + it 'evolves from full history, not just claim messages' do + history_msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 3 + ) + ] + # Claim only contains the latest message + claim_msgs = [history_msgs.last] + claim = make_claim(claim_msgs) + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_claim(claim, history: history) + + # Sync pair should be the last one, acked against claim's last message + sync_pair = pairs.last + _sync_actions, source_msg = sync_pair + expect(source_msg).to eq(claim_msgs.last) end - end - describe Sourced::Projector::EventSourced do - it 'has consumer info' do - expect(ProjectorTest::EventSourced.consumer_info.group_id).to eq('ProjectorTest::EventSourced') + it 'runs reactions only on claim messages, not full history' do + history_msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Old' }), 1 + ), + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'New' }), 2 + ) + ] + # Only the second message is in the claim + claim_msgs = [history_msgs.last] + claim = make_claim(claim_msgs, replaying: false) + history = make_history(history_msgs) + + pairs = TestItemESProjector.handle_claim(claim, history: history) + + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + # Only 1 reaction (for the claim message), not 2 + expect(append_actions.size).to eq(1) + expect(append_actions.first.messages.first).to be_a(ProjectorTestMessages::NotifyArchive) end - specify 'it builds state from history, returns sync action to persist it' do - e1 = ProjectorTest::Added.parse(stream_id: '111', payload: { amount: 10 }) - e2 = ProjectorTest::Added.parse(stream_id: '111', payload: { amount: 5 }) + it 'skips reactions when replaying' do + history_msgs = [ + Sourced::PositionedMessage.new( + ProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 + ) + ] + claim = make_claim(history_msgs, replaying: true) + history = make_history(history_msgs) - # In Sourced's arch, the new message is included in the history - actions = ProjectorTest::EventSourced.handle(e2, replaying: false, history: [e1, e2]) - run_sync_blocks(actions) + pairs = TestItemESProjector.handle_claim(claim, history: history) - obj, last_event_type = ProjectorTest::STORE['111'] - expect(obj.total).to eq(15) - expect(last_event_type).to eq('prtest.added') + append_actions = pairs.flat_map { |actions, _| Array(actions) } + .select { |a| a.is_a?(Sourced::Actions::Append) } + + expect(append_actions).to be_empty + end + + it 'is detected by Injector as needing history' do + needs = Sourced::Injector.resolve_args(TestItemESProjector, :handle_claim) + expect(needs).to include(:history) + end + + it 'StateStored is not detected as needing history' do + needs = Sourced::Injector.resolve_args(TestItemProjector, :handle_claim) + expect(needs).not_to include(:history) end end - private def run_sync_blocks(actions) - actions.filter{ |a| a.is_a?(Sourced::Actions::Sync) }.each(&:call) + describe '.context_for' do + it 'builds conditions from partition_keys × handled_messages_for_evolve' do + conditions = TestItemProjector.context_for(list_id: 'L1') + types = conditions.map(&:message_type).sort + expect(types).to include('projector_test.item.added') + expect(types).to include('projector_test.item.archived') + end end end diff --git a/spec/pubsub/pg_spec.rb b/spec/pubsub/pg_spec.rb deleted file mode 100644 index 01037feb..00000000 --- a/spec/pubsub/pg_spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/pubsub/pg' - -RSpec.describe Sourced::PubSub::PG, type: :backend do - before(:all) do - @db = Sequel.postgres('sourced_test') - Sequel.extension :fiber_concurrency if Sourced.config.executor.is_a?(Sourced::AsyncExecutor) - end - - after(:all) do - @db&.disconnect - end - - subject(:pubsub) { described_class.new(db: @db, logger: Sourced.config.logger) } - - it 'publishes and subscribes to events' do - received = [] - - Sourced.config.executor.start do |t| - t.spawn do - channel1 = pubsub.subscribe('test_channel') - channel1.start do |event, channel| - received << event - channel.stop if received.size == 2 - end - end - - t.spawn do - sleep 0.0001 - e1 = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - e2 = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - pubsub.publish('test_channel', e1) - pubsub.publish('test_channel', e2) - end - end - - expect(received.map(&:type)).to eq(%w[tests.something_happened1 tests.something_happened1]) - expect(received.map(&:seq)).to eq([1, 2]) - end -end diff --git a/spec/pubsub/test_spec.rb b/spec/pubsub/test_spec.rb deleted file mode 100644 index 42f71e67..00000000 --- a/spec/pubsub/test_spec.rb +++ /dev/null @@ -1,140 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/pubsub/test' - -RSpec.describe Sourced::PubSub::Test, type: :backend do - subject(:pubsub) { described_class.new } - - def make_event(seq:) - BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's1', seq: seq, payload: { account_id: seq }) - end - - it 'publishes and subscribes to events' do - received = [] - - Sourced.config.executor.start do |t| - t.spawn do - channel1 = pubsub.subscribe('test_channel') - channel1.start do |event, channel| - received << event - channel.stop if received.size == 2 - end - end - - t.spawn do - sleep 0.0001 - pubsub.publish('test_channel', make_event(seq: 1)) - pubsub.publish('test_channel', make_event(seq: 2)) - end - end - - expect(received.map(&:type)).to eq(%w[tests.something_happened1 tests.something_happened1]) - expect(received.map(&:seq)).to eq([1, 2]) - end - - it 'multiple subscribers each get their own copy of messages' do - received1 = [] - received2 = [] - - Sourced.config.executor.start do |t| - t.spawn do - ch = pubsub.subscribe('ch') - ch.start do |event, channel| - received1 << event - channel.stop if received1.size == 2 - end - end - - t.spawn do - ch = pubsub.subscribe('ch') - ch.start do |event, channel| - received2 << event - channel.stop if received2.size == 2 - end - end - - t.spawn do - sleep 0.001 - pubsub.publish('ch', make_event(seq: 1)) - pubsub.publish('ch', make_event(seq: 2)) - end - end - - expect(received1.map(&:seq)).to eq([1, 2]) - expect(received2.map(&:seq)).to eq([1, 2]) - end - - it 'a slow subscriber does not block a fast subscriber' do - fast_done = Queue.new - fast_received = [] - slow_received = [] - - Sourced.config.executor.start do |t| - # Fast subscriber - t.spawn do - ch = pubsub.subscribe('ch') - ch.start do |event, channel| - fast_received << event - if fast_received.size == 2 - fast_done << true - channel.stop - end - end - end - - # Slow subscriber — sleeps on each message - t.spawn do - ch = pubsub.subscribe('ch') - ch.start do |event, channel| - sleep 0.05 - slow_received << event - channel.stop if slow_received.size == 2 - end - end - - t.spawn do - sleep 0.001 - pubsub.publish('ch', make_event(seq: 1)) - pubsub.publish('ch', make_event(seq: 2)) - end - end - - # Fast subscriber got all messages - expect(fast_received.map(&:seq)).to eq([1, 2]) - # Slow subscriber also got all messages - expect(slow_received.map(&:seq)).to eq([1, 2]) - end - - it 'subscribers on different channels are independent' do - received_a = [] - received_b = [] - - Sourced.config.executor.start do |t| - t.spawn do - ch = pubsub.subscribe('channel_a') - ch.start do |event, channel| - received_a << event - channel.stop - end - end - - t.spawn do - ch = pubsub.subscribe('channel_b') - ch.start do |event, channel| - received_b << event - channel.stop - end - end - - t.spawn do - sleep 0.001 - pubsub.publish('channel_a', make_event(seq: 1)) - pubsub.publish('channel_b', make_event(seq: 2)) - end - end - - expect(received_a.map(&:seq)).to eq([1]) - expect(received_b.map(&:seq)).to eq([2]) - end -end diff --git a/spec/react_dsl_spec.rb b/spec/react_dsl_spec.rb deleted file mode 100644 index 6a3c6f0a..00000000 --- a/spec/react_dsl_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -module ReactDSLTests - Event = Sourced::Message.define('reactdsl.test') - - class Actor < Sourced::Actor - include Sourced::React - end -end - -RSpec.describe Sourced::React do - it 'raises error when resolving message symbol is not implemented' do - expect { ReactDSLTests::Actor.reaction(nil) }.to raise_error(ArgumentError, /Invalid arguments/) - end - - it 'handles no arguments (wildcard reaction)' do - expect { ReactDSLTests::Actor.reaction {} }.not_to raise_error - end - - it 'handles a single event class' do - expect { ReactDSLTests::Actor.reaction(ReactDSLTests::Event) } - .not_to raise_error - end - - it 'handles an array of events' do - expect { ReactDSLTests::Actor.reaction(ReactDSLTests::Event, ReactDSLTests::Event) } - .not_to raise_error - end - - it 'raises an error for an invalid event' do - expect { ReactDSLTests::Actor.reaction(:non_existent_event) } - .to raise_error(ArgumentError, /Cannot resolve message symbol/) - end -end diff --git a/spec/react_spec.rb b/spec/react_spec.rb index 3090eedb..f8ef0427 100644 --- a/spec/react_spec.rb +++ b/spec/react_spec.rb @@ -1,144 +1,218 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' -class ReactTestReactor - include Sourced::React - - Event1 = Sourced::Message.define('reacttest.event1') - Event2 = Sourced::Message.define('reacttest.event2') - Event3 = Sourced::Message.define('reacttest.event3') - Event4 = Sourced::Message.define('reacttest.event4') - Event5 = Sourced::Message.define('reacttest.event5') - Event6 = Sourced::Message.define('reacttest.event6') - Event7 = Sourced::Message.define('reacttest.event7') - Nope = Sourced::Message.define('reacttest.nope') - - Cmd1 = Sourced::Message.define('reacttest.cmd1') do - attribute :name, String - end - Cmd2 = Sourced::Message.define('reacttest.cmd2') - Cmd3 = Sourced::Message.define('reacttest.cmd3') - NotifyWildcardReaction = Sourced::Message.define('reacttest.NotifyWildcardReaction') do - attribute :state - attribute :event - end - NotifyVariableReaction = Sourced::Message.define('reacttest.NotifyVariableReaction') do - attribute :event +module ReactTestMessages + SomethingHappened = Sourced::Message.define('react_test.something.happened') do + attribute :thing_id, String end - def state = { name: 'test' } - - def self.handled_messages_for_evolve = [Event1, Event4, Event5] + DoNext = Sourced::Message.define('react_test.do_next') do + attribute :thing_id, String + end - reaction Event1 do |state, event| - dispatch(Cmd1, name: state[:name]).to(event) + Unhandled = Sourced::Message.define('react_test.unhandled') do + attribute :foo, String end - reaction Event2 do |_state, event| - dispatch(Cmd2) - dispatch(Cmd3) - .with_metadata(greeting: 'Hi!') - .at(Time.now + 10) + Wildcarded = Sourced::Message.define('react_test.wildcarded') do + attribute :thing_id, String end - reaction Event3 do |_state, _event| - nil + DelayedCommand = Sourced::Message.define('react_test.delayed.command') do + attribute :thing_id, String end - # This wildcard reaction will be registered - # for all events present in .handled_messages_for_evolve - # that do not have custom reactions - reaction do |state, event| - dispatch NotifyWildcardReaction, state:, event: + MultiReaction = Sourced::Message.define('react_test.multi.reaction') do + attribute :thing_id, String end - # This one will register handlers for multiple events - reaction Event6, Event7 do |state, event| - dispatch NotifyVariableReaction, event: + AnotherMultiReaction = Sourced::Message.define('react_test.another.multi.reaction') do + attribute :thing_id, String end end RSpec.describe Sourced::React do - specify '.catch_all_react_events tracks events registered via catch-all reaction' do - expect(ReactTestReactor.catch_all_react_events).to eq(Set[ - ReactTestReactor::Event4, - ReactTestReactor::Event5, - ]) + let(:reactor_class) do + Class.new do + include Sourced::React + extend Sourced::Consumer + + def state + {} + end + + reaction ReactTestMessages::SomethingHappened do |_state, msg| + ReactTestMessages::DoNext.new(payload: { thing_id: msg.payload.thing_id }) + end + end end - specify '.handled_messages_for_react' do - expect(ReactTestReactor.handled_messages_for_react).to eq([ - ReactTestReactor::Event1, - ReactTestReactor::Event2, - ReactTestReactor::Event3, - ReactTestReactor::Event4, - ReactTestReactor::Event5, - ReactTestReactor::Event6, - ReactTestReactor::Event7, - ]) + describe '#react' do + it 'returns raw messages (not correlated)' do + instance = reactor_class.new + msg = ReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + + result = instance.react(msg) + + expect(result.size).to eq(1) + expect(result.first).to be_a(ReactTestMessages::DoNext) + expect(result.first.payload.thing_id).to eq('t1') + # Not correlated — causation_id is its own id + expect(result.first.causation_id).to eq(result.first.id) + end + + it 'accepts a single message or an array of messages' do + instance = reactor_class.new + msg = ReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + + expect(instance.react([msg]).map(&:class)).to eq([ReactTestMessages::DoNext]) + end + + it 'returns empty array for unregistered types' do + instance = reactor_class.new + msg = ReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) + + result = instance.react(msg) + expect(result).to eq([]) + end end - specify '#reacts_to?(message)' do - reactor = ReactTestReactor.new - evt1 = ReactTestReactor::Event1.new(stream_id: '1') - evt2 = ReactTestReactor::Nope.new(stream_id: '1') - expect(reactor.reacts_to?(evt1)).to be(true) - expect(reactor.reacts_to?(evt2)).to be(false) + describe '#reacts_to?' do + it 'returns true for registered types' do + instance = reactor_class.new + msg = ReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + expect(instance.reacts_to?(msg)).to be true + end + + it 'returns false for unregistered types' do + instance = reactor_class.new + msg = ReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) + expect(instance.reacts_to?(msg)).to be false + end end - describe '#react' do - it 'returns messages to append or schedule' do + describe '.handled_messages_for_react' do + it 'tracks registered classes' do + expect(reactor_class.handled_messages_for_react).to contain_exactly( + ReactTestMessages::SomethingHappened + ) + end + end + + describe 'inheritance' do + it 'subclass inherits reaction handlers' do + subclass = Class.new(reactor_class) + expect(subclass.handled_messages_for_react).to contain_exactly( + ReactTestMessages::SomethingHappened + ) + + instance = subclass.new + msg = ReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + result = instance.react(msg) + expect(result.size).to eq(1) + end + end + + describe 'dispatch DSL' do + let(:dsl_reactor_class) do + Class.new do + include Sourced::React + extend Sourced::Consumer + + consumer_group 'ccc-reactor' + + def state + { source: 'state' } + end + + def self.handled_messages_for_evolve + [ReactTestMessages::Wildcarded] + end + + reaction ReactTestMessages::SomethingHappened do |_state, msg| + dispatch(ReactTestMessages::DoNext, thing_id: msg.payload.thing_id) + end + + reaction :react_test_delayed_command do |_state, msg| + dispatch(:react_test_delayed_command, thing_id: msg.payload.thing_id) + .with_metadata(foo: 'bar') + .at(Time.now + 10) + end + + reaction ReactTestMessages::MultiReaction, ReactTestMessages::AnotherMultiReaction do |_state, msg| + dispatch(ReactTestMessages::DoNext, thing_id: msg.payload.thing_id) + end + + reaction do |state, msg| + dispatch(ReactTestMessages::DoNext, thing_id: "#{state[:source]}-#{msg.payload.thing_id}") + end + end + end + + it 'supports dispatch with correlation, producer metadata, metadata chaining, and delays' do now = Time.now Timecop.freeze(now) do - evt1 = ReactTestReactor::Event1.new(stream_id: '1', seq: 1) - evt2 = ReactTestReactor::Event2.new(stream_id: '1', seq: 2) - commands = ReactTestReactor.new.react([evt1, evt2]) - expect(commands.map(&:class)).to eq([ - ReactTestReactor::Cmd1, - ReactTestReactor::Cmd2, - ReactTestReactor::Cmd3 - ]) - expect(commands.map { |e| e.metadata[:producer] }).to eq(%w[ReactTestReactor ReactTestReactor ReactTestReactor]) - expect(commands.first.causation_id).to eq(evt1.id) - expect(commands.first.created_at).to eq(now) - expect(commands.first.payload.name).to eq('test') - expect(commands.last.causation_id).to eq(evt2.id) - expect(commands.last.metadata[:greeting]).to eq('Hi!') - expect(commands.last.created_at).to eq(now + 10) + instance = dsl_reactor_class.new + source = ReactTestMessages::DelayedCommand.new(payload: { thing_id: 't1' }) + + result = instance.react(source) + + expect(result.map(&:class)).to eq([ReactTestMessages::DelayedCommand]) + expect(result.first.causation_id).to eq(source.id) + expect(result.first.correlation_id).to eq(source.correlation_id) + expect(result.first.metadata[:producer]).to eq('ccc-reactor') + expect(result.first.metadata[:foo]).to eq('bar') + expect(result.first.created_at).to eq(now + 10) end end - it 'accepts single message' do - evt1 = ReactTestReactor::Event1.new(stream_id: '1') - commands = ReactTestReactor.new.react(evt1) - expect(commands.first).to be_a(ReactTestReactor::Cmd1) - end + it 'supports dispatching multiple messages from one reaction block' do + klass = Class.new do + include Sourced::React + extend Sourced::Consumer - it 'returns an empty array if the message is not supported' do - evt1 = ReactTestReactor::Nope.new(stream_id: '1') - commands = ReactTestReactor.new.react(evt1) - expect(commands.empty?).to be(true) + def state + {} + end + + reaction ReactTestMessages::SomethingHappened do |_state, msg| + dispatch(ReactTestMessages::DoNext, thing_id: msg.payload.thing_id) + dispatch(ReactTestMessages::DelayedCommand, thing_id: msg.payload.thing_id) + end + end + + result = klass.new.react( + ReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) + ) + + expect(result.map(&:class)).to eq([ + ReactTestMessages::DoNext, + ReactTestMessages::DelayedCommand + ]) end - it 'runs wildcard reactions' do - evt4 = ReactTestReactor::Event4.new(stream_id: '1', seq: 1) - commands = ReactTestReactor.new.react(evt4) - expect(commands.map(&:class)).to eq([ReactTestReactor::NotifyWildcardReaction]) - expect(commands.first.payload.state[:name]).to eq('test') - expect(commands.first.payload.event).to eq(evt4) + it 'supports wildcard reactions for evolve types without explicit handlers' do + result = dsl_reactor_class.new.react( + ReactTestMessages::Wildcarded.new(payload: { thing_id: 't1' }) + ) + + expect(result.map(&:class)).to eq([ReactTestMessages::DoNext]) + expect(result.first.payload.thing_id).to eq('state-t1') + expect(dsl_reactor_class.catch_all_react_events).to eq(Set[ + ReactTestMessages::Wildcarded + ]) end - it 'runs reactions to multiple events' do - evt6 = ReactTestReactor::Event6.new(stream_id: '1', seq: 1) - commands = ReactTestReactor.new.react(evt6) - expect(commands.map(&:class)).to eq([ReactTestReactor::NotifyVariableReaction]) - expect(commands.first.payload.event).to eq(evt6) + it 'supports reactions registered for multiple message classes' do + instance = dsl_reactor_class.new + + first = instance.react(ReactTestMessages::MultiReaction.new(payload: { thing_id: 't1' })) + second = instance.react(ReactTestMessages::AnotherMultiReaction.new(payload: { thing_id: 't2' })) - evt7 = ReactTestReactor::Event7.new(stream_id: '1', seq: 1) - commands = ReactTestReactor.new.react(evt7) - expect(commands.map(&:class)).to eq([ReactTestReactor::NotifyVariableReaction]) - expect(commands.first.payload.event).to eq(evt7) + expect(first.map(&:class)).to eq([ReactTestMessages::DoNext]) + expect(second.map(&:class)).to eq([ReactTestMessages::DoNext]) end end end diff --git a/spec/router_spec.rb b/spec/router_spec.rb index 27a41cfb..71675107 100644 --- a/spec/router_spec.rb +++ b/spec/router_spec.rb @@ -1,440 +1,557 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' +require 'sourced/store' +require 'sequel' + +module RouterTestMessages + DeviceRegistered = Sourced::Message.define('router_test.device.registered') do + attribute :device_id, String + attribute :name, String + end -module RouterTest - AddItem = Sourced::Message.define('routertest.todos.add') - NextCommand = Sourced::Message.define('routertest.todos.next') - ItemAdded = Sourced::Message.define('routertest.todos.added') + DeviceBound = Sourced::Message.define('router_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end - class DeciderOnly - extend Sourced::Consumer + BindDevice = Sourced::Message.define('router_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end - # The Decider interface - def self.handled_commands - [AddItem] - end + NotifyBound = Sourced::Message.define('router_test.notify_bound') do + attribute :device_id, String + end - def self.handle_command(_cmd); end + # Projector messages + DeviceListed = Sourced::Message.define('router_test.device.listed') do + attribute :device_id, String end - class DeciderReactor - extend Sourced::Consumer + # Simple reactor messages + DeviceAudited = Sourced::Message.define('router_test.device.audited') do + attribute :device_id, String + attribute :event_type, String + end +end - def self.handled_messages - [ItemAdded, AddItem, NextCommand] - end +# Test decider for router specs +class RouterTestDecider < Sourced::Decider + partition_by :device_id + consumer_group 'router-test-decider' - def self.handle(evt, replaying:, history:) - cmd = NextCommand.parse(stream_id: evt.stream_id) + state { |_| { exists: false, bound: false } } - Sourced::Actions::AppendNext.new([cmd]) - end - - # Override default handle_batch to forward all kargs - def self.handle_batch(batch, history: []) - batch.map do |message, replaying| - actions = handle(message, replaying:, history:) - [actions, message] - end - end + evolve RouterTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true end - # Test reactors for argument injection - class ReactorWithNoArgs - extend Sourced::Consumer + evolve RouterTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end - def self.handled_messages - [ItemAdded] - end + command RouterTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event RouterTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end - def self.handle(event) - Sourced::Actions::OK - end + reaction RouterTestMessages::DeviceBound do |_state, evt| + RouterTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) end +end - class ReactorWithReplayingOnly - extend Sourced::Consumer +# Test projector for router specs +class RouterTestProjector < Sourced::Projector::StateStored + partition_by :device_id + consumer_group 'router-test-projector' - def self.handled_messages - [ItemAdded] - end + state { |_| { devices: [] } } - def self.handle(event, replaying:) - Sourced::Actions::OK - end + evolve RouterTestMessages::DeviceRegistered do |state, evt| + state[:devices] << evt.payload.name end - # A reactor that needs history — must override handle_batch - class ReactorWithHistoryOnly - extend Sourced::Consumer + evolve RouterTestMessages::DeviceBound do |state, _evt| + # nothing + end - def self.handled_messages - [ItemAdded] - end + sync do |state:, messages:, replaying:| + # In a real projector, this would persist to DB + state[:synced] = true + end +end - def self.handle(event, history:) - Sourced::Actions::OK - end +# Simple reactor: just extends Consumer, defines handled_messages, implements handle_claim. +# Logs an audit trail message for every DeviceRegistered or DeviceBound it sees. +class RouterTestAuditReactor + extend Sourced::Consumer - def self.handle_batch(batch, history: []) - batch.map do |message, replaying| - actions = handle(message, history:) - [actions, message] - end + partition_by :device_id + consumer_group 'router-test-audit' + + def self.handled_messages + [RouterTestMessages::DeviceRegistered, RouterTestMessages::DeviceBound] + end + + def self.handle_claim(claim) + each_with_partial_ack(claim.messages) do |msg| + audit = RouterTestMessages::DeviceAudited.new( + payload: { device_id: msg.payload.device_id, event_type: msg.type } + ) + [Sourced::Actions::Append.new(audit), msg] end end +end + +RSpec.describe Sourced::Router do + let(:db) { Sequel.sqlite } + let(:store) { Sourced::Store.new(db) } + let(:router) { Sourced::Router.new(store: store) } + + before do + store.install! + end - class ReactorWithBothArgs - extend Sourced::Consumer + describe '#register' do + it 'creates consumer group and introspects handle_claim signature' do + router.register(RouterTestDecider) - def self.handled_messages - [ItemAdded] + expect(store.consumer_group_active?('router-test-decider')).to be true + expect(router.reactors).to include(RouterTestDecider) end - def self.handle(event, replaying:, history:) - Sourced::Actions::OK + it 'detects history: for decider, none for projector' do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + + # Decider needs history, projector does not + expect(router.instance_variable_get(:@needs_history)[RouterTestDecider]).to be true + expect(router.instance_variable_get(:@needs_history)[RouterTestProjector]).to be false end - def self.handle_batch(batch, history: []) - batch.map do |message, replaying| - actions = handle(message, replaying:, history:) - [actions, message] - end + it 'passes partition_keys to register_consumer_group' do + router.register(RouterTestDecider) + + row = db[:sourced_consumer_groups].where(group_id: 'router-test-decider').first + expect(JSON.parse(row[:partition_by])).to eq(['device_id']) end end - class ReactorWithLogger - extend Sourced::Consumer + describe '#handle_next_for' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + end + + it 'returns false when no work available' do + result = router.handle_next_for(RouterTestDecider) + expect(result).to be false + end + + it 'claims, calls handle_claim, executes actions + acks in transaction' do + # Set up: register device first (as history), then send bind command + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + result = router.handle_next_for(RouterTestDecider) + expect(result).to be true + + # DeviceBound event should have been appended to store + conds = RouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + read_result = store.read(conds) + expect(read_result.messages.size).to eq(1) + expect(read_result.messages.first).to be_a(RouterTestMessages::DeviceBound) + end + + it 'reads history for decider, skips history for projector' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + + # Projector should process without history + result = router.handle_next_for(RouterTestProjector) + expect(result).to be true + end + + it 'releases on ConcurrentAppendError' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + # Stub store.append to raise ConcurrentAppendError after claim + original_append = store.method(:append) + call_count = 0 + allow(store).to receive(:append) do |*args, **kwargs| + call_count += 1 + if call_count > 0 && kwargs[:guard] + raise Sourced::ConcurrentAppendError, 'conflict' + end + original_append.call(*args, **kwargs) + end - def self.handled_messages - [ItemAdded] - end + result = router.handle_next_for(RouterTestDecider) + expect(result).to be true - def self.handle(event, logger:) - Sourced::Actions::OK + # Offset should be released (not advanced) — can re-claim + claim = store.claim_next( + 'router-test-decider', + partition_by: ['device_id'], + handled_types: RouterTestDecider.handled_messages.map(&:type), + worker_id: 'w-1' + ) + expect(claim).not_to be_nil end - end - class ReactorWithBatchSize - extend Sourced::Consumer + it 'releases on StandardError and calls on_exception' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) - consumer do |c| - c.batch_size = 10 + # Make handle_claim raise + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') + allow(RouterTestDecider).to receive(:on_exception) + + result = router.handle_next_for(RouterTestDecider) + expect(result).to be true + expect(RouterTestDecider).to have_received(:on_exception) end - def self.handled_messages - [ItemAdded] + it 'on_exception fails consumer group when default strategy' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') + + router.handle_next_for(RouterTestDecider) + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:status]).to eq('failed') end - def self.handle(event) - Sourced::Actions::OK + it 'on_exception persists error_context in the database' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') + + router.handle_next_for(RouterTestDecider) + + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:error_context]).not_to be_nil + expect(row[:status]).to eq('failed') end - end - # Reactor that fails on a configurable seq number. - # Uses default Consumer handle_batch (per-message .handle calls). - class ReactorWithPartialFailure - extend Sourced::Consumer + it 'on_exception with retry strategy sets retry_at on consumer group' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) - consumer do |c| - c.batch_size = 10 + retry_strategy = Sourced::ErrorStrategy.new do |s| + s.retry(times: 3, after: 5) + end + allow(Sourced).to receive_message_chain(:config, :error_strategy).and_return(retry_strategy) + + allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') + + router.handle_next_for(RouterTestDecider) + + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:retry_at]).not_to be_nil + expect(row[:status]).to eq('active') + + ctx = JSON.parse(row[:error_context], symbolize_names: true) + expect(ctx[:retry_count]).to eq(2) end + end - def self.handled_messages - [ItemAdded] + describe '#drain' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) end - @log = [] - @fail_on_seq = nil + it 'processes all reactors until no work remains' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + store.append( + RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) - class << self - attr_accessor :log, :fail_on_seq - end + router.drain - def self.handle(event) - raise "boom on seq #{event.seq}" if fail_on_seq == event.seq + # Decider should have produced DeviceBound + conds = RouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + read_result = store.read(conds) + expect(read_result.messages.size).to eq(1) - log << event.seq - Sourced::Actions::OK + # Projector should have processed DeviceRegistered and DeviceBound + # (it handles both via evolve) end end -end -RSpec.describe Sourced::Router do - subject(:router) { described_class.new(backend:) } + describe 'full integration: append commands → Decider → events → Projector' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) + end - let(:backend) { Sourced::Backends::TestBackend.new } + it 'events are correlated with the command that produced them' do + reg = RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) - describe '#drain' do - it 'handles and acknoledges messages for reactors, until there is none left' do - logs = [] - reactor = Class.new do - extend Sourced::Consumer - def self.handled_messages = [RouterTest::ItemAdded] - end - reactor.define_singleton_method(:handle) do |message| - logs << message.type - [] - end + cmd = RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + store.append(cmd) - e1 = RouterTest::ItemAdded.build('123') - e2 = RouterTest::ItemAdded.build('123') - e3 = RouterTest::ItemAdded.build('123') - backend.append_next_to_stream('123', [e1, e2, e3]) - router.register(reactor) router.drain - expect(logs.size).to eq(3) - expect(logs.uniq).to eq(%w[routertest.todos.added]) + + # Read the DeviceBound event + conds = RouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + bound = store.read(conds).messages.find { |m| m.is_a?(RouterTestMessages::DeviceBound) } + + expect(bound).not_to be_nil + expect(bound.causation_id).to eq(cmd.id) + expect(bound.correlation_id).to eq(cmd.correlation_id) end - end - describe '#handle_next_event_for_reactor' do - let(:event) { RouterTest::ItemAdded.new(stream_id: '123') } + it 'reaction messages are correlated with the event reacted to, not the command' do + reg = RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) - before do - router.register(RouterTest::DeciderReactor) + cmd = RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + store.append(cmd) - allow(RouterTest::DeciderReactor).to receive(:on_exception) - backend.append_to_stream('123', event) - end + router.drain - context 'when reactor returns Sourced::Actions::AppendNext' do - it 'appends messages' do - allow(backend).to receive(:append_next_to_stream) - - bool = router.handle_next_event_for_reactor(RouterTest::DeciderReactor) - expect(bool).to be(true) - expect(RouterTest::DeciderReactor).not_to have_received(:on_exception) - expect(backend).to have_received(:append_next_to_stream) do |stream_id, events| - expect(stream_id).to eq('123') - expect(events.size).to eq(1) - event = events.first - expect(event.stream_id).to eq('123') - expect(event).to be_a(RouterTest::NextCommand) - end - end + # Read the DeviceBound event (produced by command handler) + bound_conds = RouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') + bound = store.read(bound_conds).messages.find { |m| m.is_a?(RouterTestMessages::DeviceBound) } + + # Read the NotifyBound reaction message (produced by reaction handler) + notify_conds = RouterTestMessages::NotifyBound.to_conditions(device_id: 'd1') + notify = store.read(notify_conds).messages.find { |m| m.is_a?(RouterTestMessages::NotifyBound) } + + expect(notify).not_to be_nil + # Reaction is correlated with the event, not the command + expect(notify.causation_id).to eq(bound.id) + expect(notify.correlation_id).to eq(bound.correlation_id) + # The whole chain shares the same correlation_id (the command's) + expect(notify.correlation_id).to eq(cmd.correlation_id) end + end - context 'when there are no new messages for reactor' do - it 'return false' do - backend.ack_on(RouterTest::DeciderReactor.consumer_info.group_id, event.id) - bool = router.handle_next_event_for_reactor(RouterTest::DeciderReactor) - expect(bool).to be(false) - end + describe 'consumer group lifecycle' do + before do + router.register(RouterTestDecider) + router.register(RouterTestProjector) end - context 'when reactor raises exception' do - before do - expect(RouterTest::DeciderReactor).to receive(:handle_batch).and_raise('boom') - end + describe '#stop_consumer_group' do + it 'stops group and calls on_stop with class' do + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be true - it 'invokes .on_exception on reactor' do - router.handle_next_event_for_reactor(RouterTest::DeciderReactor) + router.stop_consumer_group(RouterTestDecider, 'maintenance') - expect(RouterTest::DeciderReactor).to have_received(:on_exception) do |exception, message, group| - expect(exception.message).to eq('boom') - expect(message).to eq(event) - expect(group).to respond_to(:stop) - end + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false end - it 'does not acknowledge event for reactor, so that it can be retried' do - router.handle_next_event_for_reactor(RouterTest::DeciderReactor) - groups = backend.stats.groups - expect(groups.first[:stream_count]).to eq(0) - end + it 'stops group and calls on_stop with string group_id' do + router.stop_consumer_group('router-test-decider', 'maintenance') - it 'raises immediatly is passed raise_on_error = true' do - expect { - router.handle_next_event_for_reactor(RouterTest::DeciderReactor, nil, true) - }.to raise_error(RuntimeError, 'boom') + expect(store.consumer_group_active?('router-test-decider')).to be false end - end - context 'handle_batch interface' do - let(:event) { RouterTest::ItemAdded.new(stream_id: '123') } - let(:event2) { RouterTest::ItemAdded.new(stream_id: '123', seq: 2) } + it 'invokes on_stop callback on reactor class' do + allow(RouterTestDecider).to receive(:on_stop) - before do - backend.clear! - backend.append_to_stream('123', [event, event2]) - end + router.stop_consumer_group(RouterTestDecider, 'going down') - context 'with reactor using default handle_batch (no history)' do - before { router.register(RouterTest::ReactorWithNoArgs) } - - it 'calls handle_batch with batch (no history kwarg)' do - received_batch = nil - allow(RouterTest::ReactorWithNoArgs).to receive(:handle_batch).and_wrap_original do |original, batch, **kargs| - received_batch = [batch, kargs] - original.call(batch, **kargs) - end - router.handle_next_event_for_reactor(RouterTest::ReactorWithNoArgs) - expect(received_batch).not_to be_nil - batch, kargs = received_batch - expect(batch.first.first).to eq(event) - expect(kargs).not_to have_key(:history) - end + expect(RouterTestDecider).to have_received(:on_stop).with('going down') end - context 'with reactor needing history (handle_batch accepts history:)' do - before { router.register(RouterTest::ReactorWithHistoryOnly) } - - it 'calls handle_batch with batch and history' do - allow(RouterTest::ReactorWithHistoryOnly).to receive(:handle_batch).and_call_original - router.handle_next_event_for_reactor(RouterTest::ReactorWithHistoryOnly) - expect(RouterTest::ReactorWithHistoryOnly).to have_received(:handle_batch) do |batch, **kargs| - expect(batch.first.first).to eq(event) - expect(kargs[:history]).to be_an(Array) - expect(kargs[:history].map(&:id)).to include(event.id, event2.id) - end - end + it 'passes nil message when none given' do + allow(RouterTestDecider).to receive(:on_stop) + + router.stop_consumer_group(RouterTestDecider) + + expect(RouterTestDecider).to have_received(:on_stop).with(nil) end + end - context 'with reactor needing both replaying and history' do - before { router.register(RouterTest::ReactorWithBothArgs) } + describe '#reset_consumer_group' do + it 'resets group offsets and calls on_reset with class' do + # Append a message and drain so offsets are advanced + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + router.drain - it 'calls handle_batch with batch and history' do - allow(RouterTest::ReactorWithBothArgs).to receive(:handle_batch).and_call_original - router.handle_next_event_for_reactor(RouterTest::ReactorWithBothArgs) - expect(RouterTest::ReactorWithBothArgs).to have_received(:handle_batch) do |batch, **kargs| - expect(batch.first.first).to eq(event) - expect(kargs[:history]).to be_an(Array) - end - end + router.reset_consumer_group(RouterTestDecider) + + # Offsets should be cleared (group can re-process messages) + row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first + expect(row[:discovery_position]).to eq(0) end - context 'Consumer default handle_batch forwards replaying to handle' do - before { router.register(RouterTest::ReactorWithReplayingOnly) } + it 'resets group with string group_id' do + store.append( + RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + ) + router.drain - it 'passes replaying through to handle via default handle_batch' do - # Force handle_kargs caching before spy wraps .handle (spy changes method signature) - RouterTest::ReactorWithReplayingOnly.send(:handle_kargs) - allow(RouterTest::ReactorWithReplayingOnly).to receive(:handle).and_call_original - router.handle_next_event_for_reactor(RouterTest::ReactorWithReplayingOnly) - expect(RouterTest::ReactorWithReplayingOnly).to have_received(:handle).with(event, replaying: false) - end + router.reset_consumer_group('router-test-decider') + + row = db[:sourced_consumer_groups].where(group_id: 'router-test-decider').first + expect(row[:discovery_position]).to eq(0) end - context 'Consumer default handle_batch forwards logger to handle' do - before { router.register(RouterTest::ReactorWithLogger) } + it 'invokes on_reset callback on reactor class' do + allow(RouterTestDecider).to receive(:on_reset) - it 'passes logger through to handle via default handle_batch' do - # Force handle_kargs caching before spy wraps .handle (spy changes method signature) - RouterTest::ReactorWithLogger.send(:handle_kargs) - allow(RouterTest::ReactorWithLogger).to receive(:handle).and_call_original - router.handle_next_event_for_reactor(RouterTest::ReactorWithLogger) - expect(RouterTest::ReactorWithLogger).to have_received(:handle).with(event, logger: router.logger) - end + router.reset_consumer_group(RouterTestDecider) + + expect(RouterTestDecider).to have_received(:on_reset) end + end - context 'per-reactor batch_size override' do - before do - router.register(RouterTest::ReactorWithBatchSize) - end + describe '#start_consumer_group' do + it 'starts group and calls on_start with class' do + router.stop_consumer_group(RouterTestDecider) + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false - it 'uses the reactor consumer_info.batch_size over the worker-level default' do - allow(backend).to receive(:reserve_next_for_reactor).and_call_original - router.handle_next_event_for_reactor(RouterTest::ReactorWithBatchSize, nil, false, batch_size: 1) - expect(backend).to have_received(:reserve_next_for_reactor).with( - RouterTest::ReactorWithBatchSize, - batch_size: 10, - with_history: false, - worker_id: nil - ) - end + router.start_consumer_group(RouterTestDecider) + + expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be true end - context 'partial batch failure' do - let(:e1) { RouterTest::ItemAdded.new(stream_id: '123', seq: 1) } - let(:e2) { RouterTest::ItemAdded.new(stream_id: '123', seq: 2) } - let(:e3) { RouterTest::ItemAdded.new(stream_id: '123', seq: 3) } - - before do - backend.clear! - backend.append_to_stream('123', [e1, e2, e3]) - router.register(RouterTest::ReactorWithPartialFailure) - allow(RouterTest::ReactorWithPartialFailure).to receive(:on_exception) - RouterTest::ReactorWithPartialFailure.fail_on_seq = 3 - RouterTest::ReactorWithPartialFailure.log = [] - end + it 'starts group with string group_id' do + router.stop_consumer_group('router-test-decider') - it 'ACKs successfully processed messages and retries from the failed one' do - # First call: batch of 3, e1 and e2 succeed, e3 raises - router.handle_next_event_for_reactor(RouterTest::ReactorWithPartialFailure) - expect(RouterTest::ReactorWithPartialFailure.log).to eq([1, 2]) + router.start_consumer_group('router-test-decider') - # Second call: should only get e3 (e1 and e2 already ACKed) - RouterTest::ReactorWithPartialFailure.fail_on_seq = nil - RouterTest::ReactorWithPartialFailure.log = [] - router.handle_next_event_for_reactor(RouterTest::ReactorWithPartialFailure) - expect(RouterTest::ReactorWithPartialFailure.log).to eq([3]) - end + expect(store.consumer_group_active?('router-test-decider')).to be true + end - it 'calls on_exception for the failed message' do - allow(RouterTest::ReactorWithPartialFailure).to receive(:on_exception) - router.handle_next_event_for_reactor(RouterTest::ReactorWithPartialFailure) + it 'invokes on_start callback on reactor class' do + allow(RouterTestDecider).to receive(:on_start) - expect(RouterTest::ReactorWithPartialFailure).to have_received(:on_exception) do |exception, message, group| - expect(exception.message).to eq('boom on seq 3') - expect(message.seq).to eq(3) - end - end + router.start_consumer_group(RouterTestDecider) - it 'handles failure on the first message (no partial ACK possible)' do - RouterTest::ReactorWithPartialFailure.fail_on_seq = 1 - router.handle_next_event_for_reactor(RouterTest::ReactorWithPartialFailure) - expect(RouterTest::ReactorWithPartialFailure.log).to eq([]) + expect(RouterTestDecider).to have_received(:on_start) + end + end - # Retry: all 3 messages should still be pending - RouterTest::ReactorWithPartialFailure.fail_on_seq = nil - router.handle_next_event_for_reactor(RouterTest::ReactorWithPartialFailure) - expect(RouterTest::ReactorWithPartialFailure.log).to eq([1, 2, 3]) - end + describe 'resolve_reactor_class' do + it 'raises ArgumentError for unregistered group_id' do + expect { + router.stop_consumer_group('unknown-group') + }.to raise_error(ArgumentError, /No reactor registered with group_id 'unknown-group'/) end end - end - specify 'class-level API' do - expect(router.async_reactors).to eq(Sourced::Router.async_reactors) - expect(Sourced::Router).to respond_to(:register) - expect(Sourced::Router).to respond_to(:registered?) - expect(Sourced::Router).to respond_to(:handle_next_event_for_reactor) - expect(Sourced::Router).to respond_to(:backend) + describe 'no-op default callbacks' do + it 'works fine when reactor does not override callbacks' do + # RouterTestProjector has no custom callbacks — should not raise + expect { router.stop_consumer_group(RouterTestProjector) }.not_to raise_error + expect { router.reset_consumer_group(RouterTestProjector) }.not_to raise_error + expect { router.start_consumer_group(RouterTestProjector) }.not_to raise_error + end + end end - describe '#register' do + describe 'simple Consumer reactor (no Decider/Projector)' do before do - allow(backend).to receive(:register_consumer_group) + router.register(RouterTestAuditReactor) end - it 'registers group id with configured backend' do - router.register(RouterTest::DeciderReactor) - expect(backend).to have_received(:register_consumer_group).with(RouterTest::DeciderReactor.consumer_info.group_id) + it 'registers and processes messages through the router' do + reg = RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + router.drain + + # Audit message should have been appended + conds = RouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + audits = store.read(conds).messages + expect(audits.size).to eq(1) + expect(audits.first).to be_a(RouterTestMessages::DeviceAudited) + expect(audits.first.payload.event_type).to eq('router_test.device.registered') end - it 'determines if reactor needs history from handle_batch signature' do - router.register(RouterTest::DeciderReactor) - expect(router.needs_history[RouterTest::DeciderReactor]).to be(true) + it 'appended messages are correlated with the source message' do + reg = RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) - router.register(RouterTest::ReactorWithNoArgs) - expect(router.needs_history[RouterTest::ReactorWithNoArgs]).to be(false) + router.drain + + conds = RouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + audit = store.read(conds).messages.first + + expect(audit.causation_id).to eq(reg.id) + expect(audit.correlation_id).to eq(reg.correlation_id) end - end - describe '#register' do - it 'registers Reactor interfaces and registers group' do - expect(backend).to receive(:register_consumer_group).with(RouterTest::DeciderReactor.consumer_info.group_id) - router.register(RouterTest::DeciderReactor) - expect(router.async_reactors).to include(RouterTest::DeciderReactor) - expect(router.registered?(RouterTest::DeciderReactor)).to be true + it 'handles multiple messages across partitions' do + store.append(RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor A' })) + store.append(RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor B' })) + + router.drain + + d1_conds = RouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + d2_conds = RouterTestMessages::DeviceAudited.to_conditions(device_id: 'd2') + + expect(store.read(d1_conds).messages.size).to eq(1) + expect(store.read(d2_conds).messages.size).to eq(1) end - it 'raises if registering a non-compliant interface' do - expect do - router.register('nope') - end.to raise_error(Sourced::InvalidReactorError) + it 'context_for returns empty conditions (no evolve types)' do + expect(RouterTestAuditReactor.context_for(device_id: 'd1')).to eq([]) + end + + it 'works alongside deciders and projectors' do + router.register(RouterTestDecider) + + reg = RouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + store.append(reg) + + cmd = RouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + store.append(cmd) + + router.drain + + # Audit reactor sees DeviceRegistered and DeviceBound + conds = RouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') + audits = store.read(conds).messages + types = audits.map { |m| m.payload.event_type }.sort + + expect(types).to eq([ + 'router_test.device.bound', + 'router_test.device.registered' + ]) end end end diff --git a/spec/shared_examples/backend_examples.rb b/spec/shared_examples/backend_examples.rb deleted file mode 100644 index 96e33517..00000000 --- a/spec/shared_examples/backend_examples.rb +++ /dev/null @@ -1,1338 +0,0 @@ -# frozen_string_literal: true - -module BackendExamples - module Tests - DoSomething = Sourced::Message.define('tests.do_something') do - attribute :account_id, Integer - end - SomethingHappened1 = Sourced::Message.define('tests.something_happened1') do - attribute :account_id, Integer - end - SomethingHappened2 = Sourced::Message.define('tests.something_happened2') do - attribute :account_id, Integer - end - end - - RSpec.shared_examples 'an ActiveRecord backend' do |_database_config| - before :all do - described_class.table_prefix = 'sors_ar' - - ActiveRecord::Base.establish_connection( - adapter: 'postgresql', - database: 'sors_test' - ) - - Migrator.new(table_prefix: described_class.table_prefix).up - end - - after :all do - Migrator.new(table_prefix: described_class.table_prefix).down - end - - after do - backend.clear! - end - - it_behaves_like 'a backend' do - specify 'auto-incrementing global_seq' do - cmd1 = BackendExamples::Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt1 = cmd1.follow_with_seq(BackendExamples::Tests::SomethingHappened1, 2, account_id: cmd1.payload.account_id) - evt2 = cmd1.follow_with_seq(BackendExamples::Tests::SomethingHappened1, 3, account_id: cmd1.payload.account_id) - evt3 = BackendExamples::Tests::SomethingHappened1.parse(stream_id: 's1', seq: 4, payload: { account_id: 1 }) - backend.append_to_stream(cmd1.stream_id, [evt1, evt2, evt3]) - expect(Sourced::Backends::ActiveRecordBackend::EventRecord.order(global_seq: :asc).pluck(:global_seq)) - .to eq([1, 2, 3]) - end - end - end - - RSpec.shared_examples 'a backend' do - it 'is installed' do - expect(backend.installed?).to be(true) - end - - it 'supports the Backend interface' do - expect do - Sourced::Configuration::BackendInterface.parse(backend) - end.not_to raise_error - end - - describe '#transaction' do - it 'resets append on error' do - evt = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - expect do - backend.transaction do - backend.append_to_stream('s1', [evt]) - raise 'boom' - end - end.to raise_error('boom') - expect(backend.read_stream('s1').any?).to be(false) - end - end - - describe '#schedule_messages and #update_schedule!' do - it 'schedules messages to be read in the future' do - now = Time.now - - msg0 = Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 0 }) - msg1 = Tests::DoSomething.parse(stream_id: 's1', payload: { account_id: 1 }) - msg2 = Tests::DoSomething.parse(stream_id: 's1', payload: { account_id: 2 }) - msg3 = Tests::DoSomething.parse(stream_id: 's1', payload: { account_id: 3 }) - - backend.append_to_stream('s1', [msg0]) - backend.schedule_messages([msg1, msg2], at: now + 2) - backend.schedule_messages([msg3], at: now + 10) - - backend.update_schedule! - expect(backend.read_stream('s1')).to eq([msg0]) - - Timecop.freeze(now + 3) do - backend.update_schedule! - expect(backend.read_stream('s1').map(&:id)).to eq([msg0, msg1, msg2].map(&:id)) - end - - Timecop.freeze(now + 11) do - backend.update_schedule! - messages = backend.read_stream('s1') - expect(messages.map(&:id)).to eq([msg0, msg1, msg2, msg3].map(&:id)) - expect(messages.map(&:seq)).to eq([1, 2, 3, 4]) - end - end - - end - - describe '#clear!' do - it 'clears scheduled messages and workers' do - msg = Tests::DoSomething.parse(stream_id: 's1', payload: { account_id: 1 }) - backend.schedule_messages([msg], at: Time.now + 60) - backend.worker_heartbeat(['worker-1']) - - backend.clear! - - # Scheduled messages should be gone - Timecop.freeze(Time.now + 120) do - expect(backend.update_schedule!).to eq(0) - end - - # Streams and messages should be gone - expect(backend.read_stream('s1')).to be_empty - end - end - - describe '#append_next_to_stream' do - it 'appends single event to stream incrementing :seq automatically' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', evt1) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 2 }) - expect(backend.append_next_to_stream('s1', evt2)).to be(true) - events = backend.read_stream('s1') - expect(events.map(&:stream_id)).to eq(['s1', 's1']) - expect(events.map(&:seq)).to eq([1, 2]) - end - - it 'appends multiple events to stream incrementing :seq automatically' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', evt1) - - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 2 }) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 3 }) - evt4 = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 4 }) - - expect(backend.append_next_to_stream('s1', [evt2, evt3, evt4])).to be(true) - events = backend.read_stream('s1') - expect(events.map(&:stream_id)).to eq(['s1', 's1', 's1', 's1']) - expect(events.map(&:seq)).to eq([1, 2, 3, 4]) - end - - it 'handles empty array' do - expect(backend.append_next_to_stream('s1', [])).to be(true) - events = backend.read_stream('s1') - expect(events).to eq([]) - end - - it 'creates new stream when appending to non-existent stream with array' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 2 }) - - expect(backend.append_next_to_stream('s1', [evt1, evt2])).to be(true) - events = backend.read_stream('s1') - expect(events.map(&:stream_id)).to eq(['s1', 's1']) - expect(events.map(&:seq)).to eq([1, 2]) - end - end - - describe '#append_to_stream and #reserve_next_for_reactor' do - it 'supports a time window' do - now = Time.now - cmd_a = nil - evt_a1 = nil - evt_a2 = nil - - Timecop.freeze(now - 60) do - cmd_a = Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt_a1 = cmd_a.follow_with_seq(Tests::SomethingHappened1, 2, account_id: cmd_a.payload.account_id) - end - Timecop.freeze(now - 3) do - evt_a2 = cmd_a.follow_with_seq(Tests::SomethingHappened1, 3, account_id: cmd_a.payload.account_id) - end - backend.append_to_stream('s1', [cmd_a, evt_a1, evt_a2]) - - reactor1 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new( - group_id: 'group1', - start_from: -> { Time.now - 5 } - ) - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - - backend.register_consumer_group('group1') - - messages = [] - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages).to eq([evt_a2]) - end - - it 'schedules messages and handles them in the future, setting correlation IDs properly' do - cmd_a = Tests::DoSomething.parse(stream_id: 's1', correlation_id: SecureRandom.uuid, seq: 1, payload: { account_id: 1 }) - evt_b = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - - reactor1 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::DoSomething, Tests::SomethingHappened1] - end - end - - backend.register_consumer_group('group1') - backend.append_to_stream('s1', [cmd_a]) - - now = Time.now - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::Schedule.new([evt_b], at: now + 10), msg] } - end - - expect(backend.read_stream('s1')).to eq([cmd_a]) - - Timecop.freeze(now + 11) do - backend.update_schedule! - messages = backend.read_stream('s1') - expect(messages.map(&:id)).to eq([cmd_a, evt_b].map(&:id)) - expect(messages[1]).to be_a(Tests::SomethingHappened1) - expect(messages[1].causation_id).to eq(messages[0].id) - expect(messages[1].correlation_id).to eq(messages[0].correlation_id) - - # Check that initial message cmd_a was ACKed already - # it won't be claimed again - # only the new evt_b can be claimed now - list = [] - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| list << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - expect(list.map(&:id)).to eq([evt_b.id]) - end - end - - it 'handles multiple actions' do - cmd_a = Tests::DoSomething.parse(stream_id: 's1', correlation_id: SecureRandom.uuid, seq: 1, payload: { account_id: 1 }) - evt_b = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 2 }) - evt_c = Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 3 }) - - reactor1 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::DoSomething, Tests::SomethingHappened1] - end - end - - backend.append_to_stream('s1', [cmd_a]) - backend.register_consumer_group('group1') - now = Time.now - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map do |msg, _| - action1 = Sourced::Actions::AppendNext.new([evt_b]) - action2 = Sourced::Actions::Schedule.new([evt_c], at: now + 10) - [[action1, action2], msg] - end - end - - messages = backend.read_stream('s1') - expect(messages.map(&:id)).to eq([cmd_a, evt_b].map(&:id)) - - Timecop.freeze(now + 11) do - backend.update_schedule! - messages = backend.read_stream('s1') - expect(messages.map(&:id)).to eq([cmd_a, evt_b, evt_c].map(&:id)) - end - end - - it 'appends messages and reserves them in order of arrival' do - cmd_a = Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - cmd_b = Tests::DoSomething.parse(stream_id: 's2', seq: 1, payload: { account_id: 2 }) - evt_a1 = cmd_a.with(metadata: { tid: 'evt_a1' }).follow_with_seq(Tests::SomethingHappened1, 2, account_id: cmd_a.payload.account_id) - evt_a2 = cmd_a.with(metadata: { tid: 'evt_a2' }).follow_with_seq(Tests::SomethingHappened1, 3, account_id: cmd_a.payload.account_id) - evt_b1 = cmd_b.with(metadata: { tid: 'evt_b1' }).follow_with_seq(Tests::SomethingHappened1, 2, account_id: cmd_b.payload.account_id) - - reactor1 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - - reactor2 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group2') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - - reactor3 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group3') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - - backend.register_consumer_group('group1') - backend.register_consumer_group('group2') - backend.register_consumer_group('group3') - - backend.stop_consumer_group('group3') - - expect(backend.append_to_stream('s1', [cmd_a, evt_a1, evt_a2])).to be(true) - expect(backend.append_to_stream('s2', [cmd_b, evt_b1])).to be(true) - - group1_messages = [] - group2_messages = [] - group3_messages = [] - - # Test that concurrent consumers for the same group - # never process events for the same stream - Sourced.config.executor.start do |t| - t.spawn do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - sleep 0.01 - batch.each { |msg, _| group1_messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - end - t.spawn do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| group1_messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - end - end - - expect(group1_messages).to match_array([evt_b1, evt_a1]) - - # Test that separate groups have their own cursors on streams - backend.reserve_next_for_reactor(reactor2) do |batch, _history| - batch.each { |msg, _| group2_messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(group2_messages).to eq([evt_a1]) - - # Test stopped reactors are ignored - backend.reserve_next_for_reactor(reactor3) do |batch, _history| - batch.each { |msg, _| group3_messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(group3_messages).to eq([]) - - # Test that NOOP handlers still advance the cursor - backend.reserve_next_for_reactor(reactor2) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # Test returning RETRY does not advance the cursor - backend.reserve_next_for_reactor(reactor2) do |batch, _history| - Sourced::Actions::RETRY - end - - # Verify state of groups with stats - stats = backend.stats - - expect(stats.stream_count).to eq(2) - expect(stats.max_global_seq).to eq(5) - - expect(stats.groups).to match_array([ - { group_id: 'group1', status: 'active', retry_at: nil, oldest_processed: 2, newest_processed: 5, stream_count: 2 }, - { group_id: 'group2', status: 'active', retry_at: nil, oldest_processed: 3, newest_processed: 3, stream_count: 1 }, - { group_id: 'group3', status: 'stopped', retry_at: nil, oldest_processed: 0, newest_processed: 0, stream_count: 0 } - ]) - - #  Test that reactors with events not in the stream do not advance the cursor - reactor4 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group4') - end - - def self.handled_messages - [Tests::SomethingHappened2] - end - end - - backend.register_consumer_group('group4') - - group4_messages = [] - - backend.reserve_next_for_reactor(reactor4) do |batch, _history| - batch.each { |msg, _| group4_messages << msg } - batch.map { |msg, _| [[], msg] } - end - - expect(group4_messages).to eq([]) - - expect(backend.stats.groups).to match_array([ - { group_id: 'group1', status: 'active', retry_at: nil, oldest_processed: 2, newest_processed: 5, stream_count: 2 }, - { group_id: 'group2', status: 'active', retry_at: nil, oldest_processed: 3, newest_processed: 3, stream_count: 1 }, - { group_id: 'group3', status: 'stopped', retry_at: nil, oldest_processed: 0, newest_processed: 0, stream_count: 0 }, - { group_id: 'group4', status: 'active', retry_at: nil, oldest_processed: 0, newest_processed: 0, stream_count: 0 } - ]) - - # Now append an event that Reactor4 cares about - evt_a3 = cmd_a.follow_with_seq(Tests::SomethingHappened2, 4, account_id: cmd_a.payload.account_id) - backend.append_to_stream('s1', [evt_a3]) - - backend.reserve_next_for_reactor(reactor4) do |batch, _history| - batch.each { |msg, _| group4_messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(group4_messages).to eq([evt_a3]) - - expect(backend.stats.groups).to match_array([ - { group_id: 'group1', status: 'active', retry_at: nil, oldest_processed: 2, newest_processed: 5, stream_count: 2 }, - { group_id: 'group2', status: 'active', retry_at: nil, oldest_processed: 3, newest_processed: 3, stream_count: 1 }, - { group_id: 'group3', status: 'stopped', retry_at: nil, oldest_processed: 0, newest_processed: 0, stream_count: 0 }, - { group_id: 'group4', status: 'active', retry_at: nil, oldest_processed: 6, newest_processed: 6, stream_count: 1 } - ]) - - #  Test that #reserve_next_for returns next event, or nil - evt = backend.reserve_next_for_reactor(reactor2) { |batch, _| batch.map { |msg, _| [Sourced::Actions::OK, msg] } } - expect(evt).to eq(evt_b1) - - evt = backend.reserve_next_for_reactor(reactor2) { |batch, _| batch.map { |msg, _| [Sourced::Actions::OK, msg] } } - expect(evt).to be(nil) - end - end - - describe '#reserve_next_for_reactor and #reset_consumer_group' do - it 'reserves events again after reset, yields batch with replaying flags' do - evt_a1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', [evt_a1]) - - reactor1 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - - backend.register_consumer_group('group1') - - messages = [] - replaying = [] - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, is_replaying| messages << msg; replaying << is_replaying } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # This is a noop since the event is already processed - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, is_replaying| messages << msg; replaying << is_replaying } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages).to eq([evt_a1]) - - # Anything that responds to #consumer_info.group_id - expect(backend.reset_consumer_group(reactor1)).to be(true) - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, is_replaying| messages << msg; replaying << is_replaying } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages).to eq([evt_a1, evt_a1]) - expect(replaying).to eq([false, true]) - end - end - - context '#reserve_next_for_reactor handling Sourced::Actions types' do - let(:reactor1) do - Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - end - - let(:evt1) do - Tests::SomethingHappened1.parse( - stream_id: 's1', - seq: 1, - correlation_id:, - metadata: { tid: 'evt1' }, - payload: { account_id: 1 } - ) - end - - let(:correlation_id) { SecureRandom.uuid } - - before do - backend.register_consumer_group('group1') - - backend.append_to_stream(evt1.stream_id, [evt1]) - end - - describe 'returning Sourced::Actions::OK' do - it 'ACKS the message' do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(backend.stats.groups.first[:newest_processed]).to eq(1) - end - end - - describe 'returning Sourced::Actions::RETRY' do - it 'does not ACK the message' do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - Sourced::Actions::RETRY - end - - expect(backend.stats.groups.first[:newest_processed]).to eq(0) - end - end - - describe 'returning Sourced::Actions::AppendNext' do - before do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::AppendNext.new([Tests::SomethingHappened1.parse(stream_id: 's1', payload: { account_id: 2 })]), msg] } - end - end - - it 'appends messages to stream and auto-increments seq' do - events = backend.read_stream('s1') - expect(events.map(&:seq)).to eq([1, 2]) - expect(events.map(&:payload).map(&:account_id)).to eq([1, 2]) - end - - it 'ACKs processed message' do - expect(backend.stats.groups.first[:newest_processed]).to eq(1) - end - - it 'correlates messages and copies metadata' do - events = backend.read_stream('s1') - expect(events.map(&:causation_id)).to eq([evt1.id, evt1.id]) - expect(events.map(&:correlation_id)).to eq([correlation_id, correlation_id]) - expect(events.map(&:metadata).map { |m| m[:tid] }).to eq(['evt1', 'evt1']) - end - end - - describe 'returning Sourced::Actions::Ack' do - it 'ACKS the specific message ID passed' do - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 3 }) - backend.append_to_stream(evt2.stream_id, [evt2]) - # First message yielded will be evt1 - # but we'll ACK evt2 directly - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::Ack.new(evt2.id), msg] } - end - - new_messages = [] - # No new messages to fetch now, because we ACKed the last one - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| new_messages << msg }; batch.map { |msg, _| [[], msg] } - end - - expect(new_messages).to eq([]) - expect(backend.stats.groups.first[:newest_processed]).to eq(2) - end - end - - describe 'returning Sourced::Actions::AppendAfter' do - it 'appends messages to stream if sequence do not conflict' do - new_message = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::AppendAfter.new(new_message.stream_id, [new_message]), msg] } - end - - events = backend.read_stream('s1') - expect(events.map(&:seq)).to eq([1, 2]) - expect(events.map(&:payload).map(&:account_id)).to eq([1, 2]) - end - - it 'raises Sourced::ConcurrentAppendError if sequences conflict' do - new_message = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 2 }) - expect do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::AppendAfter.new(new_message.stream_id, [new_message]), msg] } - end - end.to raise_error(Sourced::ConcurrentAppendError) - end - - it 'does not append messages if #ack_event raises' do - pending 'how to test that a failed trasaction rolls back appending messages?' - new_message = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - # allow(backend).to receive(:ack_event).and_raise(StandardError) - - expect do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::AppendAfter.new(new_message.stream_id, [new_message]), msg] } - end - end.to raise_error(StandardError) - - expect(backend.read_stream('s1').map(&:seq)).to eq([1]) - end - - it 'correlates messages and copies metadata' do - new_message = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::AppendAfter.new(new_message.stream_id, [new_message]), msg] } - end - - events = backend.read_stream('s1') - expect(events.map(&:causation_id)).to eq([evt1.id, evt1.id]) - expect(events.map(&:correlation_id)).to eq([correlation_id, correlation_id]) - expect(events.map(&:metadata).map { |m| m[:tid] }).to eq(['evt1', 'evt1']) - end - end - - describe 'returning Sourced::Actions::Sync' do - it 'runs sync work within transaction' do - worked = false - work = proc do - worked = true - end - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::Sync.new(work), msg] } - end - - expect(worked).to be(true) - end - - it 'acks' do - work = proc{} - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::Sync.new(work), msg] } - end - - expect(backend.stats.groups.first[:newest_processed]).to eq(1) - end - end - - describe 'returning empty actions' do - it 'acks by default' do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [[], msg] } - end - - expect(backend.stats.groups.first[:newest_processed]).to eq(1) - end - - it 'acks if nil action' do - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.map { |msg, _| [[nil], msg] } - end - - expect(backend.stats.groups.first[:newest_processed]).to eq(1) - end - end - end - - describe '#reserve_next_for_reactor with batch_size' do - let(:reactor1) do - Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - end - - before do - backend.register_consumer_group('group1') - end - - it 'fetches and processes multiple messages from the same stream in one call' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 3 }) - backend.append_to_stream('s1', [evt1, evt2, evt3]) - - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.map(&:id)).to eq([evt1, evt2, evt3].map(&:id)) - end - - it 'returns the first message from the batch' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.append_to_stream('s1', [evt1, evt2]) - - result = backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(result.id).to eq(evt1.id) - end - - it 'limits to batch_size messages' do - evts = 5.times.map do |i| - Tests::SomethingHappened1.parse(stream_id: 's1', seq: i + 1, payload: { account_id: i }) - end - backend.append_to_stream('s1', evts) - - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 3) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.size).to eq(3) - expect(messages.map(&:id)).to eq(evts[0..2].map(&:id)) - - # Next batch picks up remaining messages - messages2 = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 3) do |batch, _history| - batch.each { |msg, _| messages2 << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages2.size).to eq(2) - expect(messages2.map(&:id)).to eq(evts[3..4].map(&:id)) - end - - it 'ACKs all messages on success' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.append_to_stream('s1', [evt1, evt2]) - - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # Nothing left to process - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages).to be_empty - end - - it 'returns RETRY for all-or-nothing retry' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 3 }) - backend.append_to_stream('s1', [evt1, evt2, evt3]) - - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |_batch, _history| - Sourced::Actions::RETRY - end - - # No messages were ACKed, all should be available on next call - remaining = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| remaining << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(remaining.map(&:id)).to eq([evt1, evt2, evt3].map(&:id)) - end - - it 'releases offset without ACK when RETRY returned' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.append_to_stream('s1', [evt1, evt2]) - - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |_batch, _history| - Sourced::Actions::RETRY - end - - # All messages should still be available - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.map(&:id)).to eq([evt1, evt2].map(&:id)) - end - - it 'only fetches messages matching handled_messages types' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened2.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 3 }) - backend.append_to_stream('s1', [evt1, evt2, evt3]) - - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # reactor1 only handles SomethingHappened1, not SomethingHappened2 - expect(messages.map(&:id)).to eq([evt1, evt3].map(&:id)) - end - - it 'sets replaying flag correctly per message' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.append_to_stream('s1', [evt1, evt2]) - - # Process both messages first - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # Reset consumer group to trigger replaying - backend.reset_consumer_group(reactor1) - - # Add a new message (not replaying) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 3 }) - backend.append_to_stream('s1', [evt3]) - - replaying_flags = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - replaying_flags = batch.map { |_, is_replaying| is_replaying } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # evt1 and evt2 are replaying (global_seq <= highest_global_seq), evt3 is not - expect(replaying_flags).to eq([true, true, false]) - end - - it 'handles AppendNext actions within a batch' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - backend.append_to_stream('s1', [evt1, evt2]) - - new_cmd = Tests::DoSomething.parse(stream_id: 's2', payload: { account_id: 99 }) - - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.map.with_index do |(msg, _), idx| - if idx == 0 - [Sourced::Actions::AppendNext.new([new_cmd]), msg] - else - [Sourced::Actions::OK, msg] - end - end - end - - # Both messages processed, and the AppendNext command was appended - expect(backend.read_stream('s2').map(&:id)).to eq([new_cmd.id]) - end - - it 'returns nil when there are no messages to process' do - result = backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(result).to be_nil - end - - it 'handles a single message in batch mode' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', [evt1]) - - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.map(&:id)).to eq([evt1.id]) - end - - it 'releases offset when block returns empty action_pairs' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', [evt1]) - - # Return empty action_pairs — nothing to ACK - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |_batch, _history| - [] - end - - # Offset should have been released, so the message is available again - messages = [] - backend.reserve_next_for_reactor(reactor1, batch_size: 10) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.map(&:id)).to eq([evt1.id]) - end - end - - describe '#reserve_next_for_reactor with_history' do - let(:reactor1) do - Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - end - - before do - backend.register_consumer_group('group1') - end - - it 'yields history containing all stream messages when with_history: true' do - cmd = Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 3 }) - backend.append_to_stream('s1', [cmd, evt1, evt2]) - - yielded_history = nil - backend.reserve_next_for_reactor(reactor1, batch_size: 10, with_history: true) do |batch, history| - yielded_history = history - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - # History should include ALL stream messages (cmd + evt1 + evt2), not just batch - expect(yielded_history).not_to be_nil - expect(yielded_history.map(&:id)).to eq([cmd, evt1, evt2].map(&:id)) - end - - it 'yields nil history when with_history: false (default)' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', [evt1]) - - yielded_history = :not_set - backend.reserve_next_for_reactor(reactor1) do |batch, history| - yielded_history = history - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(yielded_history).to be_nil - end - - it 'history does not include messages from other streams' do - evt_s1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt_s2 = Tests::SomethingHappened1.parse(stream_id: 's2', seq: 1, payload: { account_id: 2 }) - backend.append_to_stream('s1', [evt_s1]) - backend.append_to_stream('s2', [evt_s2]) - - yielded_history = nil - backend.reserve_next_for_reactor(reactor1, with_history: true) do |batch, history| - yielded_history = history - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(yielded_history.map(&:stream_id).uniq).to eq([yielded_history.first.stream_id]) - end - end - - describe '#reserve_next_for_reactor with retry_at' do - it 'does not fetch events until retry_at is up' do - now = Time.now - - evt_a1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', [evt_a1]) - - reactor1 = Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - - backend.register_consumer_group('group1') - backend.updating_consumer_group('group1') do |gr| - gr.retry(now + 4) - end - - messages = [] - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.any?).to be(false) - - backend.updating_consumer_group('group1') do |gr| - gr.retry(now - 1) - end - - backend.reserve_next_for_reactor(reactor1) do |batch, _history| - batch.each { |msg, _| messages << msg } - batch.map { |msg, _| [Sourced::Actions::OK, msg] } - end - - expect(messages.any?).to be(true) - end - end - - describe '#ack_on' do - let(:reactor) do - Class.new do - def self.consumer_info - Sourced::Consumer::ConsumerInfo.new(group_id: 'group1') - end - - def self.handled_messages - [Tests::SomethingHappened1] - end - end - end - - let(:evt1) { Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) } - let(:evt2) { Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 1 }) } - - before do - backend.append_to_stream('s1', [evt1, evt2]) - end - - it 'advances a group_id/stream_id offset if no exception' do - backend.ack_on(reactor.consumer_info.group_id, evt1.id) { true } - - backend.reserve_next_for_reactor(reactor) { |batch, _| batch.map { |msg, _| [Sourced::Actions::OK, msg] } } - - expect(backend.stats.groups.first[:oldest_processed]).to eq(2) - end - - it 'does not advance offset if exception' do - begin - backend.ack_on(reactor.consumer_info.group_id, evt1.id) do - raise RuntimeError - end - rescue RuntimeError - end - - expect(backend.stats.groups.size).to eq(0) - end - - end - - describe '#read_correlation_batch' do - specify 'given an event ID, it returns the list of correlated events' do - cmd1 = Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt1 = cmd1.follow_with_seq(Tests::SomethingHappened1, 2, cmd1.payload) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 1 }) - evt2 = cmd1.follow_with_seq(Tests::SomethingHappened1, 4, cmd1.payload) - - expect(backend.append_to_stream('s1', [cmd1, evt1, evt2, evt3])).to be(true) - - events = backend.read_correlation_batch(evt2.id) - expect(events).to eq([cmd1, evt1, evt2]) - end - - it 'returns empty list if no event found' do - no = SecureRandom.uuid - events = backend.read_correlation_batch(no) - expect(events.empty?).to be(true) - end - end - - describe '#append_to_stream' do - it 'accepts a single event' do - evt = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - expect(backend.append_to_stream('s1', evt)).to be(true) - - events = backend.read_stream('s1') - expect(events.size).to eq(1) - expect(events.first).to eq(evt) - end - - it 'accepts an array of events' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - expect(backend.append_to_stream('s1', [evt1, evt2])).to be(true) - - events = backend.read_stream('s1') - expect(events.size).to eq(2) - expect(events).to eq([evt1, evt2]) - end - - it 'handles empty array' do - expect(backend.append_to_stream('s1', [])).to be(false) - events = backend.read_stream('s1') - expect(events).to eq([]) - end - - it 'fails if duplicate [stream_id, seq]' do - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - backend.append_to_stream('s1', evt1) - - expect do - backend.append_to_stream('s1', evt2) - end.to raise_error(Sourced::ConcurrentAppendError) - end - end - - describe '#recent_streams' do - it 'returns streams ordered by most recent activity first' do - now = Time.now - - evt1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt2 = Tests::SomethingHappened1.parse(stream_id: 's2', seq: 10, payload: { account_id: 1 }) - - # Create first stream - backend.append_to_stream('s1', [evt1]) - - # Create second stream 5 seconds later - Timecop.freeze(now + 5) do - backend.append_to_stream('s2', [evt2]) - end - - streams = backend.recent_streams(limit: 20) - - # Should be ordered by most recent first (s2, then s1) - expect(streams.map(&:stream_id)).to eq(['s2', 's1']) - expect(streams.map(&:seq)).to eq([10, 1]) - expect(streams.first.updated_at).to be_a(Time) - expect(streams.size).to eq(2) - end - - it 'respects the limit parameter' do - # Create 5 streams with different timestamps - 5.times do |i| - evt = Tests::SomethingHappened1.parse(stream_id: "s#{i}", seq: 1, payload: { account_id: 1 }) - Timecop.freeze(Time.now + i) do - backend.append_to_stream("s#{i}", [evt]) - end - end - - # Test limit smaller than total streams - streams = backend.recent_streams(limit: 3) - expect(streams.size).to eq(3) - expect(streams.map(&:stream_id)).to eq(['s4', 's3', 's2']) # Most recent first - - # Test limit larger than total streams - streams = backend.recent_streams(limit: 10) - expect(streams.size).to eq(5) # Should return all 5 streams - expect(streams.map(&:stream_id)).to eq(['s4', 's3', 's2', 's1', 's0']) - - # Test limit of 1 - streams = backend.recent_streams(limit: 1) - expect(streams.size).to eq(1) - expect(streams.first.stream_id).to eq('s4') # Most recent - end - - it 'uses default limit when not specified' do - # Create more streams than the default limit (10) - 12.times do |i| - evt = Tests::SomethingHappened1.parse(stream_id: "s#{i}", seq: 1, payload: { account_id: 1 }) - backend.append_to_stream("s#{i}", [evt]) - end - - streams = backend.recent_streams # No limit specified - expect(streams.size).to eq(10) # Should default to 10 - end - - it 'handles edge cases for limit parameter' do - # Create a few streams - 3.times do |i| - evt = Tests::SomethingHappened1.parse(stream_id: "s#{i}", seq: 1, payload: { account_id: 1 }) - backend.append_to_stream("s#{i}", [evt]) - end - - # Test limit of 0 - streams = backend.recent_streams(limit: 0) - expect(streams.size).to eq(0) - - # Test very large limit - streams = backend.recent_streams(limit: 1000) - expect(streams.size).to eq(3) # Should return all available streams - end - - it 'validates input parameters' do - # Test negative limit - expect { - backend.recent_streams(limit: -1) - }.to raise_error(ArgumentError, "limit must be a positive integer") - - expect { - backend.recent_streams(limit: -5) - }.to raise_error(ArgumentError, "limit must be a positive integer") - end - end - - describe '#read_stream' do - it 'reads full event stream in order' do - cmd1 = Tests::DoSomething.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - evt1 = cmd1.follow_with_seq(Tests::SomethingHappened1, 2, account_id: cmd1.payload.account_id) - evt2 = cmd1.follow_with_seq(Tests::SomethingHappened1, 3, account_id: cmd1.payload.account_id) - evt3 = Tests::SomethingHappened1.parse(stream_id: 's2', seq: 4, payload: { account_id: 1 }) - expect(backend.append_to_stream('s1', [evt1, evt2])).to be(true) - expect(backend.append_to_stream('s2', [evt3])).to be(true) - events = backend.read_stream('s1') - expect(events).to eq([evt1, evt2]) - end - - it ':upto and :after' do - e1 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 1, payload: { account_id: 1 }) - e2 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 2, payload: { account_id: 2 }) - e3 = Tests::SomethingHappened1.parse(stream_id: 's1', seq: 3, payload: { account_id: 2 }) - expect(backend.append_to_stream('s1', [e1, e2, e3])).to be(true) - events = backend.read_stream('s1', upto: 2) - expect(events).to eq([e1, e2]) - - events = backend.read_stream('s1', after: 1) - expect(events).to eq([e2, e3]) - end - end - - describe '#updating_consumer_group' do - specify '#retry_at(Time)' do - later = Time.now + 10 - counts = [] - backend.register_consumer_group('group1') - backend.updating_consumer_group('group1') do |group| - counts << group.error_context[:retry_count] - group.retry(later, retry_count: 1) - end - backend.updating_consumer_group('group1') do |group| - counts << group.error_context[:retry_count] - group.retry(later) - end - gr = backend.stats.groups.first - expect(gr[:group_id]).to eq('group1') - expect(gr[:status]).to eq('active') - expect(gr[:retry_at]).to eq(later) - expect(counts).to eq([nil, 1]) - end - - specify '#stop(message:)' do - backend.register_consumer_group('group1') - backend.updating_consumer_group('group1') do |group| - group.stop(message: 'operator requested shutdown') - end - - gr = backend.stats.groups.first - expect(gr[:group_id]).to eq('group1') - expect(gr[:status]).to eq('stopped') - - backend.start_consumer_group('group1') - gr = backend.stats.groups.first - expect(gr[:group_id]).to eq('group1') - expect(gr[:status]).to eq('active') - end - - specify '#fail(exception:)' do - backend.register_consumer_group('group1') - backend.updating_consumer_group('group1') do |group| - group.fail(exception: StandardError.new('boom')) - end - - gr = backend.stats.groups.first - expect(gr[:group_id]).to eq('group1') - expect(gr[:status]).to eq('failed') - - backend.start_consumer_group('group1') - gr = backend.stats.groups.first - expect(gr[:group_id]).to eq('group1') - expect(gr[:status]).to eq('active') - end - end - - describe '#notifier' do - it 'returns an object responding to subscribe, publish, notify_new_messages, notify_reactor_resumed, start, stop' do - n = backend.notifier - expect(n).to respond_to(:subscribe) - expect(n).to respond_to(:publish) - expect(n).to respond_to(:notify_new_messages) - expect(n).to respond_to(:notify_reactor_resumed) - expect(n).to respond_to(:start) - expect(n).to respond_to(:stop) - end - - it 'returns the same instance on repeated calls' do - expect(backend.notifier).to be(backend.notifier) - end - end - end - - class Migrator - attr_reader :migration_version, :table_prefix - - def initialize(table_prefix: 'sors', root_dir: File.expand_path('../..', __dir__)) - @table_prefix = table_prefix - @root_dir = root_dir - @migration_version = "[#{ActiveRecord::VERSION::STRING.to_f}]" - @migdir = File.join(@root_dir, 'spec', 'db', 'migrate') - @migfilename = File.join(@migdir, 'create_sors_tables.rb') - end - - def up - return if Sourced::Backends::ActiveRecordBackend.installed? - - migfile = File.read(File.join(@root_dir, 'lib', 'sors', 'rails', 'templates', 'create_sors_tables.rb.erb')) - migcontent = ERB.new(migfile).result(binding) - FileUtils.mkdir_p(@migdir) - File.write(@migfilename, migcontent) - require @migfilename.sub('.rb', '') - CreateSorsTables.new.change - end - - def down - Sourced::Backends::ActiveRecordBackend.uninstall! - File.delete(@migfilename) if File.exist?(@migfilename) - end - end -end diff --git a/spec/shared_examples/executor_examples.rb b/spec/shared_examples/executor_examples.rb deleted file mode 100644 index 36b2f790..00000000 --- a/spec/shared_examples/executor_examples.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module ExecutorExamples - RSpec.shared_examples 'an executor' do - it 'runs work concurrently' do - results = [] - queue = Thread::Queue.new - executor.start do |task| - task.spawn do - sleep 0.00001 - queue << 1 - end - - task.spawn do - queue << 2 - end - end - - queue.close - while (it = queue.pop) - results << it - end - - expect(results).to eq([2, 1]) - end - - it 'waits and re-raises errors' do - expect do - executor.start do |task| - task.spawn do - raise ArgumentError, 'Test error' - end - - task.spawn do - - end - end - end.to raise_error(ArgumentError, 'Test error') - end - end -end diff --git a/spec/shared_examples/unit_examples.rb b/spec/shared_examples/unit_examples.rb deleted file mode 100644 index 64822617..00000000 --- a/spec/shared_examples/unit_examples.rb +++ /dev/null @@ -1,375 +0,0 @@ -# frozen_string_literal: true - -# Shared examples for Sourced::Unit. -# Include in a describe block that defines `let(:backend)`. -# -# @example With TestBackend -# let(:backend) { Sourced::Backends::TestBackend.new } -# it_behaves_like 'a unit' -# -# @example With SequelBackend -# let(:backend) { Sourced::Backends::SequelBackend.new(db) } -# it_behaves_like 'a unit' -RSpec.shared_examples 'a unit' do - let(:stream_id) { "thing-#{SecureRandom.uuid}" } - - before do - UnitTest::SyncLog.clear - end - - describe 'full chain execution' do - it 'runs command -> event -> reaction -> command -> event synchronously' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - results = unit.handle(cmd) - - # ThingActor should have produced ThingCreated - thing_events = results.events_for(UnitTest::ThingActor) - expect(thing_events.size).to eq(1) - expect(thing_events.first).to be_a(UnitTest::ThingCreated) - expect(thing_events.first.payload.name).to eq('Widget') - - # NotifierActor should have produced ThingNotified - notifier_events = results.events_for(UnitTest::NotifierActor) - expect(notifier_events.size).to eq(1) - expect(notifier_events.first).to be_a(UnitTest::ThingNotified) - - # All messages should be in the backend - stream_messages = backend.read_stream(stream_id) - message_types = stream_messages.map(&:class) - expect(message_types).to include(UnitTest::CreateThing) - expect(message_types).to include(UnitTest::ThingCreated) - expect(message_types).to include(UnitTest::NotifyThing) - expect(message_types).to include(UnitTest::ThingNotified) - end - end - - describe 'multi-reactor handling' do - it 'routes events to multiple reactors' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::ThingProjector, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - results = unit.handle(cmd) - - # ThingActor produced ThingCreated - thing_events = results.events_for(UnitTest::ThingActor) - expect(thing_events.size).to eq(1) - - # ThingProjector also received ThingCreated (evolves state) - projector_results = results[UnitTest::ThingProjector] - expect(projector_results).not_to be_empty - instance = projector_results.keys.first - expect(instance.state[:things]).to eq(['Widget']) - end - end - - describe 'offset tracking' do - it 'ACKs messages so Router#drain finds no pending messages' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - unit.handle(cmd) - - # Set up a router with the same backend and reactor - router = Sourced::Router.new(backend: backend) - router.register(UnitTest::ThingActor) - - # drain should find nothing to process - logs = [] - allow(UnitTest::ThingActor).to receive(:handle).and_wrap_original do |m, *args, **kwargs| - logs << args.first.class - m.call(*args, **kwargs) - end - - router.drain - expect(logs).to be_empty - end - end - - describe 'correlation chain' do - it 'maintains correlation_id across the full chain' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - unit.handle(cmd) - - stream_messages = backend.read_stream(stream_id) - # All messages after the initial command should share the same correlation_id - correlation_ids = stream_messages.map(&:correlation_id).uniq - expect(correlation_ids.size).to eq(1) - end - - it 'propagates metadata from the initial command through the chain' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new( - stream_id: stream_id, - payload: { name: 'Widget' }, - metadata: { request_id: 'req-abc', user_id: 'u-1' } - ) - unit.handle(cmd) - - stream_messages = backend.read_stream(stream_id) - # Every message after the initial command should carry the original metadata - stream_messages.each do |msg| - expect(msg.metadata[:request_id]).to eq('req-abc'), "#{msg.class} missing request_id" - expect(msg.metadata[:user_id]).to eq('u-1'), "#{msg.class} missing user_id" - end - end - end - - describe 'infinite loop prevention' do - it 'raises InfiniteLoopError when max_iterations exceeded' do - unit = Sourced::Unit.new( - UnitTest::LoopingActor, - backend: backend, - max_iterations: 5 - ) - - cmd = UnitTest::LoopCmd.new(stream_id: stream_id) - expect { - unit.handle(cmd) - }.to raise_error(Sourced::Unit::InfiniteLoopError, /Exceeded 5 iterations/) - end - end - - describe 'transaction rollback' do - it 'rolls back all changes on error' do - error_actor = Class.new(Sourced::Actor) do - extend Sourced::Consumer - - consumer do |c| - c.group_id = 'UnitTest::ErrorActor' - end - - state do |id| - { id: id } - end - - command UnitTest::ThingCreated do |state, cmd| - raise 'Boom!' - end - - event UnitTest::ThingCreated do |state, event| - end - end - - unit = Sourced::Unit.new( - UnitTest::ThingActor, - error_actor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - expect { - unit.handle(cmd) - }.to raise_error(RuntimeError, 'Boom!') - - # Backend should have no messages due to rollback - expect(backend.read_stream(stream_id)).to be_empty - end - end - - describe 'scheduled messages' do - it 'schedules messages but does not execute them synchronously' do - unit = Sourced::Unit.new( - UnitTest::SchedulingActor, - backend: backend - ) - - cmd = UnitTest::ScheduleCmd.new(stream_id: stream_id) - results = unit.handle(cmd) - - # The ScheduleEvent should be produced - events = results.events_for(UnitTest::SchedulingActor) - expect(events.size).to eq(1) - expect(events.first).to be_a(UnitTest::ScheduleEvent) - - # DelayedCmd should NOT appear in the stream (it's scheduled for later) - stream_messages = backend.read_stream(stream_id) - expect(stream_messages.map(&:class)).not_to include(UnitTest::DelayedCmd) - end - end - - describe 'unhandled messages' do - it 'appends messages not handled by unit reactors for background workers' do - # Unit only has ThingActor, not NotifierActor - unit = Sourced::Unit.new( - UnitTest::ThingActor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - unit.handle(cmd) - - # NotifyThing command should still be in the store (from reaction's AppendNext) - stream_messages = backend.read_stream(stream_id) - message_types = stream_messages.map(&:class) - expect(message_types).to include(UnitTest::NotifyThing) - end - end - - describe 'Results API' do - it 'returns instance and produced events per reactor class' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::ThingProjector, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - results = unit.handle(cmd) - - # results[ThingActor] returns { instance => [events] } - actor_results = results[UnitTest::ThingActor] - expect(actor_results.size).to eq(1) - - instance, events = actor_results.first - expect(instance).to be_a(UnitTest::ThingActor) - expect(instance.state[:name]).to eq('Widget') - expect(instance.state[:status]).to eq('created') - expect(events.size).to eq(1) - expect(events.first).to be_a(UnitTest::ThingCreated) - - # results[ThingProjector] returns { instance => [events] } - projector_results = results[UnitTest::ThingProjector] - expect(projector_results.size).to eq(1) - - proj_instance, proj_events = projector_results.first - expect(proj_instance).to be_a(UnitTest::ThingProjector) - # Projectors don't produce events (AppendAfter), so this is empty - expect(proj_events).to be_empty - end - end - - describe 'sync actions' do - it 'executes sync actions within the transaction' do - unit = Sourced::Unit.new( - UnitTest::SyncActor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - unit.handle(cmd) - - expect(UnitTest::SyncLog.size).to eq(1) - log = UnitTest::SyncLog.first - expect(log[:command]).to eq(UnitTest::CreateThing) - expect(log[:events].first).to eq(UnitTest::ThingCreated) - end - end - - describe 'persist_commands: false' do - it 'does not persist commands but events are always persisted' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: backend, - persist_commands: false - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - results = unit.handle(cmd) - - # Full chain still runs synchronously - thing_events = results.events_for(UnitTest::ThingActor) - expect(thing_events.size).to eq(1) - expect(thing_events.first).to be_a(UnitTest::ThingCreated) - - notifier_events = results.events_for(UnitTest::NotifierActor) - expect(notifier_events.size).to eq(1) - expect(notifier_events.first).to be_a(UnitTest::ThingNotified) - - # Events should be in the store - stream_messages = backend.read_stream(stream_id) - message_types = stream_messages.map(&:class) - expect(message_types).to include(UnitTest::ThingCreated) - expect(message_types).to include(UnitTest::ThingNotified) - - # Commands should NOT be in the store - expect(message_types).not_to include(UnitTest::CreateThing) - expect(message_types).not_to include(UnitTest::NotifyThing) - end - - it 'still runs the full BFS chain even though commands are not persisted' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: backend, - persist_commands: false - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - results = unit.handle(cmd) - - # Both actors' state should be updated - actor_results = results[UnitTest::ThingActor] - expect(actor_results.values.flatten.size).to eq(1) - - notifier_results = results[UnitTest::NotifierActor] - expect(notifier_results.values.flatten.size).to eq(1) - end - end - - describe 'persist_commands: true (default)' do - it 'persists all messages including commands' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: backend - ) - - cmd = UnitTest::CreateThing.new(stream_id: stream_id, payload: { name: 'Widget' }) - unit.handle(cmd) - - stream_messages = backend.read_stream(stream_id) - message_types = stream_messages.map(&:class) - expect(message_types).to include(UnitTest::CreateThing) - expect(message_types).to include(UnitTest::ThingCreated) - expect(message_types).to include(UnitTest::NotifyThing) - expect(message_types).to include(UnitTest::ThingNotified) - end - end - - describe 'reusability' do - it 'can handle multiple commands without instance state mutation' do - unit = Sourced::Unit.new( - UnitTest::ThingActor, - UnitTest::NotifierActor, - backend: backend - ) - - stream_id_1 = "#{stream_id}-1" - stream_id_2 = "#{stream_id}-2" - - results1 = unit.handle(UnitTest::CreateThing.new(stream_id: stream_id_1, payload: { name: 'First' })) - results2 = unit.handle(UnitTest::CreateThing.new(stream_id: stream_id_2, payload: { name: 'Second' })) - - expect(results1.events_for(UnitTest::ThingActor).first.payload.name).to eq('First') - expect(results2.events_for(UnitTest::ThingActor).first.payload.name).to eq('Second') - - expect(backend.read_stream(stream_id_1).size).to eq(4) - expect(backend.read_stream(stream_id_2).size).to eq(4) - end - end -end diff --git a/spec/sourced/ccc/command_context_spec.rb b/spec/sourced/ccc/command_context_spec.rb deleted file mode 100644 index 84ce357c..00000000 --- a/spec/sourced/ccc/command_context_spec.rb +++ /dev/null @@ -1,242 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -module CccContextTest - Add = Sourced::CCC::Command.define('ccc_ctest.add') do - attribute :value, Integer - end - - Remove = Sourced::CCC::Command.define('ccc_ctest.remove') do - attribute :value, Integer - end - - Added = Sourced::CCC::Event.define('ccc_ctest.added') -end - -RSpec.describe Sourced::CCC::CommandContext do - describe '#build' do - it 'builds command from type string with metadata' do - ctx = described_class.new(metadata: { user_id: 10 }) - cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) - expect(cmd).to be_a(CccContextTest::Add) - expect(cmd.payload.value).to eq(1) - expect(cmd.metadata[:user_id]).to eq(10) - end - - it 'can take a command class' do - ctx = described_class.new(metadata: { user_id: 10 }) - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd).to be_a(CccContextTest::Add) - expect(cmd.payload.value).to eq(1) - expect(cmd.metadata[:user_id]).to eq(10) - end - - it 'symbolizes string keys' do - ctx = described_class.new(metadata: { user_id: 10 }) - cmd = ctx.build('type' => 'ccc_ctest.add', 'payload' => { 'value' => 1 }) - expect(cmd).to be_a(CccContextTest::Add) - expect(cmd.payload.value).to eq(1) - expect(cmd.metadata[:user_id]).to eq(10) - end - - it 'raises UnknownMessageError for unknown types' do - ctx = described_class.new(metadata: { user_id: 10 }) - expect do - ctx.build('type' => 'nope', 'payload' => { 'value' => 1 }) - end.to raise_error(Sourced::UnknownMessageError) - end - - it 'raises UnknownMessageError for event types when scoped to Command' do - ctx = described_class.new(metadata: { user_id: 10 }) - expect do - ctx.build('type' => 'ccc_ctest.added', 'payload' => {}) - end.to raise_error(Sourced::UnknownMessageError) - end - - it 'allows scoping to a custom command subclass' do - custom_scope = Class.new(Sourced::CCC::Command) - custom_cmd = custom_scope.define('ccc_ctest.custom') do - attribute :name, String - end - - ctx = described_class.new(metadata: { user_id: 10 }, scope: custom_scope) - cmd = ctx.build(type: 'ccc_ctest.custom', payload: { name: 'hello' }) - expect(cmd).to be_a(custom_cmd) - expect(cmd.payload.name).to eq('hello') - expect(cmd.metadata[:user_id]).to eq(10) - end - end - - describe 'callback hooks' do - it 'runs on block for matching command type' do - klass = Class.new(described_class) - klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } - - ctx = klass.new - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.payload.value).to eq(11) - end - - it 'registers on block for multiple command types' do - klass = Class.new(described_class) - klass.on(CccContextTest::Add, CccContextTest::Remove) { |_app, cmd| cmd.with_metadata(tagged: true) } - - ctx = klass.new - add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) - expect(add_cmd.metadata[:tagged]).to eq(true) - expect(remove_cmd.metadata[:tagged]).to eq(true) - end - - it 'accumulates multiple on blocks for the same command type' do - klass = Class.new(described_class) - klass.on(CccContextTest::Add, CccContextTest::Remove) { |_app, cmd| cmd.with_metadata(first: true) } - klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_metadata(second: true) } - - ctx = klass.new - # Add gets both blocks - add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(add_cmd.metadata[:first]).to eq(true) - expect(add_cmd.metadata[:second]).to eq(true) - - # Remove only gets the first block - remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) - expect(remove_cmd.metadata[:first]).to eq(true) - expect(remove_cmd.metadata).not_to have_key(:second) - end - - it 'does not run on block for non-matching command type' do - klass = Class.new(described_class) - klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 10) } - - ctx = klass.new - cmd = ctx.build(CccContextTest::Remove, payload: { value: 1 }) - expect(cmd.payload.value).to eq(1) - end - - it 'passes app scope to on block' do - app = double('app', session_id: 'abc') - klass = Class.new(described_class) - klass.on(CccContextTest::Add) { |a, cmd| cmd.with_metadata(session_id: a.session_id) } - - ctx = klass.new(app: app) - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.metadata[:session_id]).to eq('abc') - end - - it 'runs any block for all commands' do - klass = Class.new(described_class) - klass.any { |_app, cmd| cmd.with_metadata(source: 'web') } - - ctx = klass.new - add_cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - remove_cmd = ctx.build(CccContextTest::Remove, payload: { value: 2 }) - expect(add_cmd.metadata[:source]).to eq('web') - expect(remove_cmd.metadata[:source]).to eq('web') - end - - it 'runs on before any (pipeline order)' do - klass = Class.new(described_class) - klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_metadata(step: 'on') } - klass.any { |_app, cmd| cmd.with_metadata(step: "#{cmd.metadata[:step]}_any") } - - ctx = klass.new - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.metadata[:step]).to eq('on_any') - end - - it 'runs multiple any blocks in order' do - klass = Class.new(described_class) - klass.any { |_app, cmd| cmd.with_metadata(steps: ['first']) } - klass.any { |_app, cmd| cmd.with_metadata(steps: cmd.metadata[:steps] + ['second']) } - - ctx = klass.new - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.metadata[:steps]).to eq(%w[first second]) - end - - it 'passes through unchanged when no hooks registered' do - ctx = described_class.new(metadata: { user_id: 10 }) - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.payload.value).to eq(1) - expect(cmd.metadata[:user_id]).to eq(10) - end - - it 'app defaults to nil' do - klass = Class.new(described_class) - received_app = :not_set - klass.any { |a, cmd| received_app = a; cmd } - - ctx = klass.new - ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(received_app).to be_nil - end - - it 'subclass inherits parent blocks' do - parent = Class.new(described_class) - parent.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 100) } - parent.any { |_app, cmd| cmd.with_metadata(inherited: true) } - - child = Class.new(parent) - - ctx = child.new - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.payload.value).to eq(101) - expect(cmd.metadata[:inherited]).to eq(true) - end - - it 'subclass blocks do not affect parent' do - parent = Class.new(described_class) - child = Class.new(parent) - child.any { |_app, cmd| cmd.with_metadata(child_only: true) } - - ctx = parent.new - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.metadata).not_to have_key(:child_only) - end - - it 'works with build from type string + on block' do - klass = Class.new(described_class) - klass.on(CccContextTest::Add) { |_app, cmd| cmd.with_payload(value: cmd.payload.value + 5) } - - ctx = klass.new - cmd = ctx.build(type: 'ccc_ctest.add', payload: { value: 1 }) - expect(cmd.payload.value).to eq(6) - end - - it 'runs on blocks in the context of the instance' do - klass = Class.new(described_class) do - on(CccContextTest::Add) { |app, cmd| cmd.with_metadata(user_id: build_user_id(app)) } - - private - - def build_user_id(app) - "user-#{app.session_id}" - end - end - - app = double('app', session_id: '42') - ctx = klass.new(app: app) - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.metadata[:user_id]).to eq('user-42') - end - - it 'runs any blocks in the context of the instance' do - klass = Class.new(described_class) do - any { |app, cmd| cmd.with_metadata(source: request_source) } - - private - - def request_source - 'web' - end - end - - ctx = klass.new - cmd = ctx.build(CccContextTest::Add, payload: { value: 1 }) - expect(cmd.metadata[:source]).to eq('web') - end - end -end diff --git a/spec/sourced/ccc/configuration_spec.rb b/spec/sourced/ccc/configuration_spec.rb deleted file mode 100644 index 56272a60..00000000 --- a/spec/sourced/ccc/configuration_spec.rb +++ /dev/null @@ -1,276 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' -require 'sequel' - -RSpec.describe Sourced::CCC::Configuration do - after { Sourced::CCC.reset! } - - describe 'CCC.config' do - it 'returns a Configuration with sensible defaults' do - config = Sourced::CCC.config - expect(config).to be_a(described_class) - expect(config.worker_count).to eq(2) - expect(config.batch_size).to eq(50) - expect(config.catchup_interval).to eq(5) - expect(config.max_drain_rounds).to eq(10) - expect(config.claim_ttl_seconds).to eq(120) - expect(config.housekeeping_interval).to eq(30) - expect(config.logger).to eq(Sourced.config.logger) - end - - it 'returns the same instance on repeated calls' do - expect(Sourced::CCC.config).to be(Sourced::CCC.config) - end - end - - describe 'CCC.configure' do - it 'yields the config and freezes it after setup' do - Sourced::CCC.configure do |c| - c.worker_count = 4 - c.batch_size = 100 - end - - expect(Sourced::CCC.config.worker_count).to eq(4) - expect(Sourced::CCC.config.batch_size).to eq(100) - expect(Sourced::CCC.config).to be_frozen - end - - it 'calls setup! which creates store and router' do - Sourced::CCC.configure {} - - expect(Sourced::CCC.config.store).to be_a(Sourced::CCC::Store) - expect(Sourced::CCC.config.router).to be_a(Sourced::CCC::Router) - end - end - - describe 'CCC.register' do - let(:reactor_class) do - Class.new(Sourced::CCC::Projector::StateStored) do - def self.name = 'TestConfigReactor' - - consumer_group 'test-config-reactor' - partition_by :thing_id - - state { |_| {} } - end - end - - it 'triggers setup and delegates to router.register' do - Sourced::CCC.register(reactor_class) - - expect(Sourced::CCC.router.reactors).to include(reactor_class) - end - end - - describe 'CCC.store' do - it 'triggers setup and returns the store' do - store = Sourced::CCC.store - expect(store).to be_a(Sourced::CCC::Store) - expect(store.installed?).to be true - end - end - - describe 'CCC.router' do - it 'triggers setup and returns the router' do - router = Sourced::CCC.router - expect(router).to be_a(Sourced::CCC::Router) - expect(router.store).to be(Sourced::CCC.store) - end - end - - describe 'CCC.setup!' do - let(:reactor_class) do - Class.new(Sourced::CCC::Projector::StateStored) do - def self.name = 'SetupTestReactor' - - consumer_group 'setup-test-reactor' - partition_by :thing_id - - state { |_| {} } - end - end - - it 'replays the configure block on a fresh Configuration' do - call_count = 0 - Sourced::CCC.configure do |c| - call_count += 1 - c.worker_count = 8 - end - - expect(call_count).to eq(1) - original_config = Sourced::CCC.config - - Sourced::CCC.setup! - - expect(call_count).to eq(2) - expect(Sourced::CCC.config).not_to be(original_config) - expect(Sourced::CCC.config.worker_count).to eq(8) - expect(Sourced::CCC.config).to be_frozen - end - - it 'creates a new store connection on each call' do - Sourced::CCC.configure {} - store1 = Sourced::CCC.config.store - - Sourced::CCC.setup! - store2 = Sourced::CCC.config.store - - expect(store2).not_to be(store1) - end - - it 'works without a configure block' do - Sourced::CCC.setup! - - expect(Sourced::CCC.config.store).to be_a(Sourced::CCC::Store) - expect(Sourced::CCC.config.router).to be_a(Sourced::CCC::Router) - expect(Sourced::CCC.config).to be_frozen - end - end - - describe 'CCC.reset!' do - it 'clears the singleton config' do - original = Sourced::CCC.config - Sourced::CCC.reset! - expect(Sourced::CCC.config).not_to be(original) - end - - it 'clears the stored configure block' do - Sourced::CCC.configure do |c| - c.worker_count = 8 - end - - Sourced::CCC.reset! - Sourced::CCC.setup! - - expect(Sourced::CCC.config.worker_count).to eq(2) - end - end - - describe '#store=' do - it 'accepts a CCC::Store instance directly' do - db = Sequel.sqlite - store = Sourced::CCC::Store.new(db) - - config = described_class.new - config.store = store - expect(config.store).to be(store) - end - - it 'wraps a Sequel::SQLite::Database in a Store' do - db = Sequel.sqlite - - config = described_class.new - config.store = db - expect(config.store).to be_a(Sourced::CCC::Store) - expect(config.store.db).to be(db) - end - - it 'accepts any object implementing StoreInterface' do - fake_store = double('CustomStore', - installed?: true, install!: nil, append: nil, read: nil, - read_partition: nil, claim_next: nil, ack: nil, release: nil, - register_consumer_group: nil, worker_heartbeat: nil, - release_stale_claims: nil, notifier: nil - ) - - config = described_class.new - config.store = fake_store - expect(config.store).to be(fake_store) - end - - it 'raises for objects not implementing StoreInterface' do - config = described_class.new - expect { config.store = Object.new }.to raise_error(Plumb::ParseError) - end - end - - describe '#error_strategy' do - it 'falls through to Sourced.config.error_strategy by default' do - config = described_class.new - expect(config.error_strategy).to eq(Sourced.config.error_strategy) - end - - it 'can be overridden with a custom callable' do - custom = ->(_e, _m, _g) {} - config = described_class.new - config.error_strategy = custom - expect(config.error_strategy).to be(custom) - end - - it 'raises if assigned a non-callable' do - config = described_class.new - expect { config.error_strategy = 'not callable' }.to raise_error(ArgumentError) - end - end - - describe '#setup!' do - it 'is idempotent' do - config = described_class.new - config.setup! - store1 = config.store - router1 = config.router - config.setup! - expect(config.store).to be(store1) - expect(config.router).to be(router1) - end - - it 'defaults to in-memory SQLite store when none configured' do - config = described_class.new - config.setup! - expect(config.store).to be_a(Sourced::CCC::Store) - expect(config.store.installed?).to be true - end - - it 'uses configured store when set' do - db = Sequel.sqlite - store = Sourced::CCC::Store.new(db) - store.install! - - config = described_class.new - config.store = store - config.setup! - expect(config.store).to be(store) - end - end - - describe 'CCC.load with global store' do - let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } - - let(:decider_class) do - Class.new(Sourced::CCC::Decider) do - def self.name = 'ConfigLoadDecider' - - partition_by :thing_id - consumer_group 'config-load-decider' - - state { |_| { count: 0 } } - end - end - - before do - store.install! - Sourced::CCC.configure do |c| - c.store = store - end - end - - it 'uses global store when store: not provided' do - instance, read_result = Sourced::CCC.load(decider_class, thing_id: 'abc') - expect(instance.state[:count]).to eq(0) - expect(read_result.messages).to be_empty - end - - it 'uses override store when store: provided' do - other_db = Sequel.sqlite - other_store = Sourced::CCC::Store.new(other_db) - other_store.install! - - instance, read_result = Sourced::CCC.load(decider_class, store: other_store, thing_id: 'abc') - expect(instance.state[:count]).to eq(0) - expect(read_result.messages).to be_empty - end - end -end diff --git a/spec/sourced/ccc/dispatcher_spec.rb b/spec/sourced/ccc/dispatcher_spec.rb deleted file mode 100644 index 0cd010ed..00000000 --- a/spec/sourced/ccc/dispatcher_spec.rb +++ /dev/null @@ -1,375 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' -require 'sequel' - -module CCCDispatcherTestMessages - DeviceRegistered = Sourced::CCC::Message.define('dispatch_test.device.registered') do - attribute :device_id, String - attribute :name, String - end - - DeviceBound = Sourced::CCC::Message.define('dispatch_test.device.bound') do - attribute :device_id, String - attribute :asset_id, String - end - - BindDevice = Sourced::CCC::Message.define('dispatch_test.bind_device') do - attribute :device_id, String - attribute :asset_id, String - end - - DelayedNotify = Sourced::CCC::Message.define('dispatch_test.delayed_notify') do - attribute :device_id, String - end -end - -class DispatchTestDecider < Sourced::CCC::Decider - partition_by :device_id - consumer_group 'dispatch-test-decider' - - state { |_| { exists: false, bound: false } } - - evolve CCCDispatcherTestMessages::DeviceRegistered do |state, _evt| - state[:exists] = true - end - - evolve CCCDispatcherTestMessages::DeviceBound do |state, _evt| - state[:bound] = true - end - - command CCCDispatcherTestMessages::BindDevice do |state, cmd| - raise 'Not found' unless state[:exists] - raise 'Already bound' if state[:bound] - event CCCDispatcherTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id - end - - reaction CCCDispatcherTestMessages::DeviceBound do |_state, evt| - dispatch(CCCDispatcherTestMessages::DelayedNotify, device_id: evt.payload.device_id) - .at(Time.now + 2) - end -end - -class DispatchTestProjector < Sourced::CCC::Projector::StateStored - partition_by :device_id - consumer_group 'dispatch-test-projector' - - state { |_| { devices: [] } } - - evolve CCCDispatcherTestMessages::DeviceRegistered do |state, evt| - state[:devices] << evt.payload.name - end - - evolve CCCDispatcherTestMessages::DeviceBound do |state, _evt| - # noop - end - - sync do |state:, messages:, replaying:| - state[:synced] = true - end -end - -RSpec.describe Sourced::CCC::Dispatcher do - let(:db) { Sequel.sqlite } - let(:notifier) { Sourced::InlineNotifier.new } - let(:store) { Sourced::CCC::Store.new(db, notifier: notifier) } - let(:router) { Sourced::CCC::Router.new(store: store) } - let(:logger) { instance_double('Logger', info: nil, warn: nil, debug: nil) } - let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } - - before do - store.install! - router.register(DispatchTestDecider) - router.register(DispatchTestProjector) - end - - describe 'Store notifications' do - it 'append triggers notify_new_messages' do - expect(notifier).to receive(:notify_new_messages).with(['dispatch_test.device.registered']) - - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - end - - it 'start_consumer_group triggers notify_reactor_resumed' do - store.stop_consumer_group('dispatch-test-decider') - - expect(notifier).to receive(:notify_reactor_resumed).with('dispatch-test-decider') - - store.start_consumer_group('dispatch-test-decider') - end - - it 'empty append does not notify' do - expect(notifier).not_to receive(:notify_new_messages) - - store.append([]) - end - end - - describe 'batch_size' do - it 'claim_next with batch_size limits returned messages' do - # Append 5 messages for the same partition - 5.times do |i| - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: "Sensor #{i}" }) - ) - end - - claim = store.claim_next( - 'dispatch-test-projector', - partition_by: ['device_id'], - handled_types: DispatchTestProjector.handled_messages.map(&:type), - worker_id: 'w1', - batch_size: 2 - ) - - expect(claim).not_to be_nil - expect(claim.messages.size).to eq(2) - end - - it 'claim_next without batch_size returns all messages' do - 5.times do |i| - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: "Sensor #{i}" }) - ) - end - - claim = store.claim_next( - 'dispatch-test-projector', - partition_by: ['device_id'], - handled_types: DispatchTestProjector.handled_messages.map(&:type), - worker_id: 'w1' - ) - - expect(claim).not_to be_nil - expect(claim.messages.size).to eq(5) - end - end - - describe Sourced::CCC::Dispatcher::NotificationQueuer do - let(:queuer) do - described_class.new( - work_queue: work_queue, - reactors: [DispatchTestDecider, DispatchTestProjector] - ) - end - - it 'maps message types to interested reactors' do - # DeviceRegistered is handled by projector (via evolve), - # BindDevice is handled by decider (via command) - queuer.call('messages_appended', 'dispatch_test.device.registered,dispatch_test.bind_device') - - popped = [] - popped << work_queue.pop - popped << work_queue.pop - - expect(popped).to contain_exactly(DispatchTestDecider, DispatchTestProjector) - end - - it 'maps group_id to reactor for reactor_resumed' do - queuer.call('reactor_resumed', 'dispatch-test-decider') - - popped = work_queue.pop - expect(popped).to eq(DispatchTestDecider) - end - - it 'ignores unknown message types' do - queuer.call('messages_appended', 'unknown.type') - - # Queue should be empty — push a sentinel to avoid blocking - work_queue.push(nil) - expect(work_queue.pop).to be_nil - end - - it 'ignores unknown group_ids' do - queuer.call('reactor_resumed', 'unknown-group') - - work_queue.push(nil) - expect(work_queue.pop).to be_nil - end - end - - describe Sourced::CCC::Worker do - let(:worker) do - described_class.new( - work_queue: work_queue, - router: router, - name: 'test-worker', - batch_size: 50, - max_drain_rounds: 10, - logger: logger - ) - end - - describe '#tick' do - it 'processes one claim for a reactor' do - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - - result = worker.tick(DispatchTestProjector) - expect(result).to be true - end - - it 'returns false when no work available' do - result = worker.tick(DispatchTestDecider) - expect(result).to be false - end - end - - describe '#drain' do - it 'processes until no more work for reactor' do - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor A' }) - ) - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor B' }) - ) - - worker.instance_variable_set(:@running, true) - worker.drain(DispatchTestProjector) - - # Both partitions should have been processed — no more work - result = worker.tick(DispatchTestProjector) - expect(result).to be false - end - - it 're-enqueues reactor when max_drain_rounds reached' do - # Create a worker with max_drain_rounds: 1 - bounded_worker = described_class.new( - work_queue: work_queue, - router: router, - name: 'bounded', - batch_size: 50, - max_drain_rounds: 1, - logger: logger - ) - - # Append messages for 2 partitions - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'A' }) - ) - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'B' }) - ) - - bounded_worker.instance_variable_set(:@running, true) - bounded_worker.drain(DispatchTestProjector) - - # Should have re-enqueued — pop it back - popped = work_queue.pop - expect(popped).to eq(DispatchTestProjector) - end - end - end - - describe 'Dispatcher wiring' do - subject(:dispatcher) do - described_class.new( - router: router, - worker_count: 2, - batch_size: 50, - max_drain_rounds: 10, - catchup_interval: 5, - work_queue: work_queue, - logger: logger - ) - end - - it 'creates the requested number of workers' do - expect(dispatcher.workers.size).to eq(2) - end - - it 'creates workers with correct names' do - names = dispatcher.workers.map(&:name) - expect(names).to include(match(/worker-0$/)) - expect(names).to include(match(/worker-1$/)) - end - - it 'spawns via #spawn when task responds to spawn' do - task = double('Task') - # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns - expect(task).to receive(:spawn).exactly(6).times - dispatcher.spawn_into(task) - end - - it 'spawns via #async when task does not respond to spawn' do - task = Object.new - def task.async; end - expect(task).to receive(:async).exactly(6).times - dispatcher.spawn_into(task) - end - - it '#stop stops all components' do - dispatcher.stop - - dispatcher.workers.each do |w| - expect(w.instance_variable_get(:@running)).to eq(false) - end - end - - it 'creates zero workers when worker_count is 0' do - d = described_class.new( - router: router, - worker_count: 0, - logger: logger - ) - expect(d.workers).to be_empty - end - end - - describe 'Integration: append → notify → queue → worker' do - it 'InlineNotifier fires synchronously through the full pipeline' do - # Build dispatcher which subscribes NotificationQueuer to the store's notifier - dispatcher = described_class.new( - router: router, - worker_count: 1, - batch_size: 50, - max_drain_rounds: 10, - catchup_interval: 60, # long interval — we test synchronous path only - work_queue: work_queue, - logger: logger - ) - - # Append triggers notifier → NotificationQueuer → WorkQueue - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - - # Pop from queue — should have the reactors that handle this type - popped = work_queue.pop - expect([DispatchTestDecider, DispatchTestProjector]).to include(popped) - - # Worker processes the message - worker = dispatcher.workers.first - result = worker.tick(popped) - expect(result).to be true - - dispatcher.stop - end - end - - describe 'scheduled message promotion' do - it 'promotes delayed reactions into the main log when due' do - store.append( - CCCDispatcherTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCDispatcherTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - expect(router.handle_next_for(DispatchTestDecider)).to be true - expect(db[:sourced_scheduled_messages].count).to eq(1) - - Timecop.freeze(Time.now + 3) do - expect(store.update_schedule!).to eq(1) - end - - conds = CCCDispatcherTestMessages::DelayedNotify.to_conditions(device_id: 'd1') - result = store.read(conds) - expect(result.messages.map(&:class)).to include(CCCDispatcherTestMessages::DelayedNotify) - end - end -end diff --git a/spec/sourced/ccc/durable_workflow_spec.rb b/spec/sourced/ccc/durable_workflow_spec.rb deleted file mode 100644 index 22524b97..00000000 --- a/spec/sourced/ccc/durable_workflow_spec.rb +++ /dev/null @@ -1,352 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' -require 'sourced/ccc/store' -require 'sourced/ccc/testing/rspec' -require 'sequel' - -module CCCDurableTests - FilledStringArray = Sourced::Types::Array[String].with(size: 1..) - - class IPResolver - def self.resolve = '11.111.111' - end - - class Geolocator - def self.locate(_ip) = 'London, UK' - end - - class Task < Sourced::CCC::DurableWorkflow - def execute(name) - ip = get_ip - location = geolocate(ip) - "Hello #{name}, your IP is #{ip} and its location is #{location}" - end - - durable def get_ip - IPResolver.resolve - end - - durable def geolocate(ip) - Geolocator.locate(ip) - end - end - - class AnotherTask < Sourced::CCC::DurableWorkflow - end - - class Doubler - def self.double(num) = num * 2 - end - - class MultiArgTask < Sourced::CCC::DurableWorkflow - def execute - double(2) + double(4) + double(2) - end - - durable def double(num) - Doubler.double(num) - end - end - - class Retryable < Sourced::CCC::DurableWorkflow - def execute - compute - end - - def compute - raise 'nope' - end - - durable :compute, retries: 2 - end - - class WithContext < Sourced::CCC::DurableWorkflow - context do - { index: 0, results: [] } - end - - def execute - @numbers = [1, 2, 3, 4, 5] - iterate - context[:results] - end - - durable def iterate - index = context[:index] - @numbers[index..].each do |n| - raise 'oopsie!' if n == 3 && index == 0 - - context[:index] += 1 - context[:results] << n * 2 - end - end - end - - class WithDelay < Sourced::CCC::DurableWorkflow - def execute - name = get_name - wait 10 - notify(name) - end - - durable def get_name - 'Joe' - end - - durable def notify(_name) - true - end - end -end - -RSpec.describe Sourced::CCC::DurableWorkflow do - include Sourced::CCC::Testing::RSpec - - let(:workflow_id) { 'durable-test-1' } - let(:name) { 'Joe' } - - def make_started(klass, args: [], workflow_id: 'durable-test-1') - klass::WorkflowStarted.new(payload: { workflow_id:, args: }) - end - - context 'with happy path' do - it 'starts and produces new messages until completing workflow' do - started = make_started(CCCDurableTests::Task, args: [name]) - history = [started] - - until history.last.is_a?(CCCDurableTests::Task::WorkflowComplete) - next_action = CCCDurableTests::Task.handle(history.last, history:) - expect(next_action).to be_a Sourced::CCC::Actions::Append - history += next_action.messages - end - - expect(history.map(&:class)).to eq([ - CCCDurableTests::Task::WorkflowStarted, - CCCDurableTests::Task::StepStarted, - CCCDurableTests::Task::StepComplete, - CCCDurableTests::Task::StepStarted, - CCCDurableTests::Task::StepComplete, - CCCDurableTests::Task::WorkflowComplete - ]) - - last = history.last - expect(last.payload.output).to eq('Hello Joe, your IP is 11.111.111 and its location is London, UK') - expect(last.payload.workflow_id).to eq(workflow_id) - end - end - - context 'with failed steps' do - it 'produces StepFailed message' do - expect(CCCDurableTests::IPResolver).to receive(:resolve).and_raise('Network Error!') - - started = make_started(CCCDurableTests::Task, args: [name]) - history = [started] - - until history.last.is_a?(CCCDurableTests::Task::StepFailed) - next_action = CCCDurableTests::Task.handle(history.last, history:) - expect(next_action).to be_a Sourced::CCC::Actions::Append - history += next_action.messages - end - - expect(history.map(&:class)).to eq([ - CCCDurableTests::Task::WorkflowStarted, - CCCDurableTests::Task::StepStarted, - CCCDurableTests::Task::StepFailed - ]) - expect(history.last.payload.error_class).to eq('RuntimeError') - expect(CCCDurableTests::FilledStringArray).to be === history.last.payload.backtrace - end - end - - context 'with previously successful step' do - it 'does not invoke step again, using cached result instead' do - get_ip_key = Sourced::CCC::DurableWorkflow.step_key(:get_ip, []) - geolocate_key = Sourced::CCC::DurableWorkflow.step_key(:geolocate, ['11.111.111']) - - expect(CCCDurableTests::IPResolver).not_to receive(:resolve) - expect(CCCDurableTests::Geolocator).to receive(:locate).with('11.111.111').and_return('Santiago, Chile') - - with_reactor(CCCDurableTests::Task, workflow_id: workflow_id) - .given(CCCDurableTests::Task::WorkflowStarted, workflow_id:, args: [name]) - .given(CCCDurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) - .given(CCCDurableTests::Task::StepComplete, workflow_id:, key: get_ip_key, step_name: :get_ip, output: '11.111.111') - .when(CCCDurableTests::Task::StepStarted, workflow_id:, key: geolocate_key, step_name: :geolocate, args: ['11.111.111']) - .then( - CCCDurableTests::Task::StepComplete.new(payload: { - workflow_id:, key: geolocate_key, step_name: :geolocate, output: 'Santiago, Chile' - }) - ) - end - end - - context 'when workflow is finally failed' do - it 'does not try again' do - get_ip_key = Sourced::CCC::DurableWorkflow.step_key(:get_ip, []) - - with_reactor(CCCDurableTests::Task, workflow_id: workflow_id) - .given(CCCDurableTests::Task::WorkflowStarted, workflow_id:, args: [name]) - .given(CCCDurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) - .given(CCCDurableTests::Task::StepFailed, workflow_id:, key: get_ip_key, step_name: :get_ip, error_class: 'NetworkError', error_message: 'foo', backtrace: []) - .given(CCCDurableTests::Task::WorkflowFailed, workflow_id:) - .when(CCCDurableTests::Task::StepStarted, workflow_id:, key: get_ip_key, step_name: :get_ip, args: []) - .then(Sourced::CCC::Testing::RSpec::NONE) - end - end - - context 'with a different workflow handling irrelevant messages' do - it 'blows up' do - with_reactor(CCCDurableTests::Task, workflow_id: workflow_id) - .when(CCCDurableTests::AnotherTask::WorkflowStarted, workflow_id:, args: [name]) - .then(Sourced::CCC::DurableWorkflow::UnknownMessageError) - end - end - - describe 'caching method calls by signature' do - it 'only invokes methods with the same arguments once per workflow' do - started = make_started(CCCDurableTests::MultiArgTask) - history = [started] - - allow(CCCDurableTests::Doubler).to receive(:double).and_call_original - - until history.last.is_a?(CCCDurableTests::MultiArgTask::WorkflowComplete) - next_action = CCCDurableTests::MultiArgTask.handle(history.last, history:) - expect(next_action).to be_a Sourced::CCC::Actions::Append - history += next_action.messages - end - - expect(history.last.payload.output).to eq(16) - expect(CCCDurableTests::Doubler).to have_received(:double).with(2).once - expect(CCCDurableTests::Doubler).to have_received(:double).with(4).once - end - end - - describe 'limited retries' do - it 'retries the configured number of times until it fails the workflow' do - started = make_started(CCCDurableTests::Retryable) - history = [started] - - 6.times do - next_action = CCCDurableTests::Retryable.handle(history.last, history:) - history += next_action.messages if next_action.respond_to?(:messages) - end - - task = CCCDurableTests::Retryable.from(history) - expect(task.status).to eq(:failed) - - expect(history.map(&:class)).to eq([ - CCCDurableTests::Retryable::WorkflowStarted, - CCCDurableTests::Retryable::StepStarted, - CCCDurableTests::Retryable::StepFailed, - CCCDurableTests::Retryable::StepStarted, - CCCDurableTests::Retryable::StepFailed, - CCCDurableTests::Retryable::WorkflowFailed - ]) - end - end - - context 'with context preserved across failures' do - it 'tracks context changes in event history' do - started = make_started(CCCDurableTests::WithContext) - history = [started] - - until history.last.is_a?(CCCDurableTests::WithContext::WorkflowComplete) - next_action = CCCDurableTests::WithContext.handle(history.last, history:) - history += next_action.messages - end - - task = CCCDurableTests::WithContext.from(history) - expect(task.output).to eq([2, 4, 6, 8, 10]) - - expect(history.map(&:class)).to eq([ - CCCDurableTests::WithContext::WorkflowStarted, - CCCDurableTests::WithContext::StepStarted, - CCCDurableTests::WithContext::StepFailed, - CCCDurableTests::WithContext::ContextUpdated, - CCCDurableTests::WithContext::StepStarted, - CCCDurableTests::WithContext::StepComplete, - CCCDurableTests::WithContext::ContextUpdated, - CCCDurableTests::WithContext::WorkflowComplete - ]) - - ctx_events = history.select { |m| m.is_a?(CCCDurableTests::WithContext::ContextUpdated) } - expect(ctx_events[0].payload.context).to eq(index: 2, results: [2, 4]) - expect(ctx_events[1].payload.context).to eq(index: 5, results: [2, 4, 6, 8, 10]) - end - end - - describe '#wait' do - it 'schedules a WaitStarted and WaitEnded combo' do - now = Time.now - - Timecop.freeze(now) do - started = make_started(CCCDurableTests::WithDelay) - history = [started] - - next_action = CCCDurableTests::WithDelay.handle(history.last, history:) - history += next_action.messages # StepStarted - - next_action = CCCDurableTests::WithDelay.handle(history.last, history:) - history += next_action.messages # StepComplete - - next_action = CCCDurableTests::WithDelay.handle(history.last, history:) - history += next_action.messages # WaitStarted - - expect(history.last).to be_a(CCCDurableTests::WithDelay::WaitStarted) - expect(history.last.payload.at).to eq(now + 10) - - next_action = CCCDurableTests::WithDelay.handle(history.last, history:) - expect(next_action).to be_a(Sourced::CCC::Actions::Schedule) - expect(next_action.at).to eq(now + 10) - expect(next_action.messages.first).to be_a(CCCDurableTests::WithDelay::WaitEnded) - expect(next_action.messages.first.payload.workflow_id).to eq('durable-test-1') - - history << next_action.messages.first - - until history.last.is_a?(CCCDurableTests::WithDelay::WorkflowComplete) - next_action = CCCDurableTests::WithDelay.handle(history.last, history:) - history += next_action.messages - end - - expect(history.map(&:class)).to eq([ - CCCDurableTests::WithDelay::WorkflowStarted, - CCCDurableTests::WithDelay::StepStarted, - CCCDurableTests::WithDelay::StepComplete, - CCCDurableTests::WithDelay::WaitStarted, - CCCDurableTests::WithDelay::WaitEnded, - CCCDurableTests::WithDelay::StepStarted, - CCCDurableTests::WithDelay::StepComplete, - CCCDurableTests::WithDelay::WorkflowComplete - ]) - end - end - end - - describe 'end-to-end via store + router' do - let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } - let(:router) { Sourced::CCC::Router.new(store:) } - - before { store.install! } - - it 'drains two concurrent workflows to completion' do - router.register(CCCDurableTests::Task) - - wf1_id = "wf-#{SecureRandom.uuid}" - wf2_id = "wf-#{SecureRandom.uuid}" - store.append([CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id: wf1_id, args: ['Alice'] })]) - store.append([CCCDurableTests::Task::WorkflowStarted.new(payload: { workflow_id: wf2_id, args: ['Bob'] })]) - - router.drain - - wf1, = Sourced::CCC.load(CCCDurableTests::Task, store:, workflow_id: wf1_id) - wf2, = Sourced::CCC.load(CCCDurableTests::Task, store:, workflow_id: wf2_id) - - expect(wf1.status).to eq(:complete) - expect(wf1.output).to include('Alice') - expect(wf2.status).to eq(:complete) - expect(wf2.output).to include('Bob') - end - end -end diff --git a/spec/sourced/ccc/evolve_spec.rb b/spec/sourced/ccc/evolve_spec.rb deleted file mode 100644 index b9f6fe61..00000000 --- a/spec/sourced/ccc/evolve_spec.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -module CCCEvolveTestMessages - ItemAdded = Sourced::CCC::Message.define('evolve_test.item.added') do - attribute :item_id, String - attribute :name, String - end - - ItemRemoved = Sourced::CCC::Message.define('evolve_test.item.removed') do - attribute :item_id, String - end - - Unhandled = Sourced::CCC::Message.define('evolve_test.unhandled') do - attribute :foo, String - end -end - -RSpec.describe Sourced::CCC::Evolve do - let(:evolver_class) do - Class.new do - include Sourced::CCC::Evolve - - def initialize(partition_values = {}) - @partition_values = partition_values - end - - state do |partition_values| - { items: [], partition_values: partition_values } - end - - evolve CCCEvolveTestMessages::ItemAdded do |state, msg| - state[:items] << { id: msg.payload.item_id, name: msg.payload.name } - end - - evolve CCCEvolveTestMessages::ItemRemoved do |state, msg| - state[:items].reject! { |i| i[:id] == msg.payload.item_id } - end - end - end - - describe '.state' do - it 'initializes state with partition values hash' do - instance = evolver_class.new(key1: 'val1', key2: 'val2') - expect(instance.state[:partition_values]).to eq({ key1: 'val1', key2: 'val2' }) - end - end - - describe '#evolve' do - it 'applies registered handlers in order' do - instance = evolver_class.new - messages = [ - CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }), - CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i2', name: 'Banana' }), - CCCEvolveTestMessages::ItemRemoved.new(payload: { item_id: 'i1' }) - ] - - instance.evolve(messages) - - expect(instance.state[:items]).to eq([{ id: 'i2', name: 'Banana' }]) - end - - it 'skips unregistered message types' do - instance = evolver_class.new - messages = [ - CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }), - CCCEvolveTestMessages::Unhandled.new(payload: { foo: 'bar' }) - ] - - instance.evolve(messages) - - expect(instance.state[:items]).to eq([{ id: 'i1', name: 'Apple' }]) - end - end - - describe '.handled_messages_for_evolve' do - it 'tracks registered classes' do - expect(evolver_class.handled_messages_for_evolve).to contain_exactly( - CCCEvolveTestMessages::ItemAdded, - CCCEvolveTestMessages::ItemRemoved - ) - end - end - - describe 'inheritance' do - it 'subclass inherits evolve handlers' do - subclass = Class.new(evolver_class) - expect(subclass.handled_messages_for_evolve).to contain_exactly( - CCCEvolveTestMessages::ItemAdded, - CCCEvolveTestMessages::ItemRemoved - ) - - instance = subclass.new - instance.evolve([ - CCCEvolveTestMessages::ItemAdded.new(payload: { item_id: 'i1', name: 'Apple' }) - ]) - expect(instance.state[:items]).to eq([{ id: 'i1', name: 'Apple' }]) - end - end -end diff --git a/spec/sourced/ccc/message_spec.rb b/spec/sourced/ccc/message_spec.rb deleted file mode 100644 index 9930b49f..00000000 --- a/spec/sourced/ccc/message_spec.rb +++ /dev/null @@ -1,481 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -module CCCTestMessages - DeviceRegistered = Sourced::CCC::Message.define('device.registered') do - attribute :device_id, String - attribute :name, String - end - - AssetRegistered = Sourced::CCC::Message.define('asset.registered') do - attribute :asset_id, String - attribute :label, String - end - - SystemUpdated = Sourced::CCC::Message.define('system.updated') do - attribute :version, String - end - - OptionalFields = Sourced::CCC::Message.define('test.optional_fields') do - attribute? :required_field, String - attribute? :optional_field, String - end -end - -RSpec.describe Sourced::CCC::Message do - describe '.define' do - it 'creates a subclass with a type string' do - expect(CCCTestMessages::DeviceRegistered.type).to eq('device.registered') - end - - it 'creates a typed payload' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.payload.device_id).to eq('dev-1') - expect(msg.payload.name).to eq('Sensor A') - end - - it 'auto-generates an id' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.id).not_to be_nil - expect(msg.id).to match(/\A[0-9a-f-]{36}\z/) - end - - it 'sets created_at automatically' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.created_at).to be_a(Time) - end - - it 'sets type on the instance' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.type).to eq('device.registered') - end - - it 'defaults metadata to empty hash' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.metadata).to eq({}) - end - - it 'accepts metadata' do - msg = CCCTestMessages::DeviceRegistered.new( - payload: { device_id: 'dev-1', name: 'Sensor A' }, - metadata: { user_id: 42 } - ) - expect(msg.metadata[:user_id]).to eq(42) - end - - it 'defaults causation_id and correlation_id to id' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.causation_id).to eq(msg.id) - expect(msg.correlation_id).to eq(msg.id) - end - - it 'accepts explicit causation_id and correlation_id' do - msg = CCCTestMessages::DeviceRegistered.new( - payload: { device_id: 'dev-1', name: 'Sensor A' }, - causation_id: 'cause-1', - correlation_id: 'corr-1' - ) - expect(msg.causation_id).to eq('cause-1') - expect(msg.correlation_id).to eq('corr-1') - end - end - - describe '.from' do - it 'instantiates the correct subclass from a hash' do - msg = Sourced::CCC::Message.from(type: 'device.registered', payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg).to be_a(CCCTestMessages::DeviceRegistered) - expect(msg.payload.device_id).to eq('dev-1') - end - - it 'raises UnknownMessageError for unknown types' do - expect { - Sourced::CCC::Message.from(type: 'unknown.type', payload: {}) - }.to raise_error(Sourced::UnknownMessageError, /Unknown message type: unknown.type/) - end - end - - describe '.registry' do - it 'stores defined message types' do - expect(Sourced::CCC::Message.registry['device.registered']).to eq(CCCTestMessages::DeviceRegistered) - expect(Sourced::CCC::Message.registry['asset.registered']).to eq(CCCTestMessages::AssetRegistered) - end - - it 'is separate from Sourced::Message registry' do - expect(Sourced::CCC::Message.registry['device.registered']).not_to be_nil - expect(Sourced::Message.registry['device.registered']).to be_nil - end - end - - describe '#extracted_keys' do - it 'extracts all top-level payload attributes as string pairs' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - keys = msg.extracted_keys - expect(keys).to contain_exactly( - ['device_id', 'dev-1'], - ['name', 'Sensor A'] - ) - end - - it 'skips nil values' do - msg = CCCTestMessages::OptionalFields.new(payload: { required_field: 'present', optional_field: nil }) - keys = msg.extracted_keys - expect(keys).to eq([['required_field', 'present']]) - end - - it 'converts values to strings' do - msg = CCCTestMessages::SystemUpdated.new(payload: { version: 'v2.0.5' }) - keys = msg.extracted_keys - expect(keys).to eq([['version', 'v2.0.5']]) - end - - it 'returns empty array for messages without payload attributes' do - # Message base class with no payload definition - bare = Sourced::CCC::Message.define('test.bare') - msg = bare.new - expect(msg.extracted_keys).to eq([]) - end - end - - describe '.payload_attribute_names' do - it 'returns attribute names for a defined message class' do - expect(CCCTestMessages::DeviceRegistered.payload_attribute_names).to eq([:device_id, :name]) - end - - it 'returns empty array for a bare message class' do - bare = Sourced::CCC::Message.define('test.payload_attrs.bare') - expect(bare.payload_attribute_names).to eq([]) - end - end - - describe '.to_conditions' do - it 'returns one condition with only attributes the message class has' do - conditions = CCCTestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', asset_id: 'asset-1') - expect(conditions.size).to eq(1) - expect(conditions.first.message_type).to eq('device.registered') - expect(conditions.first.attrs).to eq({ device_id: 'dev-1' }) - end - - it 'includes all matching attributes in one condition' do - conditions = CCCTestMessages::DeviceRegistered.to_conditions(device_id: 'dev-1', name: 'Sensor A') - expect(conditions.size).to eq(1) - expect(conditions.first.attrs).to eq({ device_id: 'dev-1', name: 'Sensor A' }) - end - - it 'returns empty array when no attributes match' do - conditions = CCCTestMessages::DeviceRegistered.to_conditions(course_name: 'Algebra') - expect(conditions).to eq([]) - end - end - - describe '#correlate' do - it 'sets causation_id to source message id' do - source = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - target = CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) - - correlated = source.correlate(target) - expect(correlated.causation_id).to eq(source.id) - end - - it 'propagates correlation_id from source' do - source = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - target = CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) - - correlated = source.correlate(target) - expect(correlated.correlation_id).to eq(source.correlation_id) - end - - it 'preserves correlation_id through a chain' do - first = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - second = first.correlate(CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' })) - third = second.correlate(CCCTestMessages::SystemUpdated.new(payload: { version: 'v1' })) - - expect(third.causation_id).to eq(second.id) - expect(third.correlation_id).to eq(first.id) - end - - it 'merges metadata from both messages' do - source = CCCTestMessages::DeviceRegistered.new( - payload: { device_id: 'dev-1', name: 'Sensor A' }, - metadata: { user_id: 42 } - ) - target = CCCTestMessages::AssetRegistered.new( - payload: { asset_id: 'asset-1', label: 'Label' }, - metadata: { request_id: 'req-1' } - ) - - correlated = source.correlate(target) - expect(correlated.metadata).to eq({ user_id: 42, request_id: 'req-1' }) - end - - it 'returns a new instance without mutating the original' do - source = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - target = CCCTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Label' }) - - correlated = source.correlate(target) - expect(correlated).not_to equal(target) - expect(correlated).to be_a(CCCTestMessages::AssetRegistered) - expect(target.causation_id).to eq(target.id) # original unchanged - end - end - - describe '#with_metadata' do - it 'merges new metadata into existing metadata' do - msg = CCCTestMessages::DeviceRegistered.new( - payload: { device_id: 'dev-1', name: 'Sensor A' }, - metadata: { user_id: 42 } - ) - updated = msg.with_metadata(request_id: 'req-1') - expect(updated.metadata).to eq({ user_id: 42, request_id: 'req-1' }) - end - - it 'returns self when given empty hash' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - expect(msg.with_metadata({})).to equal(msg) - end - - it 'does not mutate the original message' do - msg = CCCTestMessages::DeviceRegistered.new( - payload: { device_id: 'dev-1', name: 'Sensor A' }, - metadata: { user_id: 42 } - ) - updated = msg.with_metadata(request_id: 'req-1') - expect(msg.metadata).to eq({ user_id: 42 }) - expect(updated).not_to equal(msg) - end - end - - describe '#with_payload' do - it 'merges new attributes into existing payload' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - updated = msg.with_payload(name: 'Sensor B') - expect(updated.payload.device_id).to eq('dev-1') - expect(updated.payload.name).to eq('Sensor B') - end - - it 'preserves id and other attributes' do - msg = CCCTestMessages::DeviceRegistered.new( - payload: { device_id: 'dev-1', name: 'Sensor A' }, - metadata: { user_id: 42 } - ) - updated = msg.with_payload(name: 'Sensor B') - expect(updated.id).to eq(msg.id) - expect(updated.metadata).to eq({ user_id: 42 }) - expect(updated.type).to eq('device.registered') - end - - it 'returns a new instance without mutating the original' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - updated = msg.with_payload(name: 'Sensor B') - expect(updated).not_to equal(msg) - expect(msg.payload.name).to eq('Sensor A') - end - - it 'works with empty hash (returns equivalent copy)' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - updated = msg.with_payload({}) - expect(updated.payload.device_id).to eq('dev-1') - expect(updated.payload.name).to eq('Sensor A') - expect(updated).not_to equal(msg) - end - end - - describe '#at' do - it 'returns new message with updated created_at' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - future = msg.created_at + 3600 - updated = msg.at(future) - expect(updated.created_at).to eq(future) - expect(updated).not_to equal(msg) - end - - it 'raises PastMessageDateError when given a past time' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - past = msg.created_at - 3600 - expect { msg.at(past) }.to raise_error(Sourced::PastMessageDateError) - end - - it 'does not mutate the original message' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - original_time = msg.created_at - msg.at(msg.created_at + 3600) - expect(msg.created_at).to eq(original_time) - end - end - - describe 'Registry' do - describe '#keys' do - it 'returns array of registered type strings' do - keys = Sourced::CCC::Message.registry.keys - expect(keys).to include('device.registered', 'asset.registered', 'system.updated') - end - end - - describe '#all' do - let!(:test_cmd) { Sourced::CCC::Command.define('test.reg_all_cmd') { attribute :name, String } } - let!(:test_evt) { Sourced::CCC::Event.define('test.reg_all_evt') { attribute :name, String } } - - it 'returns an Enumerator when no block given' do - expect(Sourced::CCC::Message.registry.all).to be_a(Enumerator) - end - - it 'includes classes from subclass registries' do - all = Sourced::CCC::Message.registry.all.to_a - expect(all).to include(test_cmd) - expect(all).to include(test_evt) - end - - it 'includes classes registered directly on Message' do - all = Sourced::CCC::Message.registry.all.to_a - expect(all).to include(CCCTestMessages::DeviceRegistered) - end - - it 'yields each class when block given' do - yielded = [] - Sourced::CCC::Message.registry.all { |c| yielded << c } - expect(yielded).to include(test_cmd, test_evt) - end - - it 'scoped to a subclass registry only includes that branch' do - cmd_all = Sourced::CCC::Command.registry.all.to_a - expect(cmd_all).to include(test_cmd) - expect(cmd_all).not_to include(test_evt) - end - end - end - - describe 'Payload' do - let(:msg) { CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) } - - describe '#[]' do - it 'returns attribute value by symbol key' do - expect(msg.payload[:device_id]).to eq('dev-1') - expect(msg.payload[:name]).to eq('Sensor A') - end - end - - describe '#fetch' do - it 'returns attribute value for existing key' do - expect(msg.payload.fetch(:device_id)).to eq('dev-1') - end - - it 'raises KeyError for missing key' do - expect { msg.payload.fetch(:missing) }.to raise_error(KeyError) - end - - it 'supports default value' do - expect(msg.payload.fetch(:missing, 'default')).to eq('default') - end - - it 'supports block fallback' do - expect(msg.payload.fetch(:missing) { 'from_block' }).to eq('from_block') - end - end - end - - describe '#initialize default payload' do - it 'creates message without explicit payload arg' do - bare = Sourced::CCC::Message.define('test.init_default') - msg = bare.new - expect(msg.payload).to be_nil - expect(msg.id).not_to be_nil - expect(msg.type).to eq('test.init_default') - end - end - - describe '#to_message' do - it 'returns self — identity implementation of the to_message contract' do - msg = CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - expect(msg.to_message).to equal(msg) - end - end - - describe '.===' do - let(:msg) { CCCTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) } - - it 'matches unwrapped instances like Module#===' do - expect(CCCTestMessages::DeviceRegistered === msg).to be true - expect(CCCTestMessages::AssetRegistered === msg).to be false - end - - it 'matches messages wrapped in a to_message-aware delegator' do - wrapper = Class.new(SimpleDelegator) do - def to_message = __getobj__ - end.new(msg) - - expect(CCCTestMessages::DeviceRegistered === wrapper).to be true - expect(CCCTestMessages::AssetRegistered === wrapper).to be false - end - - it 'makes case/when transparent across wrapped and unwrapped messages' do - classify = ->(m) do - case m - when CCCTestMessages::DeviceRegistered then :device - when CCCTestMessages::AssetRegistered then :asset - else :unknown - end - end - - wrapper = Sourced::CCC::PositionedMessage.new(msg, 1) - expect(classify.call(msg)).to eq(:device) - expect(classify.call(wrapper)).to eq(:device) - end - - it 'returns false for arbitrary non-message objects without looping' do - expect(CCCTestMessages::DeviceRegistered === 'string').to be false - expect(CCCTestMessages::DeviceRegistered === 42).to be false - expect(CCCTestMessages::DeviceRegistered === Object.new).to be false - end - end - - describe Sourced::CCC::ConsistencyGuard do - it 'is a Data struct with conditions and last_position' do - conditions = [Sourced::CCC::QueryCondition.new(message_type: 'device.registered', attrs: { device_id: 'dev-1' })] - guard = Sourced::CCC::ConsistencyGuard.new(conditions: conditions, last_position: 42) - expect(guard.conditions).to eq(conditions) - expect(guard.last_position).to eq(42) - end - end - - describe 'Command and Event subclass registries' do - let!(:test_cmd) { Sourced::CCC::Command.define('test.do_something') { attribute :name, String } } - let!(:test_evt) { Sourced::CCC::Event.define('test.something_happened') { attribute :name, String } } - - it 'registers Command types in Command registry' do - expect(Sourced::CCC::Command.registry['test.do_something']).to eq(test_cmd) - end - - it 'registers Event types in Event registry' do - expect(Sourced::CCC::Event.registry['test.something_happened']).to eq(test_evt) - end - - it 'does not register Command types in Event registry' do - expect(Sourced::CCC::Event.registry['test.do_something']).to be_nil - end - - it 'Message.registry can look up types from subclass registries' do - expect(Sourced::CCC::Message.registry['test.do_something']).to eq(test_cmd) - expect(Sourced::CCC::Message.registry['test.something_happened']).to eq(test_evt) - end - end - - describe Sourced::CCC::QueryCondition do - it 'is a Data struct with message_type and attrs hash' do - cond = Sourced::CCC::QueryCondition.new( - message_type: 'device.registered', - attrs: { device_id: 'dev-1' } - ) - expect(cond.message_type).to eq('device.registered') - expect(cond.attrs).to eq({ device_id: 'dev-1' }) - end - - it 'supports multiple attrs for compound conditions' do - cond = Sourced::CCC::QueryCondition.new( - message_type: 'seat.selected', - attrs: { showing_id: 'show-1', seat_id: 'C7' } - ) - expect(cond.attrs).to eq({ showing_id: 'show-1', seat_id: 'C7' }) - end - end -end diff --git a/spec/sourced/ccc/projector_spec.rb b/spec/sourced/ccc/projector_spec.rb deleted file mode 100644 index d56e4b75..00000000 --- a/spec/sourced/ccc/projector_spec.rb +++ /dev/null @@ -1,439 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -module CCCProjectorTestMessages - ItemAdded = Sourced::CCC::Message.define('projector_test.item.added') do - attribute :list_id, String - attribute :name, String - end - - ItemArchived = Sourced::CCC::Message.define('projector_test.item.archived') do - attribute :list_id, String - attribute :name, String - end - - NotifyArchive = Sourced::CCC::Message.define('projector_test.notify_archive') do - attribute :list_id, String - end - - DelayedNotifyArchive = Sourced::CCC::Message.define('projector_test.delayed_notify_archive') do - attribute :list_id, String - end -end - -class TestItemProjector < Sourced::CCC::Projector::StateStored - partition_by :list_id - consumer_group 'item-projector-test' - - state do |(list_id)| - { list_id: list_id, items: [], synced: false } - end - - evolve CCCProjectorTestMessages::ItemAdded do |state, msg| - state[:items] << msg.payload.name - end - - evolve CCCProjectorTestMessages::ItemArchived do |state, msg| - state[:items].delete(msg.payload.name) - end - - reaction CCCProjectorTestMessages::ItemArchived do |_state, msg| - CCCProjectorTestMessages::NotifyArchive.new(payload: { list_id: msg.payload.list_id }) - end - - sync do |state:, messages:, replaying:| - state[:synced] = true - state[:last_replaying] = replaying - end - - after_sync do |state:, messages:, replaying:| - state[:after_synced] = true - end -end - -class TestItemESProjector < Sourced::CCC::Projector::EventSourced - partition_by :list_id - consumer_group 'item-es-projector-test' - - state do |(list_id)| - { list_id: list_id, items: [], synced: false } - end - - evolve CCCProjectorTestMessages::ItemAdded do |state, msg| - state[:items] << msg.payload.name - end - - evolve CCCProjectorTestMessages::ItemArchived do |state, msg| - state[:items].delete(msg.payload.name) - end - - reaction CCCProjectorTestMessages::ItemArchived do |_state, msg| - CCCProjectorTestMessages::NotifyArchive.new(payload: { list_id: msg.payload.list_id }) - end - - sync do |state:, messages:, replaying:| - state[:synced] = true - state[:last_replaying] = replaying - end - - after_sync do |state:, messages:, replaying:| - state[:after_synced] = true - end -end - -class TestDelayedItemProjector < Sourced::CCC::Projector::StateStored - partition_by :list_id - consumer_group 'delayed-item-projector-test' - - state do |(list_id)| - { list_id: list_id, items: [] } - end - - evolve CCCProjectorTestMessages::ItemArchived do |state, msg| - state[:items].delete(msg.payload.name) - end - - reaction CCCProjectorTestMessages::ItemArchived do |_state, msg| - dispatch(CCCProjectorTestMessages::DelayedNotifyArchive, list_id: msg.payload.list_id) - .at(Time.now + 10) - end -end - -RSpec.describe Sourced::CCC::Projector do - describe '.handled_messages' do - it 'includes evolve and react types' do - msgs = TestItemProjector.handled_messages - expect(msgs).to include(CCCProjectorTestMessages::ItemAdded) - expect(msgs).to include(CCCProjectorTestMessages::ItemArchived) - end - end - - describe '.handle_batch (StateStored)' do - it 'evolves from new_messages and includes sync and after_sync actions' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 - ) - ] - - pairs = TestItemProjector.handle_batch(['L1'], msgs) - - sync_pair = pairs.last - sync_actions, source_msg = sync_pair - expect(source_msg).to eq(msgs.last) - - sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::Sync) } - expect(sync_action).not_to be_nil - - after_sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::AfterSync) } - expect(after_sync_action).not_to be_nil - end - - it 'runs reactions when not replaying' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - - pairs = TestItemProjector.handle_batch(['L1'], msgs) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions.size).to eq(1) - expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) - end - - it 'skips reactions when replaying' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - - pairs = TestItemProjector.handle_batch(['L1'], msgs, replaying: true) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions).to be_empty - end - end - - describe '.handle_batch (EventSourced)' do - let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 5) } - - def make_history(messages) - Sourced::CCC::ReadResult.new(messages: messages, guard: guard) - end - - it 'evolves from full history, not just new messages' do - history_msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 3 - ) - ] - new_msgs = [history_msgs.last] - history = make_history(history_msgs) - - pairs = TestItemESProjector.handle_batch(['L1'], new_msgs, history: history) - - sync_pair = pairs.last - _sync_actions, source_msg = sync_pair - expect(source_msg).to eq(new_msgs.last) - end - - it 'runs reactions only on new messages, not full history' do - history_msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Old' }), 1 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'New' }), 2 - ) - ] - new_msgs = [history_msgs.last] - history = make_history(history_msgs) - - pairs = TestItemESProjector.handle_batch(['L1'], new_msgs, history: history) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions.size).to eq(1) - end - - it 'skips reactions when replaying' do - history_msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - history = make_history(history_msgs) - - pairs = TestItemESProjector.handle_batch(['L1'], history_msgs, history: history, replaying: true) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions).to be_empty - end - end - - describe '.handle_claim' do - let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 2) } - - def make_claim(messages, replaying: false) - Sourced::CCC::ClaimResult.new( - offset_id: 1, key_pair_ids: [], partition_key: 'list_id:L1', - partition_value: { 'list_id' => 'L1' }, - messages: messages, replaying: replaying, guard: guard - ) - end - - it 'evolves from claim.messages and includes sync actions' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 - ) - ] - claim = make_claim(msgs) - - pairs = TestItemProjector.handle_claim(claim) - - # Last pair should contain sync actions - sync_pair = pairs.last - sync_actions, source_msg = sync_pair - expect(source_msg).to eq(msgs.last) - - sync_action = Array(sync_actions).find { |a| a.is_a?(Sourced::CCC::Actions::Sync) } - expect(sync_action).not_to be_nil - end - - it 'runs reactions when not replaying' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - claim = make_claim(msgs, replaying: false) - - pairs = TestItemProjector.handle_claim(claim) - - # Should have reaction pair + sync pair - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions.size).to eq(1) - expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) - end - - it 'returns schedule actions for delayed reactions' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - claim = make_claim(msgs, replaying: false) - - pairs = TestDelayedItemProjector.handle_claim(claim) - - schedule_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |action| action.is_a?(Sourced::CCC::Actions::Schedule) } - - expect(schedule_actions.size).to eq(1) - expect(schedule_actions.first.messages.first).to be_a(CCCProjectorTestMessages::DelayedNotifyArchive) - end - - it 'skips reactions when replaying' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - claim = make_claim(msgs, replaying: true) - - pairs = TestItemProjector.handle_claim(claim) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions).to be_empty - end - - it 'passes replaying to sync blocks' do - msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - claim = make_claim(msgs, replaying: true) - - pairs = TestItemProjector.handle_claim(claim) - - # Execute the sync action to verify replaying is passed through - sync_pair = pairs.last - sync_actions = Array(sync_pair.first).select { |a| a.is_a?(Sourced::CCC::Actions::Sync) } - expect(sync_actions).not_to be_empty - - # Call the sync to verify it runs - sync_actions.first.call - end - end - - describe 'EventSourced' do - let(:guard) { Sourced::CCC::ConsistencyGuard.new(conditions: [], last_position: 5) } - - def make_claim(messages, replaying: false) - Sourced::CCC::ClaimResult.new( - offset_id: 1, key_pair_ids: [], partition_key: 'list_id:L1', - partition_value: { 'list_id' => 'L1' }, - messages: messages, replaying: replaying, guard: guard - ) - end - - def make_history(messages) - Sourced::CCC::ReadResult.new(messages: messages, guard: guard) - end - - it 'evolves from full history, not just claim messages' do - history_msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemAdded.new(payload: { list_id: 'L1', name: 'Banana' }), 2 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 3 - ) - ] - # Claim only contains the latest message - claim_msgs = [history_msgs.last] - claim = make_claim(claim_msgs) - history = make_history(history_msgs) - - pairs = TestItemESProjector.handle_claim(claim, history: history) - - # Sync pair should be the last one, acked against claim's last message - sync_pair = pairs.last - _sync_actions, source_msg = sync_pair - expect(source_msg).to eq(claim_msgs.last) - end - - it 'runs reactions only on claim messages, not full history' do - history_msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Old' }), 1 - ), - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'New' }), 2 - ) - ] - # Only the second message is in the claim - claim_msgs = [history_msgs.last] - claim = make_claim(claim_msgs, replaying: false) - history = make_history(history_msgs) - - pairs = TestItemESProjector.handle_claim(claim, history: history) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - # Only 1 reaction (for the claim message), not 2 - expect(append_actions.size).to eq(1) - expect(append_actions.first.messages.first).to be_a(CCCProjectorTestMessages::NotifyArchive) - end - - it 'skips reactions when replaying' do - history_msgs = [ - Sourced::CCC::PositionedMessage.new( - CCCProjectorTestMessages::ItemArchived.new(payload: { list_id: 'L1', name: 'Apple' }), 1 - ) - ] - claim = make_claim(history_msgs, replaying: true) - history = make_history(history_msgs) - - pairs = TestItemESProjector.handle_claim(claim, history: history) - - append_actions = pairs.flat_map { |actions, _| Array(actions) } - .select { |a| a.is_a?(Sourced::CCC::Actions::Append) } - - expect(append_actions).to be_empty - end - - it 'is detected by Injector as needing history' do - needs = Sourced::Injector.resolve_args(TestItemESProjector, :handle_claim) - expect(needs).to include(:history) - end - - it 'StateStored is not detected as needing history' do - needs = Sourced::Injector.resolve_args(TestItemProjector, :handle_claim) - expect(needs).not_to include(:history) - end - end - - describe '.context_for' do - it 'builds conditions from partition_keys × handled_messages_for_evolve' do - conditions = TestItemProjector.context_for(list_id: 'L1') - types = conditions.map(&:message_type).sort - expect(types).to include('projector_test.item.added') - expect(types).to include('projector_test.item.archived') - end - end -end diff --git a/spec/sourced/ccc/react_spec.rb b/spec/sourced/ccc/react_spec.rb deleted file mode 100644 index 962c355c..00000000 --- a/spec/sourced/ccc/react_spec.rb +++ /dev/null @@ -1,218 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -module CCCReactTestMessages - SomethingHappened = Sourced::CCC::Message.define('react_test.something.happened') do - attribute :thing_id, String - end - - DoNext = Sourced::CCC::Message.define('react_test.do_next') do - attribute :thing_id, String - end - - Unhandled = Sourced::CCC::Message.define('react_test.unhandled') do - attribute :foo, String - end - - Wildcarded = Sourced::CCC::Message.define('react_test.wildcarded') do - attribute :thing_id, String - end - - DelayedCommand = Sourced::CCC::Message.define('react_test.delayed.command') do - attribute :thing_id, String - end - - MultiReaction = Sourced::CCC::Message.define('react_test.multi.reaction') do - attribute :thing_id, String - end - - AnotherMultiReaction = Sourced::CCC::Message.define('react_test.another.multi.reaction') do - attribute :thing_id, String - end -end - -RSpec.describe Sourced::CCC::React do - let(:reactor_class) do - Class.new do - include Sourced::CCC::React - extend Sourced::CCC::Consumer - - def state - {} - end - - reaction CCCReactTestMessages::SomethingHappened do |_state, msg| - CCCReactTestMessages::DoNext.new(payload: { thing_id: msg.payload.thing_id }) - end - end - end - - describe '#react' do - it 'returns raw messages (not correlated)' do - instance = reactor_class.new - msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) - - result = instance.react(msg) - - expect(result.size).to eq(1) - expect(result.first).to be_a(CCCReactTestMessages::DoNext) - expect(result.first.payload.thing_id).to eq('t1') - # Not correlated — causation_id is its own id - expect(result.first.causation_id).to eq(result.first.id) - end - - it 'accepts a single message or an array of messages' do - instance = reactor_class.new - msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) - - expect(instance.react([msg]).map(&:class)).to eq([CCCReactTestMessages::DoNext]) - end - - it 'returns empty array for unregistered types' do - instance = reactor_class.new - msg = CCCReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) - - result = instance.react(msg) - expect(result).to eq([]) - end - end - - describe '#reacts_to?' do - it 'returns true for registered types' do - instance = reactor_class.new - msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) - expect(instance.reacts_to?(msg)).to be true - end - - it 'returns false for unregistered types' do - instance = reactor_class.new - msg = CCCReactTestMessages::Unhandled.new(payload: { foo: 'bar' }) - expect(instance.reacts_to?(msg)).to be false - end - end - - describe '.handled_messages_for_react' do - it 'tracks registered classes' do - expect(reactor_class.handled_messages_for_react).to contain_exactly( - CCCReactTestMessages::SomethingHappened - ) - end - end - - describe 'inheritance' do - it 'subclass inherits reaction handlers' do - subclass = Class.new(reactor_class) - expect(subclass.handled_messages_for_react).to contain_exactly( - CCCReactTestMessages::SomethingHappened - ) - - instance = subclass.new - msg = CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) - result = instance.react(msg) - expect(result.size).to eq(1) - end - end - - describe 'dispatch DSL' do - let(:dsl_reactor_class) do - Class.new do - include Sourced::CCC::React - extend Sourced::CCC::Consumer - - consumer_group 'ccc-reactor' - - def state - { source: 'state' } - end - - def self.handled_messages_for_evolve - [CCCReactTestMessages::Wildcarded] - end - - reaction CCCReactTestMessages::SomethingHappened do |_state, msg| - dispatch(CCCReactTestMessages::DoNext, thing_id: msg.payload.thing_id) - end - - reaction :react_test_delayed_command do |_state, msg| - dispatch(:react_test_delayed_command, thing_id: msg.payload.thing_id) - .with_metadata(foo: 'bar') - .at(Time.now + 10) - end - - reaction CCCReactTestMessages::MultiReaction, CCCReactTestMessages::AnotherMultiReaction do |_state, msg| - dispatch(CCCReactTestMessages::DoNext, thing_id: msg.payload.thing_id) - end - - reaction do |state, msg| - dispatch(CCCReactTestMessages::DoNext, thing_id: "#{state[:source]}-#{msg.payload.thing_id}") - end - end - end - - it 'supports dispatch with correlation, producer metadata, metadata chaining, and delays' do - now = Time.now - Timecop.freeze(now) do - instance = dsl_reactor_class.new - source = CCCReactTestMessages::DelayedCommand.new(payload: { thing_id: 't1' }) - - result = instance.react(source) - - expect(result.map(&:class)).to eq([CCCReactTestMessages::DelayedCommand]) - expect(result.first.causation_id).to eq(source.id) - expect(result.first.correlation_id).to eq(source.correlation_id) - expect(result.first.metadata[:producer]).to eq('ccc-reactor') - expect(result.first.metadata[:foo]).to eq('bar') - expect(result.first.created_at).to eq(now + 10) - end - end - - it 'supports dispatching multiple messages from one reaction block' do - klass = Class.new do - include Sourced::CCC::React - extend Sourced::CCC::Consumer - - def state - {} - end - - reaction CCCReactTestMessages::SomethingHappened do |_state, msg| - dispatch(CCCReactTestMessages::DoNext, thing_id: msg.payload.thing_id) - dispatch(CCCReactTestMessages::DelayedCommand, thing_id: msg.payload.thing_id) - end - end - - result = klass.new.react( - CCCReactTestMessages::SomethingHappened.new(payload: { thing_id: 't1' }) - ) - - expect(result.map(&:class)).to eq([ - CCCReactTestMessages::DoNext, - CCCReactTestMessages::DelayedCommand - ]) - end - - it 'supports wildcard reactions for evolve types without explicit handlers' do - result = dsl_reactor_class.new.react( - CCCReactTestMessages::Wildcarded.new(payload: { thing_id: 't1' }) - ) - - expect(result.map(&:class)).to eq([CCCReactTestMessages::DoNext]) - expect(result.first.payload.thing_id).to eq('state-t1') - expect(dsl_reactor_class.catch_all_react_events).to eq(Set[ - CCCReactTestMessages::Wildcarded - ]) - end - - it 'supports reactions registered for multiple message classes' do - instance = dsl_reactor_class.new - - first = instance.react(CCCReactTestMessages::MultiReaction.new(payload: { thing_id: 't1' })) - second = instance.react(CCCReactTestMessages::AnotherMultiReaction.new(payload: { thing_id: 't2' })) - - expect(first.map(&:class)).to eq([CCCReactTestMessages::DoNext]) - expect(second.map(&:class)).to eq([CCCReactTestMessages::DoNext]) - end - end -end diff --git a/spec/sourced/ccc/router_spec.rb b/spec/sourced/ccc/router_spec.rb deleted file mode 100644 index a3d48be4..00000000 --- a/spec/sourced/ccc/router_spec.rb +++ /dev/null @@ -1,557 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' -require 'sourced/ccc/store' -require 'sequel' - -module CCCRouterTestMessages - DeviceRegistered = Sourced::CCC::Message.define('router_test.device.registered') do - attribute :device_id, String - attribute :name, String - end - - DeviceBound = Sourced::CCC::Message.define('router_test.device.bound') do - attribute :device_id, String - attribute :asset_id, String - end - - BindDevice = Sourced::CCC::Message.define('router_test.bind_device') do - attribute :device_id, String - attribute :asset_id, String - end - - NotifyBound = Sourced::CCC::Message.define('router_test.notify_bound') do - attribute :device_id, String - end - - # Projector messages - DeviceListed = Sourced::CCC::Message.define('router_test.device.listed') do - attribute :device_id, String - end - - # Simple reactor messages - DeviceAudited = Sourced::CCC::Message.define('router_test.device.audited') do - attribute :device_id, String - attribute :event_type, String - end -end - -# Test decider for router specs -class RouterTestDecider < Sourced::CCC::Decider - partition_by :device_id - consumer_group 'router-test-decider' - - state { |_| { exists: false, bound: false } } - - evolve CCCRouterTestMessages::DeviceRegistered do |state, _evt| - state[:exists] = true - end - - evolve CCCRouterTestMessages::DeviceBound do |state, _evt| - state[:bound] = true - end - - command CCCRouterTestMessages::BindDevice do |state, cmd| - raise 'Not found' unless state[:exists] - raise 'Already bound' if state[:bound] - event CCCRouterTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id - end - - reaction CCCRouterTestMessages::DeviceBound do |_state, evt| - CCCRouterTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) - end -end - -# Test projector for router specs -class RouterTestProjector < Sourced::CCC::Projector::StateStored - partition_by :device_id - consumer_group 'router-test-projector' - - state { |_| { devices: [] } } - - evolve CCCRouterTestMessages::DeviceRegistered do |state, evt| - state[:devices] << evt.payload.name - end - - evolve CCCRouterTestMessages::DeviceBound do |state, _evt| - # nothing - end - - sync do |state:, messages:, replaying:| - # In a real projector, this would persist to DB - state[:synced] = true - end -end - -# Simple reactor: just extends Consumer, defines handled_messages, implements handle_claim. -# Logs an audit trail message for every DeviceRegistered or DeviceBound it sees. -class RouterTestAuditReactor - extend Sourced::CCC::Consumer - - partition_by :device_id - consumer_group 'router-test-audit' - - def self.handled_messages - [CCCRouterTestMessages::DeviceRegistered, CCCRouterTestMessages::DeviceBound] - end - - def self.handle_claim(claim) - each_with_partial_ack(claim.messages) do |msg| - audit = CCCRouterTestMessages::DeviceAudited.new( - payload: { device_id: msg.payload.device_id, event_type: msg.type } - ) - [Sourced::CCC::Actions::Append.new(audit), msg] - end - end -end - -RSpec.describe Sourced::CCC::Router do - let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } - let(:router) { Sourced::CCC::Router.new(store: store) } - - before do - store.install! - end - - describe '#register' do - it 'creates consumer group and introspects handle_claim signature' do - router.register(RouterTestDecider) - - expect(store.consumer_group_active?('router-test-decider')).to be true - expect(router.reactors).to include(RouterTestDecider) - end - - it 'detects history: for decider, none for projector' do - router.register(RouterTestDecider) - router.register(RouterTestProjector) - - # Decider needs history, projector does not - expect(router.instance_variable_get(:@needs_history)[RouterTestDecider]).to be true - expect(router.instance_variable_get(:@needs_history)[RouterTestProjector]).to be false - end - - it 'passes partition_keys to register_consumer_group' do - router.register(RouterTestDecider) - - row = db[:sourced_consumer_groups].where(group_id: 'router-test-decider').first - expect(JSON.parse(row[:partition_by])).to eq(['device_id']) - end - end - - describe '#handle_next_for' do - before do - router.register(RouterTestDecider) - router.register(RouterTestProjector) - end - - it 'returns false when no work available' do - result = router.handle_next_for(RouterTestDecider) - expect(result).to be false - end - - it 'claims, calls handle_claim, executes actions + acks in transaction' do - # Set up: register device first (as history), then send bind command - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - result = router.handle_next_for(RouterTestDecider) - expect(result).to be true - - # DeviceBound event should have been appended to store - conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') - read_result = store.read(conds) - expect(read_result.messages.size).to eq(1) - expect(read_result.messages.first).to be_a(CCCRouterTestMessages::DeviceBound) - end - - it 'reads history for decider, skips history for projector' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - - # Projector should process without history - result = router.handle_next_for(RouterTestProjector) - expect(result).to be true - end - - it 'releases on ConcurrentAppendError' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - # Stub store.append to raise ConcurrentAppendError after claim - original_append = store.method(:append) - call_count = 0 - allow(store).to receive(:append) do |*args, **kwargs| - call_count += 1 - if call_count > 0 && kwargs[:guard] - raise Sourced::ConcurrentAppendError, 'conflict' - end - original_append.call(*args, **kwargs) - end - - result = router.handle_next_for(RouterTestDecider) - expect(result).to be true - - # Offset should be released (not advanced) — can re-claim - claim = store.claim_next( - 'router-test-decider', - partition_by: ['device_id'], - handled_types: RouterTestDecider.handled_messages.map(&:type), - worker_id: 'w-1' - ) - expect(claim).not_to be_nil - end - - it 'releases on StandardError and calls on_exception' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - # Make handle_claim raise - allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') - allow(RouterTestDecider).to receive(:on_exception) - - result = router.handle_next_for(RouterTestDecider) - expect(result).to be true - expect(RouterTestDecider).to have_received(:on_exception) - end - - it 'on_exception fails consumer group when default strategy' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') - - router.handle_next_for(RouterTestDecider) - expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false - row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first - expect(row[:status]).to eq('failed') - end - - it 'on_exception persists error_context in the database' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') - - router.handle_next_for(RouterTestDecider) - - row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first - expect(row[:error_context]).not_to be_nil - expect(row[:status]).to eq('failed') - end - - it 'on_exception with retry strategy sets retry_at on consumer group' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - retry_strategy = Sourced::ErrorStrategy.new do |s| - s.retry(times: 3, after: 5) - end - allow(Sourced::CCC).to receive_message_chain(:config, :error_strategy).and_return(retry_strategy) - - allow(RouterTestDecider).to receive(:handle_claim).and_raise(RuntimeError, 'boom') - - router.handle_next_for(RouterTestDecider) - - row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first - expect(row[:retry_at]).not_to be_nil - expect(row[:status]).to eq('active') - - ctx = JSON.parse(row[:error_context], symbolize_names: true) - expect(ctx[:retry_count]).to eq(2) - end - end - - describe '#drain' do - before do - router.register(RouterTestDecider) - router.register(RouterTestProjector) - end - - it 'processes all reactors until no work remains' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - store.append( - CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - ) - - router.drain - - # Decider should have produced DeviceBound - conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') - read_result = store.read(conds) - expect(read_result.messages.size).to eq(1) - - # Projector should have processed DeviceRegistered and DeviceBound - # (it handles both via evolve) - end - end - - describe 'full integration: append commands → Decider → events → Projector' do - before do - router.register(RouterTestDecider) - router.register(RouterTestProjector) - end - - it 'events are correlated with the command that produced them' do - reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - store.append(reg) - - cmd = CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - store.append(cmd) - - router.drain - - # Read the DeviceBound event - conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') - bound = store.read(conds).messages.find { |m| m.is_a?(CCCRouterTestMessages::DeviceBound) } - - expect(bound).not_to be_nil - expect(bound.causation_id).to eq(cmd.id) - expect(bound.correlation_id).to eq(cmd.correlation_id) - end - - it 'reaction messages are correlated with the event reacted to, not the command' do - reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - store.append(reg) - - cmd = CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - store.append(cmd) - - router.drain - - # Read the DeviceBound event (produced by command handler) - bound_conds = CCCRouterTestMessages::DeviceBound.to_conditions(device_id: 'd1') - bound = store.read(bound_conds).messages.find { |m| m.is_a?(CCCRouterTestMessages::DeviceBound) } - - # Read the NotifyBound reaction message (produced by reaction handler) - notify_conds = CCCRouterTestMessages::NotifyBound.to_conditions(device_id: 'd1') - notify = store.read(notify_conds).messages.find { |m| m.is_a?(CCCRouterTestMessages::NotifyBound) } - - expect(notify).not_to be_nil - # Reaction is correlated with the event, not the command - expect(notify.causation_id).to eq(bound.id) - expect(notify.correlation_id).to eq(bound.correlation_id) - # The whole chain shares the same correlation_id (the command's) - expect(notify.correlation_id).to eq(cmd.correlation_id) - end - end - - describe 'consumer group lifecycle' do - before do - router.register(RouterTestDecider) - router.register(RouterTestProjector) - end - - describe '#stop_consumer_group' do - it 'stops group and calls on_stop with class' do - expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be true - - router.stop_consumer_group(RouterTestDecider, 'maintenance') - - expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false - end - - it 'stops group and calls on_stop with string group_id' do - router.stop_consumer_group('router-test-decider', 'maintenance') - - expect(store.consumer_group_active?('router-test-decider')).to be false - end - - it 'invokes on_stop callback on reactor class' do - allow(RouterTestDecider).to receive(:on_stop) - - router.stop_consumer_group(RouterTestDecider, 'going down') - - expect(RouterTestDecider).to have_received(:on_stop).with('going down') - end - - it 'passes nil message when none given' do - allow(RouterTestDecider).to receive(:on_stop) - - router.stop_consumer_group(RouterTestDecider) - - expect(RouterTestDecider).to have_received(:on_stop).with(nil) - end - end - - describe '#reset_consumer_group' do - it 'resets group offsets and calls on_reset with class' do - # Append a message and drain so offsets are advanced - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - router.drain - - router.reset_consumer_group(RouterTestDecider) - - # Offsets should be cleared (group can re-process messages) - row = db[:sourced_consumer_groups].where(group_id: RouterTestDecider.group_id).first - expect(row[:discovery_position]).to eq(0) - end - - it 'resets group with string group_id' do - store.append( - CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - ) - router.drain - - router.reset_consumer_group('router-test-decider') - - row = db[:sourced_consumer_groups].where(group_id: 'router-test-decider').first - expect(row[:discovery_position]).to eq(0) - end - - it 'invokes on_reset callback on reactor class' do - allow(RouterTestDecider).to receive(:on_reset) - - router.reset_consumer_group(RouterTestDecider) - - expect(RouterTestDecider).to have_received(:on_reset) - end - end - - describe '#start_consumer_group' do - it 'starts group and calls on_start with class' do - router.stop_consumer_group(RouterTestDecider) - expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be false - - router.start_consumer_group(RouterTestDecider) - - expect(store.consumer_group_active?(RouterTestDecider.group_id)).to be true - end - - it 'starts group with string group_id' do - router.stop_consumer_group('router-test-decider') - - router.start_consumer_group('router-test-decider') - - expect(store.consumer_group_active?('router-test-decider')).to be true - end - - it 'invokes on_start callback on reactor class' do - allow(RouterTestDecider).to receive(:on_start) - - router.start_consumer_group(RouterTestDecider) - - expect(RouterTestDecider).to have_received(:on_start) - end - end - - describe 'resolve_reactor_class' do - it 'raises ArgumentError for unregistered group_id' do - expect { - router.stop_consumer_group('unknown-group') - }.to raise_error(ArgumentError, /No reactor registered with group_id 'unknown-group'/) - end - end - - describe 'no-op default callbacks' do - it 'works fine when reactor does not override callbacks' do - # RouterTestProjector has no custom callbacks — should not raise - expect { router.stop_consumer_group(RouterTestProjector) }.not_to raise_error - expect { router.reset_consumer_group(RouterTestProjector) }.not_to raise_error - expect { router.start_consumer_group(RouterTestProjector) }.not_to raise_error - end - end - end - - describe 'simple Consumer reactor (no Decider/Projector)' do - before do - router.register(RouterTestAuditReactor) - end - - it 'registers and processes messages through the router' do - reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - store.append(reg) - - router.drain - - # Audit message should have been appended - conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') - audits = store.read(conds).messages - expect(audits.size).to eq(1) - expect(audits.first).to be_a(CCCRouterTestMessages::DeviceAudited) - expect(audits.first.payload.event_type).to eq('router_test.device.registered') - end - - it 'appended messages are correlated with the source message' do - reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - store.append(reg) - - router.drain - - conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') - audit = store.read(conds).messages.first - - expect(audit.causation_id).to eq(reg.id) - expect(audit.correlation_id).to eq(reg.correlation_id) - end - - it 'handles multiple messages across partitions' do - store.append(CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor A' })) - store.append(CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd2', name: 'Sensor B' })) - - router.drain - - d1_conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') - d2_conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd2') - - expect(store.read(d1_conds).messages.size).to eq(1) - expect(store.read(d2_conds).messages.size).to eq(1) - end - - it 'context_for returns empty conditions (no evolve types)' do - expect(RouterTestAuditReactor.context_for(device_id: 'd1')).to eq([]) - end - - it 'works alongside deciders and projectors' do - router.register(RouterTestDecider) - - reg = CCCRouterTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - store.append(reg) - - cmd = CCCRouterTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) - store.append(cmd) - - router.drain - - # Audit reactor sees DeviceRegistered and DeviceBound - conds = CCCRouterTestMessages::DeviceAudited.to_conditions(device_id: 'd1') - audits = store.read(conds).messages - types = audits.map { |m| m.payload.event_type }.sort - - expect(types).to eq([ - 'router_test.device.bound', - 'router_test.device.registered' - ]) - end - end -end diff --git a/spec/sourced/ccc/supervisor_spec.rb b/spec/sourced/ccc/supervisor_spec.rb deleted file mode 100644 index ad190f17..00000000 --- a/spec/sourced/ccc/supervisor_spec.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -RSpec.describe Sourced::CCC::Supervisor do - let(:executor) { double('Executor') } - let(:logger) { instance_double('Logger', info: nil, warn: nil) } - let(:store_notifier) { Sourced::InlineNotifier.new } - let(:store) { double('Store', notifier: store_notifier) } - let(:reactor1) { double('Reactor1', handled_messages: [double(type: 'event1')], group_id: 'Reactor1', partition_keys: [:id]) } - let(:reactors) { [reactor1] } - let(:router) { instance_double(Sourced::CCC::Router, store: store, reactors: reactors) } - let(:task) { double('Task', spawn: nil) } - let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } - - before do - allow(Sourced::WorkQueue).to receive(:new).and_return(work_queue) - allow(executor).to receive(:start).and_yield(task) - allow(Signal).to receive(:trap) - end - - after do - Sourced::CCC.reset! - end - - describe '.start' do - it 'creates a new supervisor instance and starts it' do - supervisor_instance = instance_double(described_class) - expect(described_class).to receive(:new).with( - router: router, - logger: logger, - count: 3 - ).and_return(supervisor_instance) - expect(supervisor_instance).to receive(:start) - - described_class.start(router: router, logger: logger, count: 3) - end - end - - describe '#start' do - subject(:supervisor) do - described_class.new( - router: router, - logger: logger, - count: 2, - executor: executor - ) - end - - it 'sets up INT and TERM signal handlers' do - expect(Signal).to receive(:trap).with('INT') - expect(Signal).to receive(:trap).with('TERM') - supervisor.start - end - - it 'creates Dispatcher with correct params' do - expect(Sourced::CCC::Dispatcher).to receive(:new).with( - router: router, - worker_count: 2, - batch_size: 50, - max_drain_rounds: 10, - catchup_interval: 5, - housekeeping_interval: 30, - claim_ttl_seconds: 120, - logger: logger - ).and_call_original - - supervisor.start - end - - it 'passes custom params through to Dispatcher' do - custom_supervisor = described_class.new( - router: router, - logger: logger, - count: 4, - batch_size: 100, - max_drain_rounds: 20, - catchup_interval: 10, - housekeeping_interval: 60, - claim_ttl_seconds: 300, - executor: executor - ) - - expect(Sourced::CCC::Dispatcher).to receive(:new).with( - router: router, - worker_count: 4, - batch_size: 100, - max_drain_rounds: 20, - catchup_interval: 10, - housekeeping_interval: 60, - claim_ttl_seconds: 300, - logger: logger - ).and_call_original - - custom_supervisor.start - end - - it 'spawns via executor (notifier + catchup + scheduler + reaper + 2 workers = 6 spawns)' do - expect(executor).to receive(:start).and_yield(task) - # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns - expect(task).to receive(:spawn).exactly(6).times - - supervisor.start - end - end - - describe '#stop' do - subject(:supervisor) do - described_class.new( - router: router, - logger: logger, - count: 2, - executor: executor - ) - end - - before do - supervisor.start - end - - it 'logs shutdown information' do - expect(logger).to receive(:info).with('CCC::Supervisor: stopping dispatcher') - expect(logger).to receive(:info).with('CCC::Supervisor: all workers stopped') - # Dispatcher also logs - allow(logger).to receive(:info) - supervisor.stop - end - - it 'stops the dispatcher' do - dispatcher = supervisor.instance_variable_get(:@dispatcher) - expect(dispatcher).to receive(:stop) - # Suppress logs from Supervisor#stop - allow(logger).to receive(:info) - supervisor.stop - end - end - - describe 'signal handling' do - subject(:supervisor) do - described_class.new( - router: router, - logger: logger, - executor: executor - ) - end - - it 'traps INT and TERM signals to call stop' do - int_handler = nil - term_handler = nil - - allow(Signal).to receive(:trap) do |signal, &block| - int_handler = block if signal == 'INT' - term_handler = block if signal == 'TERM' - end - - supervisor.start - - expect(supervisor).to receive(:stop) - int_handler.call - - expect(supervisor).to receive(:stop) - term_handler.call - end - end -end diff --git a/spec/sourced/ccc/testing/rspec_spec.rb b/spec/sourced/ccc/testing/rspec_spec.rb deleted file mode 100644 index 072ba33b..00000000 --- a/spec/sourced/ccc/testing/rspec_spec.rb +++ /dev/null @@ -1,290 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' -require 'sourced/ccc/testing/rspec' - -# Reuse message definitions from decider_spec and projector_spec -module CCCGWTTestMessages - DeviceRegistered = Sourced::CCC::Message.define('gwt_test.device.registered') do - attribute :device_id, String - attribute :name, String - end - - DeviceBound = Sourced::CCC::Message.define('gwt_test.device.bound') do - attribute :device_id, String - attribute :asset_id, String - end - - BindDevice = Sourced::CCC::Message.define('gwt_test.bind_device') do - attribute :device_id, String - attribute :asset_id, String - end - - NotifyBound = Sourced::CCC::Message.define('gwt_test.notify_bound') do - attribute :device_id, String - end - - NoopCommand = Sourced::CCC::Message.define('gwt_test.noop_command') do - attribute :device_id, String - end - - ItemAdded = Sourced::CCC::Message.define('gwt_test.item.added') do - attribute :list_id, String - attribute :name, String - end - - ItemArchived = Sourced::CCC::Message.define('gwt_test.item.archived') do - attribute :list_id, String - attribute :name, String - end - - NotifyArchive = Sourced::CCC::Message.define('gwt_test.notify_archive') do - attribute :list_id, String - end -end - -class GWTTestDecider < Sourced::CCC::Decider - partition_by :device_id - consumer_group 'gwt-test-decider' - - state { |_| { exists: false, bound: false } } - - evolve CCCGWTTestMessages::DeviceRegistered do |state, _evt| - state[:exists] = true - end - - evolve CCCGWTTestMessages::DeviceBound do |state, _evt| - state[:bound] = true - end - - command CCCGWTTestMessages::BindDevice do |state, cmd| - raise 'Not found' unless state[:exists] - raise 'Already bound' if state[:bound] - event CCCGWTTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id - end - - command CCCGWTTestMessages::NoopCommand do |_state, _cmd| - # intentionally produces no events - end - - reaction CCCGWTTestMessages::DeviceBound do |_state, evt| - CCCGWTTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) - end - - sync do |state:, messages:, events:| - state[:synced] = true - end - - after_sync do |state:, messages:, events:| - state[:after_synced] = true - end -end - -# Decider without reactions (produces only events) -class GWTTestSimpleDecider < Sourced::CCC::Decider - partition_by :device_id - consumer_group 'gwt-test-simple-decider' - - state { |_| { exists: false } } - - evolve CCCGWTTestMessages::DeviceRegistered do |state, _evt| - state[:exists] = true - end - - command CCCGWTTestMessages::BindDevice do |state, cmd| - raise 'Not found' unless state[:exists] - event CCCGWTTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id - end -end - -class GWTTestStateStoredProjector < Sourced::CCC::Projector::StateStored - partition_by :list_id - consumer_group 'gwt-test-ss-projector' - - state do |(list_id)| - { list_id: list_id, items: [], synced: false, after_synced: false } - end - - evolve CCCGWTTestMessages::ItemAdded do |state, msg| - state[:items] << msg.payload.name - end - - evolve CCCGWTTestMessages::ItemArchived do |state, msg| - state[:items].delete(msg.payload.name) - end - - sync do |state:, messages:, replaying:| - state[:synced] = true - end - - after_sync do |state:, messages:, replaying:| - state[:after_synced] = true - end -end - -class GWTTestEventSourcedProjector < Sourced::CCC::Projector::EventSourced - partition_by :list_id - consumer_group 'gwt-test-es-projector' - - state do |(list_id)| - { list_id: list_id, items: [], synced: false, after_synced: false } - end - - evolve CCCGWTTestMessages::ItemAdded do |state, msg| - state[:items] << msg.payload.name - end - - evolve CCCGWTTestMessages::ItemArchived do |state, msg| - state[:items].delete(msg.payload.name) - end - - sync do |state:, messages:, replaying:| - state[:synced] = true - end - - after_sync do |state:, messages:, replaying:| - state[:after_synced] = true - end -end - -RSpec.describe Sourced::CCC::Testing::RSpec do - include Sourced::CCC::Testing::RSpec - - describe 'Decider' do - it 'given history + when command → then expected messages (event + reaction)' do - with_reactor(GWTTestDecider, device_id: 'd1') - .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then( - CCCGWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), - CCCGWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) - ) - end - - it 'then with shorthand (Class, **payload) for single expected message' do - with_reactor(GWTTestSimpleDecider, device_id: 'd1') - .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then(CCCGWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') - end - - it 'no given + when command → then exception (invariant violation)' do - with_reactor(GWTTestDecider, device_id: 'd1') - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then(RuntimeError, 'Not found') - end - - it 'then with block form yields action pairs' do - with_reactor(GWTTestDecider, device_id: 'd1') - .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then { |r| - expect(r.pairs).to be_a(Array) - actions, _source = r.pairs.first - append_actions = Array(actions).select { |a| a.respond_to?(:messages) } - expect(append_actions).not_to be_empty - } - end - - it 'then with [] expects no messages' do - with_reactor(GWTTestDecider, device_id: 'd1') - .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') - .when(CCCGWTTestMessages::NoopCommand, device_id: 'd1') - .then([]) - end - - it 'then! runs sync and after_sync actions' do - with_reactor(GWTTestDecider, device_id: 'd1') - .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then! { |r| - expect(r.pairs).to be_a(Array) - } - end - - it 'given with message instances' do - reg = CCCGWTTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - - with_reactor(GWTTestDecider, device_id: 'd1') - .given(reg) - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') - .then( - CCCGWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), - CCCGWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) - ) - end - - it 'supports .and as alias for .given' do - with_reactor(GWTTestDecider, device_id: 'd1') - .given(CCCGWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') - .and(CCCGWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') - .when(CCCGWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a2') - .then(RuntimeError, 'Already bound') - end - end - - describe 'Projector (StateStored)' do - it 'given events → then block asserts evolved state' do - with_reactor(GWTTestStateStoredProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then { |r| expect(r.state[:items]).to eq(['Apple']) } - end - - it 'given multiple events → then block sees cumulative state' do - with_reactor(GWTTestStateStoredProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') - .then { |r| expect(r.state[:items]).to eq(['Apple', 'Banana']) } - end - - it 'then! runs sync actions before yielding state' do - with_reactor(GWTTestStateStoredProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |r| expect(r.state[:synced]).to be true } - end - - it 'then! runs after_sync actions before yielding state' do - with_reactor(GWTTestStateStoredProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |r| expect(r.state[:after_synced]).to be true } - end - - it 'given events with archive → state reflects removal' do - with_reactor(GWTTestStateStoredProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .and(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') - .and(CCCGWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') - .then { |r| expect(r.state[:items]).to eq(['Banana']) } - end - end - - describe 'Projector (EventSourced)' do - it 'given events → then block asserts evolved state' do - with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') - .then { |r| expect(r.state[:items]).to eq(['Apple', 'Banana']) } - end - - it 'given events with archive → state reflects removal' do - with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .and(CCCGWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') - .then { |r| expect(r.state[:items]).to eq([]) } - end - - it 'then! runs sync actions before yielding state' do - with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |r| expect(r.state[:synced]).to be true } - end - - it 'then! runs after_sync actions before yielding state' do - with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') - .given(CCCGWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') - .then! { |r| expect(r.state[:after_synced]).to be true } - end - - end -end diff --git a/spec/sourced/ccc/topology_spec.rb b/spec/sourced/ccc/topology_spec.rb deleted file mode 100644 index 71ba6e91..00000000 --- a/spec/sourced/ccc/topology_spec.rb +++ /dev/null @@ -1,432 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/ccc' - -module CCCTopologyTest - # --- Messages --- - CreateWidget = Sourced::CCC::Command.define('ccc_topo.create_widget') do - attribute :widget_id, String - attribute :name, String - end - - WidgetCreated = Sourced::CCC::Event.define('ccc_topo.widget_created') do - attribute :widget_id, String - attribute :name, String - end - - NotifyWidget = Sourced::CCC::Command.define('ccc_topo.notify_widget') do - attribute :widget_id, String - end - - WidgetNotified = Sourced::CCC::Event.define('ccc_topo.widget_notified') do - attribute :widget_id, String - end - - ArchiveWidget = Sourced::CCC::Command.define('ccc_topo.archive_widget') do - attribute :widget_id, String - end - - WidgetArchived = Sourced::CCC::Event.define('ccc_topo.widget_archived') do - attribute :widget_id, String - end - - DelayedCmd = Sourced::CCC::Command.define('ccc_topo.delayed_cmd') do - attribute :widget_id, String - end - - ScheduleEvent = Sourced::CCC::Event.define('ccc_topo.schedule_event') do - attribute :widget_id, String - end - - # --- Decider --- - class WidgetDecider < Sourced::CCC::Decider - partition_by :widget_id - - state { |_| { exists: false } } - - evolve WidgetCreated do |state, _evt| - state[:exists] = true - end - - evolve WidgetArchived do |state, _evt| - state[:exists] = false - end - - command CreateWidget do |_state, cmd| - event WidgetCreated, widget_id: cmd.payload.widget_id, name: cmd.payload.name - end - - command ArchiveWidget do |_state, _cmd| - event WidgetArchived, widget_id: 'x' - end - - reaction WidgetCreated do |_state, evt| - dispatch NotifyWidget, widget_id: evt.payload.widget_id - end - end - - # --- Notifier Decider --- - class NotifierDecider < Sourced::CCC::Decider - partition_by :widget_id - - state { |_| {} } - - evolve WidgetNotified do |state, _evt| - state[:notified] = true - end - - command NotifyWidget do |_state, cmd| - event WidgetNotified, widget_id: cmd.payload.widget_id - end - end - - # --- Projector (StateStored, no reactions) --- - class WidgetListProjector < Sourced::CCC::Projector::StateStored - partition_by :widget_id - consumer_group 'CCCTopologyTest::WidgetListProjector' - - state { |_| { items: [] } } - - evolve WidgetCreated do |state, msg| - state[:items] << msg.payload.name - end - end - - # --- Projector (EventSourced, with catch-all reaction) --- - class ReactingProjector < Sourced::CCC::Projector::EventSourced - partition_by :widget_id - consumer_group 'CCCTopologyTest::ReactingProjector' - - state { |_| { items: [] } } - - evolve WidgetCreated do |state, msg| - state[:items] << msg.payload.name - end - - reaction do |_state, evt| - dispatch NotifyWidget, widget_id: 'x' - end - end - - # --- Projector with specific + catch-all reactions --- - class MixedReactingProjector < Sourced::CCC::Projector::EventSourced - partition_by :widget_id - consumer_group 'CCCTopologyTest::MixedReactingProjector' - - state { |_| { items: [] } } - - evolve WidgetCreated do |state, msg| - state[:items] << msg.payload.name - end - - evolve WidgetArchived do |_state, _msg| - end - - reaction WidgetCreated do |_state, evt| - dispatch NotifyWidget, widget_id: evt.payload.widget_id - end - - reaction do |_state, _evt| - dispatch ArchiveWidget, widget_id: 'x' - end - end - - # --- Decider with chained dispatch (.at) --- - class SchedulingDecider < Sourced::CCC::Decider - partition_by :widget_id - - state { |_| {} } - - evolve ScheduleEvent do |state, _evt| - state[:scheduled] = true - end - - command CreateWidget do |_state, cmd| - event ScheduleEvent, widget_id: cmd.payload.widget_id - end - - reaction ScheduleEvent do |_state, evt| - dispatch(DelayedCmd, widget_id: evt.payload.widget_id).at(evt.created_at + 300) - end - end -end - -RSpec.describe Sourced::CCC::Topology do - let(:nodes) { described_class.build(reactors) } - - def find_node(id) - nodes.find { |n| n.id == id } - end - - def find_nodes_by_type(type) - nodes.select { |n| n.type == type } - end - - context 'with WidgetDecider and NotifierDecider' do - let(:reactors) { [CCCTopologyTest::WidgetDecider, CCCTopologyTest::NotifierDecider] } - - it 'builds command nodes for handled commands' do - cmd_nodes = find_nodes_by_type('command') - expect(cmd_nodes.map(&:id)).to contain_exactly( - 'ccc_topo.create_widget', - 'ccc_topo.archive_widget', - 'ccc_topo.notify_widget' - ) - end - - it 'sets correct group_id on command nodes' do - node = find_node('ccc_topo.create_widget') - expect(node.group_id).to eq('CCCTopologyTest::WidgetDecider') - end - - it 'extracts produced events via Prism' do - node = find_node('ccc_topo.create_widget') - expect(node.produces).to eq(['ccc_topo.widget_created']) - end - - it 'extracts produced events for NotifierDecider' do - node = find_node('ccc_topo.notify_widget') - expect(node.produces).to eq(['ccc_topo.widget_notified']) - end - - it 'sets command name from message class' do - node = find_node('ccc_topo.create_widget') - expect(node.name).to eq('CCCTopologyTest::CreateWidget') - end - - it 'extracts schema from command payload' do - node = find_node('ccc_topo.create_widget') - expect(node.schema).to include('type' => 'object') - expect(node.schema['properties']).to include('widget_id', 'name') - end - - it 'builds event nodes deduplicated by type' do - evt_nodes = find_nodes_by_type('event') - evt_types = evt_nodes.map(&:id) - expect(evt_types).to contain_exactly( - 'ccc_topo.widget_created', - 'ccc_topo.widget_archived', - 'ccc_topo.widget_notified' - ) - end - - it 'assigns first-seen group_id to event nodes' do - node = find_node('ccc_topo.widget_created') - expect(node.group_id).to eq('CCCTopologyTest::WidgetDecider') - end - - it 'event nodes have empty produces' do - find_nodes_by_type('event').each { |n| expect(n.produces).to eq([]) } - end - - it 'extracts schema from event payload' do - node = find_node('ccc_topo.widget_created') - expect(node.schema).to include('type' => 'object') - expect(node.schema['properties']).to include('widget_id', 'name') - end - - it 'builds automation nodes for reactions' do - aut_nodes = find_nodes_by_type('automation') - expect(aut_nodes.map(&:id)).to include( - 'ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut' - ) - end - - it 'sets correct consumes on automation nodes' do - node = find_node('ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut') - expect(node.consumes).to eq(['ccc_topo.widget_created']) - end - - it 'extracts dispatched commands from reactions via Prism' do - node = find_node('ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut') - expect(node.produces).to eq(['ccc_topo.notify_widget']) - end - - it 'sets automation name from event class' do - node = find_node('ccc_topo.widget_created-CCCTopologyTest::WidgetDecider-aut') - expect(node.name).to eq('reaction(CCCTopologyTest::WidgetCreated)') - end - end - - context 'with WidgetListProjector (no reactions)' do - let(:reactors) { [CCCTopologyTest::WidgetListProjector] } - - it 'does not build command nodes' do - expect(find_nodes_by_type('command')).to be_empty - end - - it 'does not build automation nodes' do - expect(find_nodes_by_type('automation')).to be_empty - end - - it 'builds event nodes from evolve handlers' do - evt_nodes = find_nodes_by_type('event') - expect(evt_nodes.map(&:id)).to eq(['ccc_topo.widget_created']) - end - - it 'builds a readmodel node' do - node = find_node('ccc_topology_test.widget_list_projector-rm') - expect(node).not_to be_nil - expect(node.type).to eq('readmodel') - end - - it 'sets correct name and group_id on readmodel node' do - node = find_node('ccc_topology_test.widget_list_projector-rm') - expect(node.name).to eq('CCCTopologyTest::WidgetListProjector') - expect(node.group_id).to eq('CCCTopologyTest::WidgetListProjector') - end - - it 'readmodel consumes the evolved event types' do - node = find_node('ccc_topology_test.widget_list_projector-rm') - expect(node.consumes).to eq(['ccc_topo.widget_created']) - end - - it 'readmodel produces nothing when there are no reactions' do - node = find_node('ccc_topology_test.widget_list_projector-rm') - expect(node.produces).to eq([]) - end - end - - context 'with ReactingProjector (catch-all reaction)' do - let(:reactors) { [CCCTopologyTest::ReactingProjector] } - let(:rm_id) { 'ccc_topology_test.reacting_projector-rm' } - let(:aut_id) { 'ccc_topology_test.reacting_projector-aut' } - - it 'builds a readmodel node' do - node = find_node(rm_id) - expect(node).not_to be_nil - expect(node.type).to eq('readmodel') - end - - it 'readmodel consumes the evolved event types' do - node = find_node(rm_id) - expect(node.consumes).to eq(['ccc_topo.widget_created']) - end - - it 'readmodel produces a single automation node' do - node = find_node(rm_id) - expect(node.produces).to eq([aut_id]) - end - - it 'builds a single catch-all automation node' do - aut_nodes = find_nodes_by_type('automation') - expect(aut_nodes.size).to eq(1) - node = aut_nodes.first - expect(node.id).to eq(aut_id) - expect(node.name).to eq('reaction(CCCTopologyTest::ReactingProjector)') - end - - it 'automation node consumes the readmodel' do - node = find_node(aut_id) - expect(node.consumes).to eq([rm_id]) - end - - it 'automation node produces dispatched commands' do - node = find_node(aut_id) - expect(node.produces).to eq(['ccc_topo.notify_widget']) - end - end - - context 'with MixedReactingProjector (specific + catch-all reactions)' do - let(:reactors) { [CCCTopologyTest::MixedReactingProjector] } - let(:rm_id) { 'ccc_topology_test.mixed_reacting_projector-rm' } - let(:specific_aut_id) { 'ccc_topo.widget_created-CCCTopologyTest::MixedReactingProjector-aut' } - let(:catchall_aut_id) { 'ccc_topology_test.mixed_reacting_projector-aut' } - - it 'builds two automation nodes: one specific and one catch-all' do - aut_nodes = find_nodes_by_type('automation') - expect(aut_nodes.map(&:id)).to contain_exactly(specific_aut_id, catchall_aut_id) - end - - it 'specific automation is named after the event' do - node = find_node(specific_aut_id) - expect(node.name).to eq('reaction(CCCTopologyTest::WidgetCreated)') - end - - it 'catch-all automation is named after the reactor' do - node = find_node(catchall_aut_id) - expect(node.name).to eq('reaction(CCCTopologyTest::MixedReactingProjector)') - end - - it 'both automation nodes consume the readmodel' do - [specific_aut_id, catchall_aut_id].each do |id| - node = find_node(id) - expect(node.consumes).to eq([rm_id]) - end - end - - it 'readmodel produces both automation node IDs' do - node = find_node(rm_id) - expect(node.produces).to contain_exactly(specific_aut_id, catchall_aut_id) - end - end - - context 'with SchedulingDecider (chained dispatch)' do - let(:reactors) { [CCCTopologyTest::SchedulingDecider] } - - it 'detects dispatch through .at() chain' do - node = find_node('ccc_topo.schedule_event-CCCTopologyTest::SchedulingDecider-aut') - expect(node).not_to be_nil - expect(node.produces).to eq(['ccc_topo.delayed_cmd']) - end - end - - context 'event deduplication across reactors' do - let(:reactors) { [CCCTopologyTest::WidgetDecider, CCCTopologyTest::WidgetListProjector] } - - it 'deduplicates event nodes by type string' do - evt_nodes = find_nodes_by_type('event').select { |n| n.id == 'ccc_topo.widget_created' } - expect(evt_nodes.size).to eq(1) - end - - it 'uses first reactor as group_id owner' do - node = find_node('ccc_topo.widget_created') - expect(node.group_id).to eq('CCCTopologyTest::WidgetDecider') - end - end - - context 'command deduplication across reactors' do - let(:reactors) { [CCCTopologyTest::WidgetDecider, CCCTopologyTest::SchedulingDecider] } - - it 'deduplicates command nodes by type string' do - cmd_nodes = find_nodes_by_type('command').select { |n| n.id == 'ccc_topo.create_widget' } - expect(cmd_nodes.size).to eq(1) - end - - it 'produces no duplicate IDs' do - ids = nodes.map(&:id) - expect(ids).to eq(ids.uniq) - end - end - - context 'with all test reactors' do - let(:reactors) do - [ - CCCTopologyTest::WidgetDecider, - CCCTopologyTest::NotifierDecider, - CCCTopologyTest::WidgetListProjector, - CCCTopologyTest::ReactingProjector, - CCCTopologyTest::SchedulingDecider - ] - end - - it 'returns flat array of node structs' do - nodes.each do |n| - expect(n).to be_a(Struct) - expect(%w[command event automation readmodel]).to include(n.type) - end - end - - it 'all command nodes have produces arrays' do - find_nodes_by_type('command').each { |n| expect(n.produces).to be_an(Array) } - end - - it 'all automation nodes have consumes and produces arrays' do - find_nodes_by_type('automation').each do |n| - expect(n.consumes).to be_an(Array) - expect(n.produces).to be_an(Array) - end - end - end -end diff --git a/spec/sourced_spec.rb b/spec/sourced_spec.rb deleted file mode 100644 index ca531702..00000000 --- a/spec/sourced_spec.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Sourced do - it 'has a version number' do - expect(Sourced::VERSION).not_to be nil - end - - describe '.new_stream_id' do - specify 'no prefix' do - si1 = Sourced.new_stream_id - si2 = Sourced.new_stream_id - expect(si1).not_to eq(si2) - end - - specify 'with prefix' do - si1 = Sourced.new_stream_id('cart') - si2 = Sourced.new_stream_id('cart') - expect(si1).not_to eq(si2) - expect(si1).to start_with('cart') - end - end - - describe '.dispatch(message)' do - before(:all) do - @message_class = Sourced::Message.define('dispatch.test') do - attribute :name, String - end - end - - it 'appends message' do - msg = @message_class.parse(stream_id: 'aaa', payload: { name: 'Joe' }) - expect(Sourced.dispatch(msg)).to eq(msg) - expect(Sourced.config.backend.read_stream('aaa').map(&:id)).to eq([msg.id]) - end - - it 'raises if message is invalid' do - msg = @message_class.new(stream_id: 'aaa', payload: { name: 22 }) - expect { - Sourced.dispatch(msg) - }.to raise_error(Sourced::InvalidMessageError) - end - - it 'raises if backend fails to append' do - msg = @message_class.parse(stream_id: 'aaa', payload: { name: 'Joe' }) - allow(Sourced.config.backend).to receive(:append_next_to_stream).and_return false - expect { - Sourced.dispatch(msg) - }.to raise_error(Sourced::BackendError) - end - end - - specify '.registered?' do - reactor1 = Class.new do - extend Sourced::Consumer - - consumer do |info| - info.group_id = 'reactor1' - end - - def self.handled_messages = [Sourced::Event] - def self.handle(...) = [] - end - - reactor2 = Class.new do - extend Sourced::Consumer - - consumer do |info| - info.group_id = 'reactor2' - end - - def self.handled_messages = [Sourced::Event] - def self.handle(...) = [] - end - - Sourced.register(reactor1) - - expect(Sourced.registered?(reactor1)).to be true - expect(Sourced.registered?(reactor2)).to be false - end -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 25381d91..71512560 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,37 +1,19 @@ # frozen_string_literal: true -require 'dotenv/load' require 'sourced' -require 'debug' require 'logger' require 'timecop' require 'sourced/testing/rspec' -require_relative './shared_examples/backend_examples' -require_relative './shared_examples/executor_examples' ENV['ENVIRONMENT'] ||= 'test' -Sourced.configure do |config| - if ENV['LOGS_DIR'] - FileUtils.mkdir_p(ENV['LOGS_DIR']) - config.logger = Logger.new(File.join(ENV['LOGS_DIR'], 'test.log')) - else - config.logger = Logger.new(STDOUT) - end -end - RSpec.configure do |config| - # Enable flags like --only-failures and --next-failure config.example_status_persistence_file_path = '.rspec_status' - - # Disable RSpec exposing methods globally on `Module` and `main` config.disable_monkey_patching! config.expect_with :rspec do |c| c.syntax = :expect end - config.include BackendExamples, type: :backend - config.include ExecutorExamples, type: :executor config.include Sourced::Testing::RSpec end diff --git a/spec/sourced/ccc/stale_claim_reaper_spec.rb b/spec/stale_claim_reaper_spec.rb similarity index 93% rename from spec/sourced/ccc/stale_claim_reaper_spec.rb rename to spec/stale_claim_reaper_spec.rb index ab01e028..f60d111a 100644 --- a/spec/sourced/ccc/stale_claim_reaper_spec.rb +++ b/spec/stale_claim_reaper_spec.rb @@ -1,17 +1,17 @@ # frozen_string_literal: true require 'spec_helper' -require 'sourced/ccc' +require 'sourced' require 'sequel' module StaleClaimReaperTestMessages - DeviceRegistered = Sourced::CCC::Message.define('reaper_test.device.registered') do + DeviceRegistered = Sourced::Message.define('reaper_test.device.registered') do attribute :device_id, String attribute :name, String end end -class ReaperTestProjector < Sourced::CCC::Projector::StateStored +class ReaperTestProjector < Sourced::Projector::StateStored partition_by :device_id consumer_group 'reaper-test-projector' @@ -24,7 +24,7 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored RSpec.describe 'Store worker heartbeat and stale claim release' do let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } + let(:store) { Sourced::Store.new(db) } before { store.install! } @@ -157,9 +157,9 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored end end -RSpec.describe Sourced::CCC::StaleClaimReaper do +RSpec.describe Sourced::StaleClaimReaper do let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } + let(:store) { Sourced::Store.new(db) } let(:logger) { instance_double('Logger', info: nil, warn: nil, debug: nil) } before { store.install! } @@ -213,7 +213,7 @@ class ReaperTestProjector < Sourced::CCC::Projector::StateStored reaper.stop thread.join(1) - expect(logger).to have_received(:info).with('CCC::StaleClaimReaper: stopped') + expect(logger).to have_received(:info).with('Sourced::StaleClaimReaper: stopped') end end end diff --git a/spec/sourced/ccc/store_spec.rb b/spec/store_spec.rb similarity index 76% rename from spec/sourced/ccc/store_spec.rb rename to spec/store_spec.rb index a92848c1..2237c1ec 100644 --- a/spec/sourced/ccc/store_spec.rb +++ b/spec/store_spec.rb @@ -1,50 +1,50 @@ # frozen_string_literal: true require 'spec_helper' -require 'sourced/ccc' -require 'sourced/ccc/store' +require 'sourced' +require 'sourced/store' require 'sequel' # Define test messages for store specs (namespaced to avoid collisions) -module CCCStoreTestMessages - DeviceRegistered = Sourced::CCC::Message.define('store_test.device.registered') do +module StoreTestMessages + DeviceRegistered = Sourced::Message.define('store_test.device.registered') do attribute :device_id, String attribute :name, String end - AssetRegistered = Sourced::CCC::Message.define('store_test.asset.registered') do + AssetRegistered = Sourced::Message.define('store_test.asset.registered') do attribute :asset_id, String attribute :label, String end - DeviceBound = Sourced::CCC::Message.define('store_test.device.bound') do + DeviceBound = Sourced::Message.define('store_test.device.bound') do attribute :device_id, String attribute :asset_id, String end # Course/user messages for composite partition tests - CourseCreated = Sourced::CCC::Message.define('store_test.course.created') do + CourseCreated = Sourced::Message.define('store_test.course.created') do attribute :course_name, String end - UserRegistered = Sourced::CCC::Message.define('store_test.user.registered') do + UserRegistered = Sourced::Message.define('store_test.user.registered') do attribute :user_id, String attribute :name, String end - UserJoinedCourse = Sourced::CCC::Message.define('store_test.user.joined_course') do + UserJoinedCourse = Sourced::Message.define('store_test.user.joined_course') do attribute :course_name, String attribute :user_id, String end - CourseClosed = Sourced::CCC::Message.define('store_test.course.closed') do + CourseClosed = Sourced::Message.define('store_test.course.closed') do attribute :course_name, String end end -RSpec.describe Sourced::CCC::Store do +RSpec.describe Sourced::Store do let(:db) { Sequel.sqlite } - let(:store) { Sourced::CCC::Store.new(db) } + let(:store) { Sourced::Store.new(db) } before do store.install! @@ -57,7 +57,7 @@ module CCCStoreTestMessages it 'returns false before install!' do fresh_db = Sequel.sqlite - fresh_store = Sourced::CCC::Store.new(fresh_db) + fresh_store = Sourced::Store.new(fresh_db) expect(fresh_store.installed?).to be false end end @@ -81,7 +81,7 @@ module CCCStoreTestMessages describe '#append' do it 'appends a single message and returns position' do - msg = CCCStoreTestMessages::DeviceRegistered.new( + msg = StoreTestMessages::DeviceRegistered.new( payload: { device_id: 'dev-1', name: 'Sensor A' } ) pos = store.append(msg) @@ -90,15 +90,15 @@ module CCCStoreTestMessages it 'appends multiple messages and returns last position' do msgs = [ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }) ] pos = store.append(msgs) expect(pos).to eq(2) end it 'extracts and indexes key pairs' do - msg = CCCStoreTestMessages::DeviceRegistered.new( + msg = StoreTestMessages::DeviceRegistered.new( payload: { device_id: 'dev-1', name: 'Sensor A' } ) store.append(msg) @@ -114,8 +114,8 @@ module CCCStoreTestMessages end it 'deduplicates key pairs across messages' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) - msg2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor B' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg2 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor B' }) store.append([msg1, msg2]) # 'device_id'/'dev-1' should exist once in key_pairs @@ -129,7 +129,7 @@ module CCCStoreTestMessages end it 'stores metadata as JSON' do - msg = CCCStoreTestMessages::DeviceRegistered.new( + msg = StoreTestMessages::DeviceRegistered.new( payload: { device_id: 'dev-1', name: 'Sensor A' }, metadata: { user_id: 42 } ) @@ -141,16 +141,16 @@ module CCCStoreTestMessages end it 'persists and round-trips causation_id and correlation_id' do - source = CCCStoreTestMessages::DeviceRegistered.new( + source = StoreTestMessages::DeviceRegistered.new( payload: { device_id: 'dev-1', name: 'Sensor A' } ) caused = source.correlate( - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }) + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }) ) store.append([source, caused]) - cond1 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' }) - cond2 = Sourced::CCC::QueryCondition.new(message_type: 'store_test.asset.registered', attrs: { asset_id: 'asset-1' }) + cond1 = Sourced::QueryCondition.new(message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' }) + cond2 = Sourced::QueryCondition.new(message_type: 'store_test.asset.registered', attrs: { asset_id: 'asset-1' }) messages, = store.read([cond1, cond2]) src = messages.find { |m| m.type == 'store_test.device.registered' } @@ -171,7 +171,7 @@ module CCCStoreTestMessages describe '#schedule_messages and #update_schedule!' do it 'stores delayed messages outside the main log until due' do now = Time.now - delayed = CCCStoreTestMessages::DeviceRegistered.new( + delayed = StoreTestMessages::DeviceRegistered.new( payload: { device_id: 'dev-1', name: 'Sensor A' } ).at(now + 60) @@ -183,7 +183,7 @@ module CCCStoreTestMessages it 'promotes due messages into the flat log and preserves metadata' do now = Time.now - due = CCCStoreTestMessages::DeviceRegistered.new( + due = StoreTestMessages::DeviceRegistered.new( payload: { device_id: 'dev-1', name: 'Sensor A' }, metadata: { source: 'test' } ).at(now + 2) @@ -197,14 +197,14 @@ module CCCStoreTestMessages expect(db[:sourced_scheduled_messages].count).to eq(0) expect(store.latest_position).to eq(1) - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: due.type, attrs: { device_id: 'dev-1' } ) result = store.read([cond]) msg = result.messages.first - expect(msg).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(msg).to be_a(StoreTestMessages::DeviceRegistered) expect(msg.created_at).to be >= Time.at((now + 3).to_i) expect(msg.metadata[:source]).to eq('test') expect(Time.parse(msg.metadata[:scheduled_at])).to be_a(Time) @@ -217,53 +217,53 @@ module CCCStoreTestMessages describe '#append with guard (conditional append)' do let(:cond) do - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) end it 'succeeds when no conflicts exist' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg1) _events, guard = store.read([cond]) - msg2 = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + msg2 = StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) pos = store.append(msg2, guard: guard) expect(pos).to eq(2) end it 'raises ConcurrentAppendError when conflicts exist' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg1) _events, guard = store.read([cond]) # Concurrent write by another process - conflicting = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) + conflicting = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) store.append(conflicting) - new_msg = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + new_msg = StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) expect { store.append(new_msg, guard: guard) }.to raise_error(Sourced::ConcurrentAppendError) end it 'is atomic — failed append does not change store state' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg1) _events, guard = store.read([cond]) # Concurrent write - conflicting = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) + conflicting = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) store.append(conflicting) position_before = store.latest_position count_before = db[:sourced_messages].count - new_msg = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + new_msg = StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) expect { store.append(new_msg, guard: guard) }.to raise_error(Sourced::ConcurrentAppendError) @@ -273,21 +273,21 @@ module CCCStoreTestMessages end it 'works with a manually constructed guard' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg1) - guard = Sourced::CCC::ConsistencyGuard.new(conditions: [cond], last_position: store.latest_position) + guard = Sourced::ConsistencyGuard.new(conditions: [cond], last_position: store.latest_position) - msg2 = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + msg2 = StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) pos = store.append(msg2, guard: guard) expect(pos).to eq(2) end it 'append without guard still works unconditionally' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg1) - msg2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) + msg2 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A v2' }) pos = store.append(msg2) expect(pos).to eq(2) end @@ -296,13 +296,13 @@ module CCCStoreTestMessages describe '#read_all' do it 'returns a ReadAllResult with messages and last_position' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' }) ]) result = store.read_all - expect(result).to be_a(Sourced::CCC::ReadAllResult) + expect(result).to be_a(Sourced::ReadAllResult) expect(result.messages.size).to eq(3) expect(result.messages.map(&:position)).to eq([1, 2, 3]) expect(result.last_position).to eq(3) @@ -310,8 +310,8 @@ module CCCStoreTestMessages it 'supports #each to iterate current page messages' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }) ]) result = store.read_all @@ -321,8 +321,8 @@ module CCCStoreTestMessages it '#each without a block returns an enumerator for chaining' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' }) ]) result = store.read_all @@ -331,7 +331,7 @@ module CCCStoreTestMessages end it 'supports destructuring into messages and last_position' do - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) messages, last_position = store.read_all expect(messages.size).to eq(1) @@ -340,7 +340,7 @@ module CCCStoreTestMessages it 'paginates with from_position and limit' do 5.times do |i| - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) end result1 = store.read_all(limit: 2) @@ -362,32 +362,32 @@ module CCCStoreTestMessages end it 'returns PositionedMessage instances' do - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) result = store.read_all - expect(result.messages.first).to be_a(Sourced::CCC::PositionedMessage) - expect(result.messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(result.messages.first).to be_a(Sourced::PositionedMessage) + expect(result.messages.first).to be_a(StoreTestMessages::DeviceRegistered) end describe 'PositionedMessage#to_message' do - it 'unwraps to the underlying CCC::Message instance' do - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + it 'unwraps to the underlying Sourced::Message instance' do + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) wrapped = store.read_all.messages.first inner = wrapped.to_message - expect(wrapped).to be_a(Sourced::CCC::PositionedMessage) - expect(inner).to be_a(CCCStoreTestMessages::DeviceRegistered) - expect(inner).not_to be_a(Sourced::CCC::PositionedMessage) + expect(wrapped).to be_a(Sourced::PositionedMessage) + expect(inner).to be_a(StoreTestMessages::DeviceRegistered) + expect(inner).not_to be_a(Sourced::PositionedMessage) expect(inner.payload.device_id).to eq('dev-1') end - it 'allows case/when against wrapped messages via CCC::Message.===' do - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + it 'allows case/when against wrapped messages via Sourced::Message.===' do + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) wrapped = store.read_all.messages.first matched = case wrapped - when CCCStoreTestMessages::DeviceRegistered then :device + when StoreTestMessages::DeviceRegistered then :device else :other end expect(matched).to eq(:device) @@ -397,7 +397,7 @@ module CCCStoreTestMessages context 'with order: :desc' do before do 5.times do |i| - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) end end @@ -429,7 +429,7 @@ module CCCStoreTestMessages describe '#to_enum' do before do 5.times do |i| - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: "dev-#{i}", name: "D#{i}" })) end end @@ -467,15 +467,15 @@ module CCCStoreTestMessages context 'with conditions' do before do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'Sensor B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'Sensor B' }) ]) end it 'filters messages matching conditions' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -485,11 +485,11 @@ module CCCStoreTestMessages end it 'ORs multiple conditions' do - cond1 = Sourced::CCC::QueryCondition.new( + cond1 = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) - cond2 = Sourced::CCC::QueryCondition.new( + cond2 = Sourced::QueryCondition.new( message_type: 'store_test.asset.registered', attrs: { asset_id: 'asset-1' } ) @@ -499,7 +499,7 @@ module CCCStoreTestMessages end it 'combines conditions with from_position' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-2' } ) @@ -514,7 +514,7 @@ module CCCStoreTestMessages end it 'returns empty messages when conditions match nothing' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'nonexistent' } ) @@ -523,11 +523,11 @@ module CCCStoreTestMessages end it 'paginates with to_enum' do - cond1 = Sourced::CCC::QueryCondition.new( + cond1 = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) - cond2 = Sourced::CCC::QueryCondition.new( + cond2 = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-2' } ) @@ -542,15 +542,15 @@ module CCCStoreTestMessages describe '#read' do before do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'Sensor B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'asset-1', label: 'Truck' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'Sensor B' }) ]) end it 'queries by message_type and key condition' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -561,7 +561,7 @@ module CCCStoreTestMessages end it 'returns messages with position' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -571,11 +571,11 @@ module CCCStoreTestMessages it 'queries with multiple OR conditions' do conditions = [ - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ), - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1' } ) @@ -589,7 +589,7 @@ module CCCStoreTestMessages end it 'filters with after_position' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -604,11 +604,11 @@ module CCCStoreTestMessages it 'applies limit' do conditions = [ - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ), - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1' } ) @@ -619,7 +619,7 @@ module CCCStoreTestMessages end it 'returns empty for non-matching conditions' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'nonexistent' } ) @@ -633,29 +633,29 @@ module CCCStoreTestMessages end it 'deserializes into correct message subclasses' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) results, _guard = store.read([cond]) msg = results.first - expect(msg).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(msg).to be_a(StoreTestMessages::DeviceRegistered) expect(msg.id).to match(/\A[0-9a-f-]{36}\z/) expect(msg.created_at).to be_a(Time) end it 'returns a ConsistencyGuard with correct conditions' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) _results, guard = store.read([cond]) - expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(guard).to be_a(Sourced::ConsistencyGuard) expect(guard.conditions).to eq([cond]) end it 'guard last_position reflects the last result position' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-2' } ) @@ -666,7 +666,7 @@ module CCCStoreTestMessages end it 'guard last_position falls back to latest_position when no results' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'nonexistent' } ) @@ -675,7 +675,7 @@ module CCCStoreTestMessages end it 'guard last_position falls back to after_position when no results and after_position given' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'nonexistent' } ) @@ -688,16 +688,16 @@ module CCCStoreTestMessages before do store.append([ # Two DeviceBound events with different (device_id, asset_id) combinations - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-2' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-2', asset_id: 'asset-1' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-2' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-2', asset_id: 'asset-1' }), # A single-attr message sharing device_id - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }), ]) end it 'compound condition ANDs attrs — only matches messages with all specified key pairs' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1', asset_id: 'asset-1' } ) @@ -710,7 +710,7 @@ module CCCStoreTestMessages it 'excludes messages that match only one attr of a compound condition' do # dev-1/asset-2 shares device_id but not asset_id # dev-2/asset-1 shares asset_id but not device_id - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1', asset_id: 'asset-1' } ) @@ -723,12 +723,12 @@ module CCCStoreTestMessages it 'ORs across separate conditions — compound and single-attr mixed' do conditions = [ # Compound: only dev-1/asset-1 - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1', asset_id: 'asset-1' } ), # Single-attr: dev-1 registered - Sourced::CCC::QueryCondition.new( + Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -742,19 +742,19 @@ module CCCStoreTestMessages end it 'guard with compound condition detects conflicts correctly' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1', asset_id: 'asset-1' } ) _results, guard = store.read([cond]) # Concurrent write: same device_id + asset_id combination - conflicting = CCCStoreTestMessages::DeviceBound.new( + conflicting = StoreTestMessages::DeviceBound.new( payload: { device_id: 'dev-1', asset_id: 'asset-1' } ) store.append(conflicting) - new_msg = CCCStoreTestMessages::DeviceBound.new( + new_msg = StoreTestMessages::DeviceBound.new( payload: { device_id: 'dev-1', asset_id: 'asset-1' } ) expect { @@ -763,19 +763,19 @@ module CCCStoreTestMessages end it 'guard with compound condition does NOT flag writes to a different partition as conflicts' do - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.bound', attrs: { device_id: 'dev-1', asset_id: 'asset-1' } ) _results, guard = store.read([cond]) # Write to a DIFFERENT partition (dev-1/asset-2) — should NOT conflict - other_partition = CCCStoreTestMessages::DeviceBound.new( + other_partition = StoreTestMessages::DeviceBound.new( payload: { device_id: 'dev-1', asset_id: 'asset-2' } ) store.append(other_partition) - new_msg = CCCStoreTestMessages::DeviceBound.new( + new_msg = StoreTestMessages::DeviceBound.new( payload: { device_id: 'dev-1', asset_id: 'asset-1' } ) expect { @@ -786,14 +786,14 @@ module CCCStoreTestMessages describe '#messages_since' do it 'returns messages after the given position' do - msg1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg1) pos = store.latest_position - msg2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A Updated' }) + msg2 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A Updated' }) store.append(msg2) - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -801,15 +801,15 @@ module CCCStoreTestMessages conflicts, guard = store.messages_since([cond], pos) expect(conflicts.size).to eq(1) expect(conflicts.first.payload.name).to eq('Sensor A Updated') - expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(guard).to be_a(Sourced::ConsistencyGuard) end it 'returns empty when no new messages' do - msg = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) + msg = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor A' }) store.append(msg) pos = store.latest_position - cond = Sourced::CCC::QueryCondition.new( + cond = Sourced::QueryCondition.new( message_type: 'store_test.device.registered', attrs: { device_id: 'dev-1' } ) @@ -826,8 +826,8 @@ module CCCStoreTestMessages it 'returns max position after appends' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) expect(store.latest_position).to eq(2) end @@ -836,7 +836,7 @@ module CCCStoreTestMessages describe '#clear!' do it 'deletes all data and resets position' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) expect(store.latest_position).to eq(1) @@ -850,12 +850,12 @@ module CCCStoreTestMessages it 'resets autoincrement so next position starts from 1' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.clear! pos = store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ) expect(pos).to eq(1) end @@ -863,7 +863,7 @@ module CCCStoreTestMessages it 'clears consumer groups, offsets, and offset_key_pairs' do store.register_consumer_group('test-group') store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.claim_next('test-group', partition_by: 'device_id', @@ -967,7 +967,7 @@ module CCCStoreTestMessages it 'yields a GroupUpdater and persists error_context' do store.updating_consumer_group('my-group') do |group| - expect(group).to be_a(Sourced::CCC::GroupUpdater) + expect(group).to be_a(Sourced::GroupUpdater) group.retry(Time.now + 30, retry_count: 1) end @@ -1036,7 +1036,7 @@ module CCCStoreTestMessages it 'deletes all offsets' do store.register_consumer_group('my-group') store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.claim_next('my-group', partition_by: 'device_id', @@ -1051,7 +1051,7 @@ module CCCStoreTestMessages it 'accepts an object responding to #group_id' do store.register_consumer_group('my-group') store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.claim_next('my-group', partition_by: 'device_id', @@ -1075,7 +1075,7 @@ module CCCStoreTestMessages it 'lazily discovers and creates offsets for new partitions' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1089,27 +1089,27 @@ module CCCStoreTestMessages it 'returns messages for next unclaimed partition with correct shape' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') expect(result).not_to be_nil - expect(result).to be_a(Sourced::CCC::ClaimResult) + expect(result).to be_a(Sourced::ClaimResult) expect(result.offset_id).to be_a(Integer) expect(result.key_pair_ids).to be_a(Array) expect(result.partition_key).to eq('device_id:dev-1') expect(result.partition_value).to eq({ 'device_id' => 'dev-1' }) expect(result.messages).to be_a(Array) expect(result.messages.size).to eq(1) - expect(result.messages.first).to be_a(CCCStoreTestMessages::DeviceRegistered) + expect(result.messages.first).to be_a(StoreTestMessages::DeviceRegistered) expect(result.messages.first.position).to eq(1) - expect(result.guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(result.guard).to be_a(Sourced::ConsistencyGuard) end it 'returns multiple pending messages for same partition' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) ]) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1118,8 +1118,8 @@ module CCCStoreTestMessages it 'skips claimed partitions — second worker gets different partition' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1130,7 +1130,7 @@ module CCCStoreTestMessages it 'returns nil when all partitions claimed' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1140,7 +1140,7 @@ module CCCStoreTestMessages it 'respects handled_types filter' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: ['store_test.asset.registered'], worker_id: 'w-1') @@ -1149,7 +1149,7 @@ module CCCStoreTestMessages it 'only returns messages after last_position' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1157,7 +1157,7 @@ module CCCStoreTestMessages # Append another message for same partition store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A updated' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A updated' }) ) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1167,7 +1167,7 @@ module CCCStoreTestMessages it 'returns nil for stopped consumer group' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.stop_consumer_group(group_id) @@ -1177,7 +1177,7 @@ module CCCStoreTestMessages it 'returns nil when retry_at is in the future' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) # Set retry_at to the future @@ -1191,7 +1191,7 @@ module CCCStoreTestMessages it 'returns claims when retry_at is in the past' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) # Set retry_at to the past @@ -1205,10 +1205,10 @@ module CCCStoreTestMessages it 'prioritizes partition with earliest pending message' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ) store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1218,7 +1218,7 @@ module CCCStoreTestMessages it 'bootstraps newly appeared partitions on subsequent calls' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1226,7 +1226,7 @@ module CCCStoreTestMessages # New partition appears store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) ) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1236,13 +1236,13 @@ module CCCStoreTestMessages it 'returns a guard with conditions only for attrs each type actually has' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') guard = result.guard - expect(guard).to be_a(Sourced::CCC::ConsistencyGuard) + expect(guard).to be_a(Sourced::ConsistencyGuard) expect(guard.last_position).to eq(result.messages.last.position) # 1 handled_type with device_id attr = 1 condition @@ -1254,13 +1254,13 @@ module CCCStoreTestMessages it 'guard can be used for optimistic concurrency on append' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') # No concurrent writes — append with guard succeeds - new_event = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + new_event = StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) expect { store.append(new_event, guard: result.guard) }.not_to raise_error store.ack(group_id, offset_id: result.offset_id, position: result.messages.last.position) @@ -1268,17 +1268,17 @@ module CCCStoreTestMessages it 'guard detects concurrent conflicting writes' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') # Simulate concurrent write after claim store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Concurrent' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Concurrent' }) ) - new_event = CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + new_event = StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) expect { store.append(new_event, guard: result.guard) }.to raise_error(Sourced::ConcurrentAppendError) @@ -1286,7 +1286,7 @@ module CCCStoreTestMessages it 'replaying is false when consumer group has never processed the partition' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1295,7 +1295,7 @@ module CCCStoreTestMessages it 'replaying is false for new messages after ack' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1303,7 +1303,7 @@ module CCCStoreTestMessages # New message arrives after ack store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) ) r2 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1312,8 +1312,8 @@ module CCCStoreTestMessages it 'replaying is true when offset is reset and messages are re-claimed' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) ]) # Process and ack — highest_position advances to 2 @@ -1331,7 +1331,7 @@ module CCCStoreTestMessages it 'replaying transitions to false once consumer passes highest_position after reset' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) # Process and ack up to position 1 @@ -1343,7 +1343,7 @@ module CCCStoreTestMessages # Add a new message at position 2 store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) ) # Re-claim: both messages, but last position (2) > highest_position (1) @@ -1363,7 +1363,7 @@ module CCCStoreTestMessages it 'advances discovery_position after discovering partitions' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1374,8 +1374,8 @@ module CCCStoreTestMessages it 'does not re-scan earlier messages on subsequent discovery calls' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) # First claim discovers both partitions and claims one @@ -1391,7 +1391,7 @@ module CCCStoreTestMessages it 'discovers new partitions added after initial discovery' do # First batch of messages store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1403,7 +1403,7 @@ module CCCStoreTestMessages # New partition arrives store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ) # Should discover and claim the new partition @@ -1414,7 +1414,7 @@ module CCCStoreTestMessages it 'resets discovery_position when consumer group is reset' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) r1 = store.claim_next(group_id, partition_by: 'device_id', handled_types: handled_types, worker_id: 'w-1') @@ -1432,9 +1432,9 @@ module CCCStoreTestMessages it 'a new reactor catches up on old messages without explicit bootstrap' do # Pre-populate with multiple partitions store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) ]) # New reactor with no prior offsets @@ -1459,11 +1459,11 @@ module CCCStoreTestMessages # Regression: highest_position short-circuit skipped unprocessed partitions # when one partition's last message happened to be at types_max_pos. store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-4', name: 'D' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-5', name: 'E' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-4', name: 'D' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-5', name: 'E' }) ]) new_group = 'multi-catch-up' @@ -1487,8 +1487,8 @@ module CCCStoreTestMessages it 'only discovers partitions matching handled_types' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'Truck' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'Truck' }) ]) # Only handles device events, not asset events @@ -1521,7 +1521,7 @@ module CCCStoreTestMessages it 'bootstraps composite partitions (only messages with ALL attributes create partitions)' do # CourseCreated only has course_name — not enough for a (course_name, user_id) partition store.append( - CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }) + StoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }) ) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1529,7 +1529,7 @@ module CCCStoreTestMessages # UserJoinedCourse has both — NOW a partition is created store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1539,9 +1539,9 @@ module CCCStoreTestMessages it 'fetches messages with single partition attribute matching' do store.append([ - CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), - CCCStoreTestMessages::UserRegistered.new(payload: { user_id: 'joe', name: 'Joe' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + StoreTestMessages::UserRegistered.new(payload: { user_id: 'joe', name: 'Joe' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1555,8 +1555,8 @@ module CCCStoreTestMessages it 'different composite partitions can be claimed in parallel' do store.append([ - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Physics', user_id: 'jake' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Physics', user_id: 'jake' }) ]) r1 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1569,9 +1569,9 @@ module CCCStoreTestMessages it 'excludes messages with ALL partition attributes that do not match ALL values' do store.append([ - CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) + StoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1594,7 +1594,7 @@ module CCCStoreTestMessages it 'messages with ALL partition attributes matching are not duplicated' do store.append([ - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1604,8 +1604,8 @@ module CCCStoreTestMessages it 'excludes messages with partial attributes matching wrong value' do store.append([ - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'History', user_id: 'joe' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'History', user_id: 'joe' }) ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1617,8 +1617,8 @@ module CCCStoreTestMessages it 'returns guard with one condition per message type containing only matching attrs' do store.append([ - CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') @@ -1649,18 +1649,18 @@ module CCCStoreTestMessages it 'guard detects concurrent writes in composite partition' do store.append([ - CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' }), + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ]) result = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') # Concurrent write: course closed while decider was processing store.append( - CCCStoreTestMessages::CourseClosed.new(payload: { course_name: 'Algebra' }) + StoreTestMessages::CourseClosed.new(payload: { course_name: 'Algebra' }) ) - new_event = CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + new_event = StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) expect { store.append(new_event, guard: result.guard) }.to raise_error(Sourced::ConcurrentAppendError) @@ -1688,13 +1688,13 @@ module CCCStoreTestMessages # Step 1: append and process both partitions store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ) r1 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') store.ack(group_id, offset_id: r1.offset_id, position: r1.messages.last.position) store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) ) r2 = store.claim_next(group_id, partition_by: ['course_name', 'user_id'], handled_types: handled_types, worker_id: 'w-1') store.ack(group_id, offset_id: r2.offset_id, position: r2.messages.last.position) @@ -1703,7 +1703,7 @@ module CCCStoreTestMessages # Step 2: new message for jake only store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'jake' }) ) # Step 3: claim_next should find jake (the partition with real work), @@ -1725,7 +1725,7 @@ module CCCStoreTestMessages # Append and process one message per seat seats.each do |seat| store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: seat }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: seat }) ) end seats.size.times do @@ -1736,10 +1736,10 @@ module CCCStoreTestMessages # All caught up. Now append new messages for A3 and A5 only. store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'A3' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'A3' }) ) store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'A5' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'A5' }) ) # Should claim A3 and A5, not be blocked by A1/A2/A4's stale OR matches @@ -1768,7 +1768,7 @@ module CCCStoreTestMessages it 'advances offset and releases claim' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', @@ -1785,8 +1785,8 @@ module CCCStoreTestMessages it 'after ack, subsequent claim skips processed messages' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) ]) r1 = store.claim_next(group_id, partition_by: 'device_id', @@ -1810,7 +1810,7 @@ module CCCStoreTestMessages it 'creates and advances offset for a new partition' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.advance_offset(group_id, @@ -1826,7 +1826,7 @@ module CCCStoreTestMessages it 'advances highest_position on the consumer group' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.advance_offset(group_id, @@ -1840,8 +1840,8 @@ module CCCStoreTestMessages it 'never decreases the offset' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'B' }) ]) store.advance_offset(group_id, @@ -1863,8 +1863,8 @@ module CCCStoreTestMessages it 'never decreases highest_position' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) store.advance_offset(group_id, @@ -1883,7 +1883,7 @@ module CCCStoreTestMessages it 'causes claim_next to skip the advanced partition' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.advance_offset(group_id, @@ -1901,8 +1901,8 @@ module CCCStoreTestMessages it 'only skips messages up to the advanced position' do store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a1' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a1' }) ]) # Advance past only the first message @@ -1923,7 +1923,7 @@ module CCCStoreTestMessages it 'is a no-op for nonexistent consumer group' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) expect { @@ -1949,7 +1949,7 @@ module CCCStoreTestMessages it 'works with composite partitions' do store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ) store.advance_offset(group_id, @@ -1968,7 +1968,7 @@ module CCCStoreTestMessages describe '#stats' do it 'returns zeroes for an empty store' do result = store.stats - expect(result).to be_a(Sourced::CCC::Stats) + expect(result).to be_a(Sourced::Stats) expect(result.max_position).to eq(0) expect(result.groups).to eq([]) end @@ -1994,8 +1994,8 @@ module CCCStoreTestMessages store.register_consumer_group(group_id) store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) # Claim and ack first partition @@ -2048,7 +2048,7 @@ module CCCStoreTestMessages describe '#read_offsets' do it 'returns empty result when no offsets exist' do result = store.read_offsets - expect(result).to be_a(Sourced::CCC::OffsetsResult) + expect(result).to be_a(Sourced::OffsetsResult) expect(result.offsets).to eq([]) expect(result.total_count).to eq(0) end @@ -2058,8 +2058,8 @@ module CCCStoreTestMessages store.register_consumer_group('group-b') store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) # Claim to bootstrap offsets in both groups @@ -2092,7 +2092,7 @@ module CCCStoreTestMessages store.register_consumer_group('group-b') store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ]) store.claim_next('group-a', partition_by: 'device_id', @@ -2109,9 +2109,9 @@ module CCCStoreTestMessages store.register_consumer_group('group-a') store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) ]) # Claim each to bootstrap offsets @@ -2139,9 +2139,9 @@ module CCCStoreTestMessages store.register_consumer_group('group-a') store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-3', name: 'C' }) ]) 3.times do @@ -2160,7 +2160,7 @@ module CCCStoreTestMessages store.register_consumer_group('group-a') store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ]) # Claim creates an offset that is claimed @@ -2192,11 +2192,11 @@ module CCCStoreTestMessages describe '#read_correlation_batch' do it 'returns all messages sharing the same correlation_id, ordered by position' do # Create a command (source of the correlation chain) - cmd = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor' }) + cmd = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'Sensor' }) # Create correlated events - evt1 = cmd.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) - evt2 = cmd.correlate(CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'Asset A' })) + evt1 = cmd.correlate(StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) + evt2 = cmd.correlate(StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'Asset A' })) store.append([cmd, evt1, evt2]) @@ -2211,12 +2211,12 @@ module CCCStoreTestMessages end it 'excludes messages with a different correlation_id' do - cmd1 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) - evt1 = cmd1.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) + cmd1 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + evt1 = cmd1.correlate(StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) # Unrelated chain - cmd2 = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) - evt2 = cmd2.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-2', asset_id: 'a-2' })) + cmd2 = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + evt2 = cmd2.correlate(StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-2', asset_id: 'a-2' })) store.append([cmd1, evt1, cmd2, evt2]) @@ -2226,8 +2226,8 @@ module CCCStoreTestMessages end it 'can be queried from any message in the chain' do - cmd = CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) - evt = cmd.correlate(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) + cmd = StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + evt = cmd.correlate(StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'a-1' })) store.append([cmd, evt]) @@ -2247,7 +2247,7 @@ module CCCStoreTestMessages it 'releases claim without advancing' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) result = store.claim_next(group_id, partition_by: 'device_id', @@ -2262,7 +2262,7 @@ module CCCStoreTestMessages it 'after release, same partition re-claimed with same messages' do store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) r1 = store.claim_next(group_id, partition_by: 'device_id', @@ -2284,7 +2284,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:device_id]) store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) offsets = db[:sourced_offsets].all @@ -2296,8 +2296,8 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:device_id]) store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-2', name: 'B' }) ]) offsets = db[:sourced_offsets].order(:partition_key).all @@ -2309,8 +2309,8 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:device_id]) store.append([ - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), - CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }), + StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' }) ]) offsets = db[:sourced_offsets].all @@ -2320,8 +2320,8 @@ module CCCStoreTestMessages it 'is idempotent across multiple appends' do store.register_consumer_group(group_id, partition_by: [:device_id]) - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) - store.append(CCCStoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(StoreTestMessages::DeviceBound.new(payload: { device_id: 'dev-1', asset_id: 'asset-1' })) offsets = db[:sourced_offsets].all expect(offsets.size).to eq(1) @@ -2331,7 +2331,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:device_id]) # AssetRegistered has no device_id - store.append(CCCStoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' })) + store.append(StoreTestMessages::AssetRegistered.new(payload: { asset_id: 'a-1', label: 'X' })) offsets = db[:sourced_offsets].all expect(offsets).to be_empty @@ -2341,7 +2341,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:course_name, :user_id]) store.append( - CCCStoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) + StoreTestMessages::UserJoinedCourse.new(payload: { course_name: 'Algebra', user_id: 'joe' }) ) offsets = db[:sourced_offsets].all @@ -2353,7 +2353,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:course_name, :user_id]) # CourseCreated only has course_name, not user_id - store.append(CCCStoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' })) + store.append(StoreTestMessages::CourseCreated.new(payload: { course_name: 'Algebra' })) offsets = db[:sourced_offsets].all expect(offsets).to be_empty @@ -2361,7 +2361,7 @@ module CCCStoreTestMessages it 'does not create offsets when no consumer groups are registered' do # No register_consumer_group call - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) offsets = db[:sourced_offsets].all expect(offsets).to be_empty @@ -2370,7 +2370,7 @@ module CCCStoreTestMessages it 'does not create offsets when partition_by is nil (legacy group)' do store.register_consumer_group(group_id) # no partition_by - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) offsets = db[:sourced_offsets].all expect(offsets).to be_empty @@ -2380,7 +2380,7 @@ module CCCStoreTestMessages store.register_consumer_group('group-a', partition_by: [:device_id]) store.register_consumer_group('group-b', partition_by: [:device_id]) - store.append(CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) + store.append(StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' })) cg_a = db[:sourced_consumer_groups].where(group_id: 'group-a').first cg_b = db[:sourced_consumer_groups].where(group_id: 'group-b').first @@ -2411,7 +2411,7 @@ module CCCStoreTestMessages store.register_consumer_group(group_id, partition_by: [:device_id]) store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) # Offsets already exist from append — claim should work without discovery @@ -2430,7 +2430,7 @@ module CCCStoreTestMessages it 'claim_next falls back to discovery for pre-existing messages' do # Append BEFORE registering — no eager offsets store.append( - CCCStoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) + StoreTestMessages::DeviceRegistered.new(payload: { device_id: 'dev-1', name: 'A' }) ) store.register_consumer_group(group_id, partition_by: [:device_id]) diff --git a/spec/supervisor_spec.rb b/spec/supervisor_spec.rb index 27b47a51..e908cfd4 100644 --- a/spec/supervisor_spec.rb +++ b/spec/supervisor_spec.rb @@ -1,86 +1,117 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' RSpec.describe Sourced::Supervisor do let(:executor) { double('Executor') } let(:logger) { instance_double('Logger', info: nil, warn: nil) } - let(:backend_notifier) { Sourced::InlineNotifier.new } - let(:backend) { double('Backend', notifier: backend_notifier) } - let(:reactor1) { double('Reactor1', handled_messages: [double(type: 'event1')], consumer_info: double(group_id: 'Reactor1')) } - let(:reactors) { Set.new([reactor1]) } - let(:router) { instance_double(Sourced::Router, backend: backend, async_reactors: reactors) } - let(:housekeeper) { instance_double(Sourced::HouseKeeper, work: nil, stop: nil) } + let(:store_notifier) { Sourced::InlineNotifier.new } + let(:store) { double('Store', notifier: store_notifier) } + let(:reactor1) { double('Reactor1', handled_messages: [double(type: 'event1')], group_id: 'Reactor1', partition_keys: [:id]) } + let(:reactors) { [reactor1] } + let(:router) { instance_double(Sourced::Router, store: store, reactors: reactors) } let(:task) { double('Task', spawn: nil) } let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } before do - allow(Sourced::HouseKeeper).to receive(:new).and_return(housekeeper) allow(Sourced::WorkQueue).to receive(:new).and_return(work_queue) allow(executor).to receive(:start).and_yield(task) allow(Signal).to receive(:trap) end + after do + Sourced.reset! + end + + describe '.start' do + it 'creates a new supervisor instance and starts it' do + supervisor_instance = instance_double(described_class) + expect(described_class).to receive(:new).with( + router: router, + logger: logger, + count: 3 + ).and_return(supervisor_instance) + expect(supervisor_instance).to receive(:start) + + described_class.start(router: router, logger: logger, count: 3) + end + end + describe '#start' do subject(:supervisor) do described_class.new( - logger:, + router: router, + logger: logger, count: 2, - housekeeping_count: 1, - executor: executor, - router: router + executor: executor ) end - it 'sets up signal handlers' do + it 'sets up INT and TERM signal handlers' do expect(Signal).to receive(:trap).with('INT') expect(Signal).to receive(:trap).with('TERM') supervisor.start end - it 'creates the correct number of housekeepers with proper configuration' do - expect(Sourced::HouseKeeper).to receive(:new).with( - hash_including( - logger:, - backend: router.backend, - name: 'HouseKeeper-0' - ) - ).and_return(housekeeper) + it 'creates Dispatcher with correct params' do + expect(Sourced::Dispatcher).to receive(:new).with( + router: router, + worker_count: 2, + batch_size: 50, + max_drain_rounds: 10, + catchup_interval: 5, + housekeeping_interval: 30, + claim_ttl_seconds: 120, + logger: logger + ).and_call_original supervisor.start end - it 'spawns tasks for housekeepers and dispatcher components via executor' do - expect(executor).to receive(:start).and_yield(task) - # 1 housekeeper + 1 notifier + 1 catchup_poller + 2 workers = 5 spawns - expect(task).to receive(:spawn).exactly(5).times + it 'passes custom params through to Dispatcher' do + custom_supervisor = described_class.new( + router: router, + logger: logger, + count: 4, + batch_size: 100, + max_drain_rounds: 20, + catchup_interval: 10, + housekeeping_interval: 60, + claim_ttl_seconds: 300, + executor: executor + ) - supervisor.start + expect(Sourced::Dispatcher).to receive(:new).with( + router: router, + worker_count: 4, + batch_size: 100, + max_drain_rounds: 20, + catchup_interval: 10, + housekeeping_interval: 60, + claim_ttl_seconds: 300, + logger: logger + ).and_call_original + + custom_supervisor.start end - it 'provides worker_ids_provider proc to housekeepers that returns live worker names' do - worker_ids_provider = nil - allow(Sourced::HouseKeeper).to receive(:new) do |args| - worker_ids_provider = args[:worker_ids_provider] - housekeeper - end + it 'spawns via executor (notifier + catchup + scheduler + reaper + 2 workers = 6 spawns)' do + expect(executor).to receive(:start).and_yield(task) + # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns + expect(task).to receive(:spawn).exactly(6).times supervisor.start - - names = worker_ids_provider.call - expect(names.size).to eq(2) - expect(names).to all(match(/worker-\d$/)) end end describe '#stop' do subject(:supervisor) do described_class.new( - logger:, + router: router, + logger: logger, count: 2, - housekeeping_count: 1, - executor:, - router: + executor: executor ) end @@ -89,38 +120,28 @@ end it 'logs shutdown information' do - expect(logger).to receive(:info).with(/Stopping dispatcher/) - expect(logger).to receive(:info).with('All workers stopped') + expect(logger).to receive(:info).with('Sourced::Supervisor: stopping dispatcher') + expect(logger).to receive(:info).with('Sourced::Supervisor: all workers stopped') # Dispatcher also logs allow(logger).to receive(:info) supervisor.stop end - it 'calls stop on all housekeepers' do - expect(housekeeper).to receive(:stop) + it 'stops the dispatcher' do + dispatcher = supervisor.instance_variable_get(:@dispatcher) + expect(dispatcher).to receive(:stop) + # Suppress logs from Supervisor#stop + allow(logger).to receive(:info) supervisor.stop end end - describe '.start' do - it 'creates a new supervisor instance and starts it' do - supervisor_instance = instance_double(described_class) - expect(described_class).to receive(:new).with( - logger:, - count: 3 - ).and_return(supervisor_instance) - expect(supervisor_instance).to receive(:start) - - described_class.start(logger:, count: 3) - end - end - describe 'signal handling' do subject(:supervisor) do described_class.new( - logger:, - executor:, - router: + router: router, + logger: logger, + executor: executor ) end diff --git a/spec/support/unit_test_fixtures.rb b/spec/support/unit_test_fixtures.rb deleted file mode 100644 index 348be785..00000000 --- a/spec/support/unit_test_fixtures.rb +++ /dev/null @@ -1,240 +0,0 @@ -# frozen_string_literal: true - -# Shared message and reactor definitions for Sourced::Unit specs. -# Required by both spec/unit_spec.rb and spec/unit_sequel_postgres_spec.rb. -module UnitTest - # Messages - CreateThing = Sourced::Command.define('unittest.create_thing') do - attribute :name, String - end - - ThingCreated = Sourced::Event.define('unittest.thing_created') do - attribute :name, String - end - - NotifyThing = Sourced::Command.define('unittest.notify_thing') do - attribute :name, String - end - - ThingNotified = Sourced::Event.define('unittest.thing_notified') do - attribute :name, String - end - - # An Actor that handles CreateThing command, produces ThingCreated event, - # and has a reaction that dispatches NotifyThing command. - class ThingActor < Sourced::Actor - state do |id| - { id: id, name: nil, status: 'new' } - end - - command CreateThing do |state, cmd| - event ThingCreated, name: cmd.payload.name - end - - event ThingCreated do |state, event| - state[:name] = event.payload.name - state[:status] = 'created' - end - - reaction ThingCreated do |state, event| - dispatch(NotifyThing, name: state[:name]) - end - end - - # A second Actor that handles NotifyThing command - class NotifierActor < Sourced::Actor - state do |id| - { id: id, notified: false, name: nil } - end - - command NotifyThing do |state, cmd| - event ThingNotified, name: cmd.payload.name - end - - event ThingNotified do |state, event| - state[:notified] = true - state[:name] = event.payload.name - end - end - - # A projector that tracks ThingCreated events - class ThingProjector < Sourced::Projector::StateStored - state do |id| - { id: id, things: [] } - end - - event ThingCreated do |state, event| - state[:things] << event.payload.name - end - end - - # A projector that tracks ThingNotified events - class NotifiedProjector < Sourced::Projector::StateStored - state do |id| - { id: id, notifications: [] } - end - - event ThingNotified do |state, event| - state[:notifications] << event.payload.name - end - end - - # A projector with a catch-all reaction (reacts to all evolved events) - class ReactingProjector < Sourced::Projector::StateStored - state do |id| - { id: id, things: [] } - end - - event ThingCreated do |state, event| - state[:things] << event.payload.name - end - - reaction do |state, event| - dispatch(NotifyThing, name: state[:things].last) - end - end - - # A projector that evolves multiple events, with a specific reaction on one - # and a catch-all reaction covering the rest - class MixedReactingProjector < Sourced::Projector::StateStored - state do |id| - { id: id, things: [], notifications: [] } - end - - event ThingCreated do |state, event| - state[:things] << event.payload.name - end - - event ThingNotified do |state, event| - state[:notifications] << event.payload.name - end - - # Specific reaction for ThingCreated - reaction ThingCreated do |state, event| - dispatch(NotifyThing, name: state[:things].last) - end - - # Catch-all covers ThingNotified (ThingCreated already has a specific reaction) - reaction do |state, event| - dispatch(CreateThing, name: 'from-catchall') - end - end - - # For infinite loop tests: an actor that reacts to its own events - LoopCmd = Sourced::Command.define('unittest.loop_cmd') - LoopEvent = Sourced::Event.define('unittest.loop_event') - - class LoopingActor < Sourced::Actor - state do |id| - { id: id } - end - - command LoopCmd do |state, cmd| - event LoopEvent - end - - event LoopEvent do |state, event| - end - - reaction LoopEvent do |state, event| - dispatch(LoopCmd) - end - end - - # For scheduled messages tests - ScheduleCmd = Sourced::Command.define('unittest.schedule_cmd') - ScheduleEvent = Sourced::Event.define('unittest.schedule_event') - DelayedCmd = Sourced::Command.define('unittest.delayed_cmd') - - class SchedulingActor < Sourced::Actor - state do |id| - { id: id } - end - - command ScheduleCmd do |state, cmd| - event ScheduleEvent - end - - event ScheduleEvent do |state, event| - end - - reaction ScheduleEvent do |state, event| - dispatch(DelayedCmd).at(Time.now + 3600) - end - end - - # For testing bracket-accessor dispatch syntax: Reactor[:command_name] - # Uses inline command definitions so that Reactor[] can resolve them. - class InlineNotifierActor < Sourced::Actor - state do |id| - { id: id, notified: false } - end - - command :notify_inline, name: String do |state, cmd| - event :inline_notified, name: cmd.payload.name - end - - event :inline_notified, name: String do |state, event| - state[:notified] = true - end - end - - class BracketDispatchActor < Sourced::Actor - state do |id| - { id: id } - end - - command CreateThing do |state, cmd| - event ThingCreated, name: cmd.payload.name - end - - event ThingCreated do |state, event| - state[:name] = event.payload.name - end - - reaction ThingCreated do |state, event| - dispatch(InlineNotifierActor[:notify_inline], name: state[:name]) - end - end - - # For testing multi-line command definitions with symbol event names - class MultiLineCommandActor < Sourced::Actor - state do |id| - { id: id } - end - - command( - :process_item, - list_id: String, - item_id: String, - text: String - ) do |_, cmd| - event :item_processed, list_id: cmd.payload.list_id, item_id: cmd.payload.item_id - end - - event :item_processed, list_id: String, item_id: String do |state, event| - state[:item_id] = event.payload.item_id - end - end - - # For testing sync actions - SyncLog = [] - - class SyncActor < Sourced::Actor - state do |id| - { id: id } - end - - command CreateThing do |state, cmd| - event ThingCreated, name: cmd.payload.name - end - - event ThingCreated do |state, event| - state[:name] = event.payload.name - end - - sync do |command:, events:, state:| - SyncLog << { command: command.class, events: events.map(&:class), state: state } - end - end -end diff --git a/spec/sync_spec.rb b/spec/sync_spec.rb deleted file mode 100644 index 7e0b2ae7..00000000 --- a/spec/sync_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::Sync do - let(:host) do - Class.new do - include Sourced::Sync - - attr_reader :calls1, :calls2 - - def initialize - @calls1 = [] - @calls2 = [] - end - - sync do |name:, age:| - @calls1 << [name, age] - end - - sync do |name:, age:| - @calls2 << age + 2 - end - end - end - - context 'with Procs' do - it 'returns sync blocks bound to host instance and arguments' do - object = host.new - blocks = object.sync_blocks_with(name: 'Joe', age: 30) - blocks.each(&:call) - expect(object.calls1).to eq([['Joe', 30]]) - expect(object.calls2).to eq([32]) - end - end - - context 'with custom #call interfaces' do - it 'returns sync blocks bound to passed arguments' do - host = Class.new do - include Sourced::Sync - end - - collaborator = Struct.new(:args) do - def call(**args) - self.args = args - end - end - - synccer = collaborator.new(nil) - - host.sync synccer - - object = host.new - blocks = object.sync_blocks_with(name: 'Joe', age: 30) - blocks.each(&:call) - expect(synccer.args).to eq(name: 'Joe', age: 30) - end - end -end diff --git a/spec/testing/rspec_spec.rb b/spec/testing/rspec_spec.rb index f55f6305..871520cb 100644 --- a/spec/testing/rspec_spec.rb +++ b/spec/testing/rspec_spec.rb @@ -1,305 +1,290 @@ # frozen_string_literal: true require 'spec_helper' +require 'sourced' +require 'sourced/testing/rspec' -module Testing - Start = Sourced::Message.define('sourced.testing.start') do +# Reuse message definitions from decider_spec and projector_spec +module GWTTestMessages + DeviceRegistered = Sourced::Message.define('gwt_test.device.registered') do + attribute :device_id, String attribute :name, String end - Started = Sourced::Message.define('sourced.testing.started') do + DeviceBound = Sourced::Message.define('gwt_test.device.bound') do + attribute :device_id, String + attribute :asset_id, String + end + + BindDevice = Sourced::Message.define('gwt_test.bind_device') do + attribute :device_id, String + attribute :asset_id, String + end + + NotifyBound = Sourced::Message.define('gwt_test.notify_bound') do + attribute :device_id, String + end + + NoopCommand = Sourced::Message.define('gwt_test.noop_command') do + attribute :device_id, String + end + + ItemAdded = Sourced::Message.define('gwt_test.item.added') do + attribute :list_id, String + attribute :name, String + end + + ItemArchived = Sourced::Message.define('gwt_test.item.archived') do + attribute :list_id, String attribute :name, String end - class Reactor - extend Sourced::Consumer + NotifyArchive = Sourced::Message.define('gwt_test.notify_archive') do + attribute :list_id, String + end +end - def self.handled_messages = [Start] +class GWTTestDecider < Sourced::Decider + partition_by :device_id + consumer_group 'gwt-test-decider' - def self.handle(message, history: []) - actions = [] - if Start === message && history.none? { |m| Started === m } - actions << Sourced::Actions::AppendNext.new([message.follow(Started, name: message.payload.name)]) - end - actions - end + state { |_| { exists: false, bound: false } } + + evolve GWTTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true end - class Order < Sourced::Actor - state do |id| - { id:, name: nil} - end + evolve GWTTestMessages::DeviceBound do |state, _evt| + state[:bound] = true + end + + command GWTTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + raise 'Already bound' if state[:bound] + event GWTTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end + + command GWTTestMessages::NoopCommand do |_state, _cmd| + # intentionally produces no events + end + + reaction GWTTestMessages::DeviceBound do |_state, evt| + GWTTestMessages::NotifyBound.new(payload: { device_id: evt.payload.device_id }) + end + + sync do |state:, messages:, events:| + state[:synced] = true + end + + after_sync do |state:, messages:, events:| + state[:after_synced] = true + end +end + +# Decider without reactions (produces only events) +class GWTTestSimpleDecider < Sourced::Decider + partition_by :device_id + consumer_group 'gwt-test-simple-decider' + + state { |_| { exists: false } } + + evolve GWTTestMessages::DeviceRegistered do |state, _evt| + state[:exists] = true + end + + command GWTTestMessages::BindDevice do |state, cmd| + raise 'Not found' unless state[:exists] + event GWTTestMessages::DeviceBound, device_id: cmd.payload.device_id, asset_id: cmd.payload.asset_id + end +end + +class GWTTestStateStoredProjector < Sourced::Projector::StateStored + partition_by :list_id + consumer_group 'gwt-test-ss-projector' - command Start do |state, cmd| - if state[:name].nil? - event Started, cmd.payload - end + state do |(list_id)| + { list_id: list_id, items: [], synced: false, after_synced: false } + end + + evolve GWTTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end + + evolve GWTTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end +end + +class GWTTestEventSourcedProjector < Sourced::Projector::EventSourced + partition_by :list_id + consumer_group 'gwt-test-es-projector' + + state do |(list_id)| + { list_id: list_id, items: [], synced: false, after_synced: false } + end + + evolve GWTTestMessages::ItemAdded do |state, msg| + state[:items] << msg.payload.name + end + + evolve GWTTestMessages::ItemArchived do |state, msg| + state[:items].delete(msg.payload.name) + end + + sync do |state:, messages:, replaying:| + state[:synced] = true + end + + after_sync do |state:, messages:, replaying:| + state[:after_synced] = true + end +end + +RSpec.describe Sourced::Testing::RSpec do + include Sourced::Testing::RSpec + + describe 'Decider' do + it 'given history + when command → then expected messages (event + reaction)' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then( + GWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), + GWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) + ) end - event Started do |state, evt| - state[:name] = evt.payload.name + it 'then with shorthand (Class, **payload) for single expected message' do + with_reactor(GWTTestSimpleDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then(GWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') end - command :start_payment do |_, cmd| - if state[:name] - event :payment_started - end + it 'no given + when command → then exception (invariant violation)' do + with_reactor(GWTTestDecider, device_id: 'd1') + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then(RuntimeError, 'Not found') end - event :payment_started + it 'then with block form yields action pairs' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then { |r| + expect(r.pairs).to be_a(Array) + actions, _source = r.pairs.first + append_actions = Array(actions).select { |a| a.respond_to?(:messages) } + expect(append_actions).not_to be_empty + } + end - reaction :payment_started do |_, evt| - dispatch(Payment::Process).to("#{evt.stream_id}-payment") + it 'then with [] expects no messages' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(GWTTestMessages::NoopCommand, device_id: 'd1') + .then([]) end - end - class Payment < Sourced::Actor - command :process do |_, cmd| - event :processed + it 'then! runs sync and after_sync actions' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then! { |r| + expect(r.pairs).to be_a(Array) + } end - event :processed - end + it 'given with message instances' do + reg = GWTTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) - class Telemetry - STREAM_ID = 'telemetry-stream' - include Sourced::Handler + with_reactor(GWTTestDecider, device_id: 'd1') + .given(reg) + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') + .then( + GWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), + GWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) + ) + end - Logged = Sourced::Message.define('test-telemetry.logged') do - attribute :source_stream, String - attribute :message_type, String + it 'supports .and as alias for .given' do + with_reactor(GWTTestDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .and(GWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') + .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a2') + .then(RuntimeError, 'Already bound') end + end - consumer do |c| - c.group_id = 'test-telemetry' + describe 'Projector (StateStored)' do + it 'given events → then block asserts evolved state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then { |r| expect(r.state[:items]).to eq(['Apple']) } end - on Order::PaymentStarted, Payment::Processed do |event| - logged = Logged.build(STREAM_ID, source_stream: event.stream_id, message_type: event.type) - [logged] + it 'given multiple events → then block sees cumulative state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') + .then { |r| expect(r.state[:items]).to eq(['Apple', 'Banana']) } end - end -end -RSpec.describe Sourced::Testing::RSpec do - describe 'with_reactor' do - context 'with Reactor interface' do - it 'works' do - with_reactor(Testing::Reactor, 'a') - .when(Testing::Start, name: 'Joe') - .then(Testing::Started.build('a', name: 'Joe')) - - with_reactor(Testing::Reactor, 'a') - .given(Testing::Started, name: 'Joe') - .when(Testing::Start, name: 'Joe') - .then([]) - - # If supports any .handle() interface, including u classes - with_reactor(Testing::Order, 'a') - .when(Testing::Start, name: 'Joe') - .then(Testing::Started.build('a', name: 'Joe')) - end + it 'then! runs sync actions before yielding state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |r| expect(r.state[:synced]).to be true } end - context 'with Actor instance' do - it 'works' do - with_reactor(Testing::Order.new(id: 'a')) - .when(Testing::Start, name: 'Joe') - .then(Testing::Started.build('a', name: 'Joe')) - - with_reactor(Testing::Order.new(id: 'a')) - .when(Testing::Start, name: 'Joe') - .then(Testing::Started, name: 'Joe') - - with_reactor(Testing::Order.new(id: 'a')) - .when(Testing::Start, name: 'Joe') - .then([Testing::Started.build('a', name: 'Joe')]) - - with_reactor(Testing::Order.new(id: 'a')) - .given(Testing::Started, name: 'Joe') - .when(Testing::Start, name: 'Joe') - .then([]) - end + it 'then! runs after_sync actions before yielding state' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |r| expect(r.state[:after_synced]).to be true } end - context 'when expecting an exception' do - it 'passes when the expected exception is raised' do - error_class = Class.new(StandardError) - - klass = Class.new do - extend Sourced::Consumer - def self.handled_messages = [Testing::Start] - end - klass.define_singleton_method(:handle) do |message, history:| - raise error_class, 'boom' - end - - with_reactor(klass, 'a') - .when(Testing::Start, name: 'Joe') - .then(error_class) - end - - it 'fails when expected exception is not raised' do - error_class = Class.new(StandardError) - - with_reactor(Testing::Reactor, 'a') - .when(Testing::Start, name: 'Joe') - .then(Testing::Started.build('a', name: 'Joe')) - - expect { - with_reactor(Testing::Reactor, 'a') - .when(Testing::Start, name: 'Joe') - .then(error_class) - }.to raise_error(RSpec::Expectations::ExpectationNotMetError, /expected .* to be raised/) - end - - it 'works with Actor instances' do - error_class = Class.new(StandardError) - - klass = Class.new(Sourced::Actor) do - state do |id| - { id: } - end - - command Testing::Start do |state, cmd| - raise error_class, 'not allowed' - end - end - - with_reactor(klass.new(id: 'a')) - .when(Testing::Start, name: 'Joe') - .then(error_class) - end - - context 'with exception instance' do - it 'matches on class and message' do - error_class = Class.new(StandardError) - - klass = Class.new do - extend Sourced::Consumer - def self.handled_messages = [Testing::Start] - end - klass.define_singleton_method(:handle) do |message, history:| - raise error_class, 'specific message' - end - - with_reactor(klass, 'a') - .when(Testing::Start, name: 'Joe') - .then(error_class.new('specific message')) - end - - it 'fails when message does not match' do - error_class = Class.new(StandardError) - - klass = Class.new do - extend Sourced::Consumer - def self.handled_messages = [Testing::Start] - end - klass.define_singleton_method(:handle) do |message, history:| - raise error_class, 'actual message' - end - - expect { - with_reactor(klass, 'a') - .when(Testing::Start, name: 'Joe') - .then(error_class.new('expected message')) - }.to raise_error( - RSpec::Expectations::ExpectationNotMetError, - /expected .* with message "expected message", but got "actual message"/ - ) - end - end + it 'given events with archive → state reflects removal' do + with_reactor(GWTTestStateStoredProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .and(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') + .and(GWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') + .then { |r| expect(r.state[:items]).to eq(['Banana']) } end + end - specify 'it raises when adding events after assertion' do - expect { - with_reactor(Testing::Reactor, 'a') - .given(Testing::Started, name: 'Joe') - .when(Testing::Start, name: 'Joe') - .then([]) - .given(Testing::Started, name: 'Joe') # <= can't add more state after .then() assertion - }.to raise_error(Sourced::Testing::RSpec::FinishedTestCase) + describe 'Projector (EventSourced)' do + it 'given events → then block asserts evolved state' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Banana') + .then { |r| expect(r.state[:items]).to eq(['Apple', 'Banana']) } end - context 'with block given to #then' do - it 'evaluates block' do - received = [] - - klass = Class.new do - extend Sourced::Consumer - def self.handled_messages = [Testing::Start] - end - klass.define_singleton_method(:handle) do |message, history:| - received << message - [] - end - - with_reactor(klass, 'abc') - .when(Testing::Start, name: 'Joe') - .then do |actions| - expect(actions).to eq([]) - expect(received).to match_sourced_messages(Testing::Start.build('abc', name: 'Joe')) - end - end + it 'given events with archive → state reflects removal' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .and(GWTTestMessages::ItemArchived, list_id: 'L1', name: 'Apple') + .then { |r| expect(r.state[:items]).to eq([]) } end - describe '.then!' do - it 'evaluates sync blocks' do - received = [] - - klass = Class.new do - extend Sourced::Consumer - def self.handled_messages = [Testing::Start] - end - klass.define_singleton_method(:handle) do |message, history:| - sync = proc do - received << 10 - end - started = message.follow(Testing::Started, message.payload) - [ - Sourced::Actions::Sync.new(sync), - Sourced::Actions::AppendNext.new([started]) - ] - end - - with_reactor(klass, 'abc') - .when(Testing::Start, name: 'Joe') - .then! do |actions| - expect(actions.first).to be_a(Sourced::Actions::Sync) - expect(received).to eq([10]) - end - .then(Testing::Started.build('abc', name: 'Joe')) - end + it 'then! runs sync actions before yielding state' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |r| expect(r.state[:synced]).to be true } end - end - describe 'with_reactors' do - it 'tests collaboration of reactors' do - order_stream = 'actor-1' - payment_stream = 'actor-1-payment' - telemetry_stream = Testing::Telemetry::STREAM_ID - - # With these reactors - with_reactors(Testing::Order, Testing::Payment, Testing::Telemetry) - # GIVEN that these events exist in history - .given(Testing::Started.build(order_stream, name: 'foo')) - # WHEN I dispatch this new command - .when(Testing::Order::StartPayment.build(order_stream)) - # Then I expect - .then do |_, new_messages| - # The different reactors collaborated and - # left this message trail behind - # Backend#messages is only available in the TestBackend - expect(new_messages).to match_sourced_messages([ - Testing::Started.build(order_stream, name: 'foo'), - Testing::Order::StartPayment.build(order_stream), - Testing::Order::PaymentStarted.build(order_stream), - Testing::Telemetry::Logged.build(telemetry_stream, source_stream: order_stream, message_type: 'testing.order.payment_started'), - Testing::Payment::Process.build(payment_stream), - Testing::Payment::Processed.build(payment_stream), - Testing::Telemetry::Logged.build(telemetry_stream, source_stream: payment_stream, message_type: 'testing.payment.processed'), - ]) - end + it 'then! runs after_sync actions before yielding state' do + with_reactor(GWTTestEventSourcedProjector, list_id: 'L1') + .given(GWTTestMessages::ItemAdded, list_id: 'L1', name: 'Apple') + .then! { |r| expect(r.state[:after_synced]).to be true } end + end end diff --git a/spec/thread_executor_spec.rb b/spec/thread_executor_spec.rb deleted file mode 100644 index bd5ee39d..00000000 --- a/spec/thread_executor_spec.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/thread_executor' - -RSpec.describe Sourced::ThreadExecutor, type: :executor do - subject(:executor) { described_class.new } - - it_behaves_like 'an executor' -end diff --git a/spec/topology_spec.rb b/spec/topology_spec.rb index 2a3bd566..b97082ac 100644 --- a/spec/topology_spec.rb +++ b/spec/topology_spec.rb @@ -1,7 +1,156 @@ # frozen_string_literal: true require 'spec_helper' -require_relative 'support/unit_test_fixtures' +require 'sourced' + +module TopologyTest + # --- Messages --- + CreateWidget = Sourced::Command.define('ccc_topo.create_widget') do + attribute :widget_id, String + attribute :name, String + end + + WidgetCreated = Sourced::Event.define('ccc_topo.widget_created') do + attribute :widget_id, String + attribute :name, String + end + + NotifyWidget = Sourced::Command.define('ccc_topo.notify_widget') do + attribute :widget_id, String + end + + WidgetNotified = Sourced::Event.define('ccc_topo.widget_notified') do + attribute :widget_id, String + end + + ArchiveWidget = Sourced::Command.define('ccc_topo.archive_widget') do + attribute :widget_id, String + end + + WidgetArchived = Sourced::Event.define('ccc_topo.widget_archived') do + attribute :widget_id, String + end + + DelayedCmd = Sourced::Command.define('ccc_topo.delayed_cmd') do + attribute :widget_id, String + end + + ScheduleEvent = Sourced::Event.define('ccc_topo.schedule_event') do + attribute :widget_id, String + end + + # --- Decider --- + class WidgetDecider < Sourced::Decider + partition_by :widget_id + + state { |_| { exists: false } } + + evolve WidgetCreated do |state, _evt| + state[:exists] = true + end + + evolve WidgetArchived do |state, _evt| + state[:exists] = false + end + + command CreateWidget do |_state, cmd| + event WidgetCreated, widget_id: cmd.payload.widget_id, name: cmd.payload.name + end + + command ArchiveWidget do |_state, _cmd| + event WidgetArchived, widget_id: 'x' + end + + reaction WidgetCreated do |_state, evt| + dispatch NotifyWidget, widget_id: evt.payload.widget_id + end + end + + # --- Notifier Decider --- + class NotifierDecider < Sourced::Decider + partition_by :widget_id + + state { |_| {} } + + evolve WidgetNotified do |state, _evt| + state[:notified] = true + end + + command NotifyWidget do |_state, cmd| + event WidgetNotified, widget_id: cmd.payload.widget_id + end + end + + # --- Projector (StateStored, no reactions) --- + class WidgetListProjector < Sourced::Projector::StateStored + partition_by :widget_id + consumer_group 'TopologyTest::WidgetListProjector' + + state { |_| { items: [] } } + + evolve WidgetCreated do |state, msg| + state[:items] << msg.payload.name + end + end + + # --- Projector (EventSourced, with catch-all reaction) --- + class ReactingProjector < Sourced::Projector::EventSourced + partition_by :widget_id + consumer_group 'TopologyTest::ReactingProjector' + + state { |_| { items: [] } } + + evolve WidgetCreated do |state, msg| + state[:items] << msg.payload.name + end + + reaction do |_state, evt| + dispatch NotifyWidget, widget_id: 'x' + end + end + + # --- Projector with specific + catch-all reactions --- + class MixedReactingProjector < Sourced::Projector::EventSourced + partition_by :widget_id + consumer_group 'TopologyTest::MixedReactingProjector' + + state { |_| { items: [] } } + + evolve WidgetCreated do |state, msg| + state[:items] << msg.payload.name + end + + evolve WidgetArchived do |_state, _msg| + end + + reaction WidgetCreated do |_state, evt| + dispatch NotifyWidget, widget_id: evt.payload.widget_id + end + + reaction do |_state, _evt| + dispatch ArchiveWidget, widget_id: 'x' + end + end + + # --- Decider with chained dispatch (.at) --- + class SchedulingDecider < Sourced::Decider + partition_by :widget_id + + state { |_| {} } + + evolve ScheduleEvent do |state, _evt| + state[:scheduled] = true + end + + command CreateWidget do |_state, cmd| + event ScheduleEvent, widget_id: cmd.payload.widget_id + end + + reaction ScheduleEvent do |_state, evt| + dispatch(DelayedCmd, widget_id: evt.payload.widget_id).at(evt.created_at + 300) + end + end +end RSpec.describe Sourced::Topology do let(:nodes) { described_class.build(reactors) } @@ -14,99 +163,94 @@ def find_nodes_by_type(type) nodes.select { |n| n.type == type } end - context 'with ThingActor and NotifierActor' do - let(:reactors) { [UnitTest::ThingActor, UnitTest::NotifierActor] } + context 'with WidgetDecider and NotifierDecider' do + let(:reactors) { [TopologyTest::WidgetDecider, TopologyTest::NotifierDecider] } it 'builds command nodes for handled commands' do cmd_nodes = find_nodes_by_type('command') expect(cmd_nodes.map(&:id)).to contain_exactly( - 'unittest.create_thing', - 'unittest.notify_thing' + 'ccc_topo.create_widget', + 'ccc_topo.archive_widget', + 'ccc_topo.notify_widget' ) end it 'sets correct group_id on command nodes' do - node = find_node('unittest.create_thing') - expect(node.group_id).to eq('UnitTest::ThingActor') + node = find_node('ccc_topo.create_widget') + expect(node.group_id).to eq('TopologyTest::WidgetDecider') end it 'extracts produced events via Prism' do - node = find_node('unittest.create_thing') - expect(node.produces).to eq(['unittest.thing_created']) + node = find_node('ccc_topo.create_widget') + expect(node.produces).to eq(['ccc_topo.widget_created']) end - it 'extracts produced events for NotifierActor' do - node = find_node('unittest.notify_thing') - expect(node.produces).to eq(['unittest.thing_notified']) + it 'extracts produced events for NotifierDecider' do + node = find_node('ccc_topo.notify_widget') + expect(node.produces).to eq(['ccc_topo.widget_notified']) end it 'sets command name from message class' do - node = find_node('unittest.create_thing') - expect(node.name).to eq('UnitTest::CreateThing') + node = find_node('ccc_topo.create_widget') + expect(node.name).to eq('TopologyTest::CreateWidget') end it 'extracts schema from command payload' do - node = find_node('unittest.create_thing') - expect(node.schema).to include( - 'type' => 'object', - 'properties' => { 'name' => { 'type' => 'string' } } - ) + node = find_node('ccc_topo.create_widget') + expect(node.schema).to include('type' => 'object') + expect(node.schema['properties']).to include('widget_id', 'name') end it 'builds event nodes deduplicated by type' do evt_nodes = find_nodes_by_type('event') evt_types = evt_nodes.map(&:id) expect(evt_types).to contain_exactly( - 'unittest.thing_created', - 'unittest.thing_notified' + 'ccc_topo.widget_created', + 'ccc_topo.widget_archived', + 'ccc_topo.widget_notified' ) end it 'assigns first-seen group_id to event nodes' do - node = find_node('unittest.thing_created') - expect(node.group_id).to eq('UnitTest::ThingActor') + node = find_node('ccc_topo.widget_created') + expect(node.group_id).to eq('TopologyTest::WidgetDecider') end it 'event nodes have empty produces' do - evt_nodes = find_nodes_by_type('event') - evt_nodes.each do |n| - expect(n.produces).to eq([]) - end + find_nodes_by_type('event').each { |n| expect(n.produces).to eq([]) } end it 'extracts schema from event payload' do - node = find_node('unittest.thing_created') - expect(node.schema).to include( - 'type' => 'object', - 'properties' => { 'name' => { 'type' => 'string' } } - ) + node = find_node('ccc_topo.widget_created') + expect(node.schema).to include('type' => 'object') + expect(node.schema['properties']).to include('widget_id', 'name') end it 'builds automation nodes for reactions' do aut_nodes = find_nodes_by_type('automation') expect(aut_nodes.map(&:id)).to include( - 'unittest.thing_created-UnitTest::ThingActor-aut' + 'ccc_topo.widget_created-TopologyTest::WidgetDecider-aut' ) end it 'sets correct consumes on automation nodes' do - node = find_node('unittest.thing_created-UnitTest::ThingActor-aut') - expect(node.consumes).to eq(['unittest.thing_created']) + node = find_node('ccc_topo.widget_created-TopologyTest::WidgetDecider-aut') + expect(node.consumes).to eq(['ccc_topo.widget_created']) end it 'extracts dispatched commands from reactions via Prism' do - node = find_node('unittest.thing_created-UnitTest::ThingActor-aut') - expect(node.produces).to eq(['unittest.notify_thing']) + node = find_node('ccc_topo.widget_created-TopologyTest::WidgetDecider-aut') + expect(node.produces).to eq(['ccc_topo.notify_widget']) end it 'sets automation name from event class' do - node = find_node('unittest.thing_created-UnitTest::ThingActor-aut') - expect(node.name).to eq('reaction(UnitTest::ThingCreated)') + node = find_node('ccc_topo.widget_created-TopologyTest::WidgetDecider-aut') + expect(node.name).to eq('reaction(TopologyTest::WidgetCreated)') end end - context 'with ThingProjector (no commands, no reactions)' do - let(:reactors) { [UnitTest::ThingProjector] } + context 'with WidgetListProjector (no reactions)' do + let(:reactors) { [TopologyTest::WidgetListProjector] } it 'does not build command nodes' do expect(find_nodes_by_type('command')).to be_empty @@ -118,46 +262,36 @@ def find_nodes_by_type(type) it 'builds event nodes from evolve handlers' do evt_nodes = find_nodes_by_type('event') - expect(evt_nodes.map(&:id)).to eq(['unittest.thing_created']) - end - - it 'sets projector group_id on event nodes' do - node = find_node('unittest.thing_created') - expect(node.group_id).to eq('UnitTest::ThingProjector') + expect(evt_nodes.map(&:id)).to eq(['ccc_topo.widget_created']) end it 'builds a readmodel node' do - node = find_node('unit_test.thing_projector-rm') + node = find_node('topology_test.widget_list_projector-rm') expect(node).not_to be_nil expect(node.type).to eq('readmodel') end it 'sets correct name and group_id on readmodel node' do - node = find_node('unit_test.thing_projector-rm') - expect(node.name).to eq('UnitTest::ThingProjector') - expect(node.group_id).to eq('UnitTest::ThingProjector') + node = find_node('topology_test.widget_list_projector-rm') + expect(node.name).to eq('TopologyTest::WidgetListProjector') + expect(node.group_id).to eq('TopologyTest::WidgetListProjector') end it 'readmodel consumes the evolved event types' do - node = find_node('unit_test.thing_projector-rm') - expect(node.consumes).to eq(['unittest.thing_created']) + node = find_node('topology_test.widget_list_projector-rm') + expect(node.consumes).to eq(['ccc_topo.widget_created']) end - it 'readmodel produces nothing' do - node = find_node('unit_test.thing_projector-rm') + it 'readmodel produces nothing when there are no reactions' do + node = find_node('topology_test.widget_list_projector-rm') expect(node.produces).to eq([]) end - - it 'readmodel schema is empty' do - node = find_node('unit_test.thing_projector-rm') - expect(node.schema).to eq({}) - end end - context 'with ReactingProjector (projector with catch-all reaction)' do - let(:reactors) { [UnitTest::ReactingProjector] } - let(:rm_id) { 'unit_test.reacting_projector-rm' } - let(:aut_id) { 'unit_test.reacting_projector-aut' } + context 'with ReactingProjector (catch-all reaction)' do + let(:reactors) { [TopologyTest::ReactingProjector] } + let(:rm_id) { 'topology_test.reacting_projector-rm' } + let(:aut_id) { 'topology_test.reacting_projector-aut' } it 'builds a readmodel node' do node = find_node(rm_id) @@ -167,7 +301,7 @@ def find_nodes_by_type(type) it 'readmodel consumes the evolved event types' do node = find_node(rm_id) - expect(node.consumes).to eq(['unittest.thing_created']) + expect(node.consumes).to eq(['ccc_topo.widget_created']) end it 'readmodel produces a single automation node' do @@ -180,7 +314,7 @@ def find_nodes_by_type(type) expect(aut_nodes.size).to eq(1) node = aut_nodes.first expect(node.id).to eq(aut_id) - expect(node.name).to eq('reaction(UnitTest::ReactingProjector)') + expect(node.name).to eq('reaction(TopologyTest::ReactingProjector)') end it 'automation node consumes the readmodel' do @@ -190,15 +324,15 @@ def find_nodes_by_type(type) it 'automation node produces dispatched commands' do node = find_node(aut_id) - expect(node.produces).to eq(['unittest.notify_thing']) + expect(node.produces).to eq(['ccc_topo.notify_widget']) end end context 'with MixedReactingProjector (specific + catch-all reactions)' do - let(:reactors) { [UnitTest::MixedReactingProjector] } - let(:rm_id) { 'unit_test.mixed_reacting_projector-rm' } - let(:specific_aut_id) { 'unittest.thing_created-UnitTest::MixedReactingProjector-aut' } - let(:catchall_aut_id) { 'unit_test.mixed_reacting_projector-aut' } + let(:reactors) { [TopologyTest::MixedReactingProjector] } + let(:rm_id) { 'topology_test.mixed_reacting_projector-rm' } + let(:specific_aut_id) { 'ccc_topo.widget_created-TopologyTest::MixedReactingProjector-aut' } + let(:catchall_aut_id) { 'topology_test.mixed_reacting_projector-aut' } it 'builds two automation nodes: one specific and one catch-all' do aut_nodes = find_nodes_by_type('automation') @@ -207,12 +341,12 @@ def find_nodes_by_type(type) it 'specific automation is named after the event' do node = find_node(specific_aut_id) - expect(node.name).to eq('reaction(UnitTest::ThingCreated)') + expect(node.name).to eq('reaction(TopologyTest::WidgetCreated)') end it 'catch-all automation is named after the reactor' do node = find_node(catchall_aut_id) - expect(node.name).to eq('reaction(UnitTest::MixedReactingProjector)') + expect(node.name).to eq('reaction(TopologyTest::MixedReactingProjector)') end it 'both automation nodes consume the readmodel' do @@ -228,100 +362,38 @@ def find_nodes_by_type(type) end end - context 'with SchedulingActor (chained dispatch)' do - let(:reactors) { [UnitTest::SchedulingActor] } + context 'with SchedulingDecider (chained dispatch)' do + let(:reactors) { [TopologyTest::SchedulingDecider] } it 'detects dispatch through .at() chain' do - node = find_node('unittest.schedule_event-UnitTest::SchedulingActor-aut') - expect(node).not_to be_nil - expect(node.produces).to eq(['unittest.delayed_cmd']) - end - end - - context 'with BracketDispatchActor (Reactor[:command] syntax)' do - let(:reactors) { [UnitTest::BracketDispatchActor, UnitTest::InlineNotifierActor] } - - it 'detects dispatched commands via bracket accessor syntax' do - node = find_node('unittest.thing_created-UnitTest::BracketDispatchActor-aut') + node = find_node('ccc_topo.schedule_event-TopologyTest::SchedulingDecider-aut') expect(node).not_to be_nil - expect(node.produces).to eq(['unit_test.inline_notifier_actor.notify_inline']) - end - end - - context 'with MultiLineCommandActor (multi-line command with symbol events)' do - let(:reactors) { [UnitTest::MultiLineCommandActor] } - - it 'extracts produced events from multi-line command definitions' do - node = find_node('unit_test.multi_line_command_actor.process_item') - expect(node).not_to be_nil - expect(node.produces).to eq(['unit_test.multi_line_command_actor.item_processed']) - end - end - - context 'with LoopingActor (self-referencing)' do - let(:reactors) { [UnitTest::LoopingActor] } - - it 'detects self-dispatched commands' do - node = find_node('unittest.loop_event-UnitTest::LoopingActor-aut') - expect(node).not_to be_nil - expect(node.produces).to eq(['unittest.loop_cmd']) - end - - it 'produces both command and event nodes' do - expect(find_node('unittest.loop_cmd')).not_to be_nil - expect(find_node('unittest.loop_event')).not_to be_nil + expect(node.produces).to eq(['ccc_topo.delayed_cmd']) end end context 'event deduplication across reactors' do - let(:reactors) { [UnitTest::ThingActor, UnitTest::ThingProjector] } + let(:reactors) { [TopologyTest::WidgetDecider, TopologyTest::WidgetListProjector] } it 'deduplicates event nodes by type string' do - evt_nodes = find_nodes_by_type('event').select { |n| n.id == 'unittest.thing_created' } + evt_nodes = find_nodes_by_type('event').select { |n| n.id == 'ccc_topo.widget_created' } expect(evt_nodes.size).to eq(1) end it 'uses first reactor as group_id owner' do - node = find_node('unittest.thing_created') - expect(node.group_id).to eq('UnitTest::ThingActor') + node = find_node('ccc_topo.widget_created') + expect(node.group_id).to eq('TopologyTest::WidgetDecider') end end context 'command deduplication across reactors' do - let(:reactors) { [UnitTest::ThingActor, UnitTest::SyncActor] } + let(:reactors) { [TopologyTest::WidgetDecider, TopologyTest::SchedulingDecider] } it 'deduplicates command nodes by type string' do - cmd_nodes = find_nodes_by_type('command').select { |n| n.id == 'unittest.create_thing' } + cmd_nodes = find_nodes_by_type('command').select { |n| n.id == 'ccc_topo.create_widget' } expect(cmd_nodes.size).to eq(1) end - it 'uses first reactor as group_id owner for commands' do - node = find_node('unittest.create_thing') - expect(node.group_id).to eq('UnitTest::ThingActor') - end - - it 'produces no duplicate IDs' do - ids = nodes.map(&:id) - expect(ids).to eq(ids.uniq) - end - end - - context 'when handled_messages_for_evolve contains a command class' do - let(:reactors) { [UnitTest::ThingActor] } - - around do |example| - # Temporarily inject a command class into handled_messages_for_evolve - UnitTest::ThingActor.handled_messages_for_evolve << UnitTest::CreateThing - example.run - ensure - UnitTest::ThingActor.handled_messages_for_evolve.delete(UnitTest::CreateThing) - end - - it 'skips command classes and does not create event nodes for them' do - event_ids = find_nodes_by_type('event').map(&:id) - expect(event_ids).not_to include('unittest.create_thing') - end - it 'produces no duplicate IDs' do ids = nodes.map(&:id) expect(ids).to eq(ids.uniq) @@ -331,12 +403,11 @@ def find_nodes_by_type(type) context 'with all test reactors' do let(:reactors) do [ - UnitTest::ThingActor, - UnitTest::NotifierActor, - UnitTest::ThingProjector, - UnitTest::ReactingProjector, - UnitTest::SchedulingActor, - UnitTest::LoopingActor + TopologyTest::WidgetDecider, + TopologyTest::NotifierDecider, + TopologyTest::WidgetListProjector, + TopologyTest::ReactingProjector, + TopologyTest::SchedulingDecider ] end @@ -348,9 +419,7 @@ def find_nodes_by_type(type) end it 'all command nodes have produces arrays' do - find_nodes_by_type('command').each do |n| - expect(n.produces).to be_an(Array) - end + find_nodes_by_type('command').each { |n| expect(n.produces).to be_an(Array) } end it 'all automation nodes have consumes and produces arrays' do diff --git a/spec/unit_sequel_postgres_spec.rb b/spec/unit_sequel_postgres_spec.rb deleted file mode 100644 index 47330e6a..00000000 --- a/spec/unit_sequel_postgres_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'sourced/backends/pg_backend' -require_relative 'support/unit_test_fixtures' -require_relative 'shared_examples/unit_examples' - -RSpec.describe 'Sourced::Unit with PGBackend (Postgres)' do - let(:db) { Sequel.postgres('sourced_test') } - let(:backend) do - b = Sourced::Backends::PGBackend.new(db) - b.setup!(Sourced.config) - b - end - - before do - backend.uninstall if backend.installed? - backend.install - end - - it_behaves_like 'a unit' -end diff --git a/spec/unit_spec.rb b/spec/unit_spec.rb deleted file mode 100644 index ef7992e5..00000000 --- a/spec/unit_spec.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'support/unit_test_fixtures' -require_relative 'shared_examples/unit_examples' - -RSpec.describe Sourced::Unit do - describe 'with TestBackend' do - let(:backend) { Sourced::Backends::TestBackend.new } - - it_behaves_like 'a unit' - end -end diff --git a/spec/work_queue_spec.rb b/spec/work_queue_spec.rb deleted file mode 100644 index 7a4e1978..00000000 --- a/spec/work_queue_spec.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::WorkQueue do - let(:queue) { described_class.new(max_per_reactor: 2, queue: Queue.new) } - let(:reactor_a) { double('ReactorA') } - let(:reactor_b) { double('ReactorB') } - - describe '#push / #pop' do - it 'enqueues and dequeues a reactor' do - queue.push(reactor_a) - expect(queue.pop).to eq(reactor_a) - end - - it 'returns reactors in FIFO order' do - queue.push(reactor_a) - queue.push(reactor_b) - expect(queue.pop).to eq(reactor_a) - expect(queue.pop).to eq(reactor_b) - end - end - - describe 'cap enforcement' do - it 'rejects pushes beyond max_per_reactor' do - expect(queue.push(reactor_a)).to eq(true) - expect(queue.push(reactor_a)).to eq(true) - expect(queue.push(reactor_a)).to eq(false) # at cap - - # Different reactor is still allowed - expect(queue.push(reactor_b)).to eq(true) - end - - it 'allows pushing again after pop decrements the count' do - queue.push(reactor_a) - queue.push(reactor_a) - expect(queue.push(reactor_a)).to eq(false) - - queue.pop # decrements count - expect(queue.push(reactor_a)).to eq(true) - end - end - - describe '#close' do - it 'pushes nil sentinels to unblock workers' do - queue.close(3) - expect(queue.pop).to be_nil - expect(queue.pop).to be_nil - expect(queue.pop).to be_nil - end - end - - describe 'concurrent push/pop safety' do - it 'handles concurrent access without errors' do - q = described_class.new(max_per_reactor: 100, queue: Queue.new) - results = [] - mutex = Mutex.new - - producers = 5.times.map do - Thread.new do - 20.times { q.push(reactor_a) } - end - end - - consumers = 5.times.map do - Thread.new do - 20.times do - r = q.pop - mutex.synchronize { results << r } - end - end - end - - producers.each(&:join) - consumers.each(&:join) - - expect(results.size).to eq(100) - expect(results).to all(eq(reactor_a)) - end - end -end diff --git a/spec/worker_spec.rb b/spec/worker_spec.rb deleted file mode 100644 index e1de5082..00000000 --- a/spec/worker_spec.rb +++ /dev/null @@ -1,138 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Sourced::Worker do - let(:router) { instance_double(Sourced::Router) } - let(:logger) { instance_double('Logger', info: nil) } - let(:work_queue) { Sourced::WorkQueue.new(max_per_reactor: 2, queue: Queue.new) } - let(:reactor1) { double('Reactor1') } - let(:reactor2) { double('Reactor2') } - - before do - allow(router).to receive(:handle_next_event_for_reactor).and_return(false) - end - - describe '#tick' do - subject(:worker) do - described_class.new( - work_queue: work_queue, - logger: logger, - name: 'test-worker', - router: router - ) - end - - it 'delegates to Router#handle_next_event_for_reactor with given reactor and worker name' do - expect(router).to receive(:handle_next_event_for_reactor) - .with(reactor1, worker.name, batch_size: 1) - .and_return(true) - - result = worker.tick(reactor1) - expect(result).to eq(true) - end - - it 'returns the result from Router#handle_next_event_for_reactor' do - allow(router).to receive(:handle_next_event_for_reactor).and_return(false) - expect(worker.tick(reactor1)).to eq(false) - - allow(router).to receive(:handle_next_event_for_reactor).and_return(true) - expect(worker.tick(reactor1)).to eq(true) - end - end - - describe '#run' do - subject(:worker) do - described_class.new( - work_queue: work_queue, - logger: logger, - name: 'test-worker', - router: router, - max_drain_rounds: 3 - ) - end - - it 'pops reactors from the work queue and drains them' do - call_count = 0 - allow(router).to receive(:handle_next_event_for_reactor) do - call_count += 1 - call_count <= 2 # return true for first 2, false for 3rd - end - - work_queue.push(reactor1) - - # Run in a thread, then stop after processing - t = Thread.new { worker.run } - sleep 0.05 # give worker time to drain - worker.stop - work_queue.close(1) # unblock the pop - t.join(1) - - expect(call_count).to be >= 1 - end - - it 'stops on nil sentinel (shutdown)' do - work_queue.close(1) - t = Thread.new { worker.run } - t.join(1) - expect(t.alive?).to eq(false) - end - end - - describe '#drain' do - subject(:worker) do - described_class.new( - work_queue: work_queue, - logger: logger, - name: 'test-worker', - router: router, - max_drain_rounds: 3 - ) - end - - before do - # Mark as running so drain loop operates - worker.instance_variable_set(:@running, true) - end - - it 'processes messages until none found' do - call_count = 0 - allow(router).to receive(:handle_next_event_for_reactor) do - call_count += 1 - call_count <= 2 - end - - worker.drain(reactor1) - - # Should have called 3 times: true, true, false - expect(call_count).to eq(3) - end - - it 'stops at max_drain_rounds and re-enqueues the reactor' do - allow(router).to receive(:handle_next_event_for_reactor).and_return(true) - - worker.drain(reactor1) - - # Should have called exactly max_drain_rounds times - expect(router).to have_received(:handle_next_event_for_reactor).exactly(3).times - - # Reactor should be re-enqueued - expect(work_queue.pop).to eq(reactor1) - end - - it 'does not re-enqueue when fewer than max_drain_rounds processed' do - call_count = 0 - allow(router).to receive(:handle_next_event_for_reactor) do - call_count += 1 - call_count <= 1 # only 1 message found - end - - worker.drain(reactor1) - - # Work queue should be empty (no re-enqueue) - # Push and pop to verify nothing was there before - work_queue.push(reactor2) - expect(work_queue.pop).to eq(reactor2) - end - end -end From 05bf988ce7d42121e831baba70470376c0c9f9d5 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 12:48:34 +0100 Subject: [PATCH 104/115] Simplify --- lib/sourced.rb | 27 +++++++++++---------------- lib/sourced/decider.rb | 6 ++++-- lib/sourced/message.rb | 1 - lib/sourced/topology.rb | 2 +- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/lib/sourced.rb b/lib/sourced.rb index b3cc79d3..e64cc6be 100644 --- a/lib/sourced.rb +++ b/lib/sourced.rb @@ -56,6 +56,7 @@ def self.setup! def self.register(reactor) config.setup! config.router.register(reactor) + @topology = nil end # @return [Sourced::Store] @@ -112,19 +113,19 @@ def to_ary = [command, reactor, events] # Handle a command synchronously: validate, load history, decide, append, ACK. def self.handle!(reactor_class, command, store: nil) - store ||= self.store - partition_attrs = extract_partition_attrs(command, reactor_class) values = reactor_class.partition_keys.map { |k| partition_attrs[k]&.to_s } - instance = reactor_class.new(values) unless command.valid? - return HandleResult.new(command: command, reactor: instance, events: []) + return HandleResult.new(command: command, reactor: reactor_class.new(values), events: []) end + store ||= self.store needs_history = Injector.resolve_args(reactor_class, :handle_claim).include?(:history) if needs_history instance, read_result = load(reactor_class, store: store, **partition_attrs) + else + instance = reactor_class.new(values) end raw_events = instance.decide(command) @@ -160,19 +161,13 @@ def self.load(reactor_class, store: nil, **values) end private_class_method def self.advance_registered_offsets(store, reactor_class, partition_attrs, position) - return unless config.router - - partition = partition_attrs.transform_keys(&:to_s) + return unless config.router&.reactors&.include?(reactor_class) - config.router.reactors.each do |registered_reactor| - next unless registered_reactor == reactor_class - - store.advance_offset( - registered_reactor.group_id, - partition: partition, - position: position - ) - end + store.advance_offset( + reactor_class.group_id, + partition: partition_attrs.transform_keys(&:to_s), + position: position + ) end end diff --git a/lib/sourced/decider.rb b/lib/sourced/decider.rb index e389f973..917ab4db 100644 --- a/lib/sourced/decider.rb +++ b/lib/sourced/decider.rb @@ -8,6 +8,8 @@ class Decider include Sourced::Sync extend Sourced::Consumer + PREFIX = 'sourced_decide' + class << self # @return [Array] command message classes handled by this decider def handled_commands @@ -29,7 +31,7 @@ def handled_messages # @return [void] def command(message_class, &block) handled_commands << message_class - define_method(Sourced.message_method_name('sourced_decide', message_class.to_s), &block) + define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) end def handle_batch(partition_values, new_messages, history:, replaying: false) @@ -98,7 +100,7 @@ def initialize(partition_values = {}) # @return [Array] newly produced events def decide(command) @uncommitted_events = [] - method_name = Sourced.message_method_name('sourced_decide', command.class.to_s) + method_name = Sourced.message_method_name(PREFIX, command.class.to_s) send(method_name, state, command) if respond_to?(method_name) @uncommitted_events.dup end diff --git a/lib/sourced/message.rb b/lib/sourced/message.rb index bd9d66db..c0b1e492 100644 --- a/lib/sourced/message.rb +++ b/lib/sourced/message.rb @@ -35,7 +35,6 @@ class Message < Types::Data attribute :payload, Types::Static[nil] # Lookup table mapping type strings to message subclasses. - # Separate from {Sourced::Message}'s registry. class Registry # @param message_class [Class] the root message class for this registry def initialize(message_class) diff --git a/lib/sourced/topology.rb b/lib/sourced/topology.rb index 489b7eae..9f2771de 100644 --- a/lib/sourced/topology.rb +++ b/lib/sourced/topology.rb @@ -268,7 +268,7 @@ def initialize def events_produced_by(reactor, cmd_class) return [] unless @prism_available - method_name = Sourced.message_method_name('sourced_decide', cmd_class.name) + method_name = Sourced.message_method_name(Decider::PREFIX, cmd_class.name) extract_calls_from_handler(reactor, method_name, :event) end From 582e163936ec5ed96c55b0df1086b42f21a77bca Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 13:39:22 +0100 Subject: [PATCH 105/115] Improve README --- README.md | 425 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 285 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index aac13856..61d5cbda 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,21 @@ store.append(new_events, guard: result.guard) # were appended after the read ``` +### Delayed messages + +Any message can be stamped with a future time via `#at(time)`. Scheduled messages live in a separate `sourced_scheduled_messages` table and are promoted into the main log when their `available_at` passes. + +```ruby +cmd = SendReminder.new(payload: { course_id: 'c1' }).at(Time.now + 3600) +store.schedule_messages([cmd]) + +# Normally the ScheduledMessagePoller (started by the Dispatcher) promotes +# due messages automatically. In tests or scripts, do it manually: +store.update_schedule! # => number of messages promoted +``` + +In a reaction handler, `dispatch(Cmd, ...).at(time)` uses this pipeline under the hood — see [Reactions](#reactions). + ### Partition reads `read_partition` uses AND semantics — a message is included only when every partition attribute it declares matches the given value. @@ -140,66 +155,17 @@ store.read_all(limit: 20).to_enum.lazy.select { |m| }.first(5) ``` -### Database setup - -`Store#install!` creates all required tables directly (useful for scripts, tests, and quick prototyping). For production apps using Sequel migrations, the store can export a migration file instead. - -#### Quick setup (e.g. scripts, tests) - -```ruby -db = Sequel.sqlite('my_app.db') -store = Sourced::Store.new(db) -store.install! -``` - -#### Exporting a Sequel migration - -Use `Store#copy_migration_to` to generate a migration file compatible with `Sequel::Migrator`: - -```ruby -db = Sequel.sqlite('my_app.db') -store = Sourced::Store.new(db) - -# Option 1: pass a directory (uses a default filename) -store.copy_migration_to('db/migrations') - -# Option 2: pass a block for full control over the path -store.copy_migration_to do - "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_sourced_tables.rb" -end -``` - -Then run your migrations as usual: - -```bash -sequel -m db/migrations sqlite://my_app.db -``` - -#### Custom table prefix - -By default, tables are prefixed with `sourced_` (e.g. `sourced_messages`, `sourced_consumer_groups`). Pass a `prefix:` to `Store.new` to customise this — for example when running multiple Sourced stores in the same database: - -```ruby -store = Sourced::Store.new(db, prefix: 'billing') -store.install! -# Creates: billing_messages, billing_key_pairs, billing_consumer_groups, ... -``` - -The prefix is carried through to exported migrations automatically. +## Reactors -#### Using the Installer directly +Sourced provides three reactor base classes that share the same lifecycle (claim → evolve → handle → append → ack) and the same consumer-group machinery: -The installer is also available as a standalone object, which is useful for Rake tasks or setup scripts: +- [Deciders](#deciders) — handle commands, enforce invariants, produce events +- [Projectors](#projectors) — build read models by consuming events +- [Durable workflows](#durable-workflows) — imperative orchestrations whose steps are memoised across re-entries -```ruby -installer = Sourced::Installer.new(db, logger: Logger.new($stdout), prefix: 'sourced') -installer.install # create tables -installer.installed? # check if tables exist -installer.uninstall # drop tables (test env only) -installer.copy_migration_to('db/migrations') -``` +Any reactor can produce follow-up messages via [Reactions](#reactions), and run side effects via [Sync and after-sync blocks](#sync-and-after-sync-blocks). -## Deciders +### Deciders Deciders handle commands, enforce invariants, and produce events. They rebuild state from event history before each decision. @@ -229,7 +195,7 @@ class CourseDecider < Sourced::Decider end ``` -### Synchronous command handling +#### Synchronous command handling `Sourced.handle!` loads history, runs the decider, appends the command + events, and advances consumer group offsets — all in one call. Designed for web controllers. @@ -246,7 +212,7 @@ end Raises `Sourced::ConcurrentAppendError` on conflicts, or `RuntimeError` on domain invariant violations (e.g. "Course already exists"). -### CommandContext +#### CommandContext `Sourced::CommandContext` is a factory for building Sourced commands from raw Hash attributes (e.g. HTTP params), injecting defaults like `metadata`. It mirrors `Sourced::CommandContext` but without `stream_id`, since Sourced messages are stream-less. @@ -266,7 +232,7 @@ cmd = ctx.build(CreateCourse, payload: { course_id: 'c1', course_name: 'Algebra' String keys are automatically symbolized, so `ctx.build('type' => '...', 'payload' => { ... })` works too. -#### Callback hooks (`on` and `any`) +##### Callback hooks (`on` and `any`) Subclass `CommandContext` and register class-level hooks to enrich or transform commands at build time — e.g. injecting session data or adding metadata from the request scope. @@ -333,7 +299,7 @@ class AppCommandContext < Sourced::CommandContext end ``` -#### Scoping to a command subset +##### Scoping to a command subset By default, `CommandContext` looks up types in `Sourced::Command.registry`. Pass a `scope:` to restrict lookups to a specific command subclass — attempts to build commands outside the scope raise `Sourced::UnknownMessageError`. @@ -351,18 +317,18 @@ ctx.build(type: 'courses.create', payload: { ... }) # OK ctx.build(type: 'admin.delete_all', payload: {}) # raises UnknownMessageError ``` -### Loading a decider's state +#### Loading a decider's state ```ruby decider, read_result = Sourced.load(CourseDecider, course_name: 'Algebra') decider.state # => { name_taken: true } ``` -## Projectors +### Projectors Projectors consume events to build read models. Two flavours: -### EventSourced projector +#### EventSourced projector Rebuilds state from full history on every batch (like deciders). @@ -399,7 +365,7 @@ class CourseCatalogProjector < Sourced::Projector::EventSourced end ``` -### StateStored projector +#### StateStored projector Loads persisted state via the `state` block, evolves only new (unprocessed) messages. @@ -423,7 +389,92 @@ class MyProjector < Sourced::Projector::StateStored end ``` -## Reactions +### Durable workflows + +`Sourced::DurableWorkflow` models long-running, side-effect-heavy workflows as an imperative `execute` method whose steps are memoised across re-entries. A workflow instance is identified by a `workflow_id` (auto-assigned) which doubles as its partition key. + +Subclassing `DurableWorkflow` auto-registers a full lifecycle event set (`WorkflowStarted`, `StepStarted`, `StepComplete`, `StepFailed`, `ContextUpdated`, `WaitStarted`, `WaitEnded`, `WorkflowComplete`, `WorkflowFailed`) under the class name. + +```ruby +class GreetingTask < Sourced::DurableWorkflow + def execute(name) + ip = resolve_ip + location = geolocate(ip) + "Hello #{name}, your IP is #{ip} and its location is #{location}" + end + + durable def resolve_ip + IPResolver.resolve + end + + def geolocate(ip) + Geolocator.locate(ip) + end + # retry the step up to 3 times before failing the workflow + durable :geolocate, retries: 3 +end +``` + +`durable` wraps a method so its result is memoised (via `StepComplete` events) and its failures are recorded (`StepFailed`). Re-entering `execute` replays past steps from memory instead of re-running their side effects. The `durable def foo; ...; end` form works because `def` returns the method's symbol. + +#### Starting and observing a workflow + +```ruby +# Register the workflow so the Dispatcher drives it forward +Sourced.register(GreetingTask) + +# Start a new run — appends WorkflowStarted and returns a Waiter +waiter = GreetingTask.execute('Ada', store: Sourced.store) +waiter.workflow_id # => "workflow-" + +# Block until the workflow reaches a terminal state (:complete or :failed) +instance = waiter.wait(timeout: 30) +instance.status # => :complete +instance.output # => "Hello Ada, your IP is ... and its location is ..." +``` + +#### Rehydrating from the store + +```ruby +instance = GreetingTask.load('workflow-abc-123') +instance.status # => :started | :complete | :failed +instance.context # => hash set via the `context` DSL +``` + +#### Initial context + +```ruby +class IndexPages < Sourced::DurableWorkflow + context do + { visited: [] } + end + + def execute(urls) + urls.each { |u| fetch(u) } + end + + durable def fetch(url) + # ... returns parsed page + context[:visited] << url + end +end +``` + +The `context` block runs once per replay and seeds `#context`, which is persisted as `ContextUpdated` events whenever it changes. + +#### Waiting inside a workflow + +Inside `execute`, `wait(seconds)` suspends the run by appending `WaitStarted` with an `at:` time. The `ScheduledMessagePoller` resumes the workflow by promoting the corresponding `WaitEnded` when the time elapses. + +```ruby +def execute + notify_started + wait(300) # sleep for 5 minutes + notify_finished +end +``` + +### Reactions Both deciders and projectors can react to events to produce new commands or events, enabling workflow orchestration. @@ -442,7 +493,7 @@ end Reactions are skipped during replay (when `replaying: true`), so side effects don't re-fire. -## Sync and After-Sync Blocks +### Sync and After-Sync Blocks Both deciders and projectors support `sync` and `after_sync` blocks for running side effects during message processing. @@ -476,78 +527,6 @@ end Multiple `sync` and `after_sync` blocks can be registered; they execute in registration order. Blocks are inherited by subclasses. -## Configuration - -```ruby -require 'sourced' - -Sourced.configure do |c| - # Pass a Sequel SQLite connection or a Sourced::Store instance - c.store = Sequel.sqlite('my_app.db') - - # Optional settings - c.worker_count = 4 # background worker fibers (default: 2) - c.batch_size = 50 # messages per claim (default: 50) - c.catchup_interval = 5 # seconds between catch-up polls (default: 5) - c.max_drain_rounds = 10 # max drain iterations per pickup (default: 10) - c.claim_ttl_seconds = 120 # stale claim threshold (default: 120) - c.housekeeping_interval = 30 # heartbeat/reap cycle (default: 30) -end -``` - -## Failure handling and retries - -Sourced already supports consumer-group retries on failure. - -- On reactor errors, `Router#handle_next_for` calls the reactor's `on_exception` hook. -- By default, that hook uses `Sourced.config.error_strategy`. -- The default `Sourced::ErrorStrategy` marks the consumer group as failed immediately. -- If you configure a retrying error strategy, Sourced stores the next retry time in the consumer group's `retry_at` column and skips claiming work for that group until that time has passed. - -So retries are built in already, but they are opt-in via the error strategy configuration. - -### Example: exponential backoff retries - -```ruby -require 'sourced' - -Sourced.configure do |c| - c.store = Sequel.sqlite('my_app.db') - - c.error_strategy = Sourced::ErrorStrategy.new do |s| - s.retry( - times: 5, - after: 2, - backoff: ->(retry_after, retry_count) { retry_after * (2**(retry_count - 1)) } - ) - - s.on_retry do |retry_count, exception, message, later| - LOGGER.warn( - "Sourced retry ##{retry_count} for #{message.type} (#{message.id}) " \ - "at #{later}: #{exception.class}: #{exception.message}" - ) - end - - s.on_fail do |exception, message| - LOGGER.error( - "Sourced failing consumer group after retries for #{message.type} (#{message.id}): " \ - "#{exception.class}: #{exception.message}" - ) - end - end -end -``` - -With the configuration above, failures retry after: - -- retry 1: 2 seconds -- retry 2: 4 seconds -- retry 3: 8 seconds -- retry 4: 16 seconds -- retry 5: 32 seconds - -After the configured retries are exhausted, the consumer group is marked as failed. - ## Registering reactors ```ruby @@ -616,8 +595,9 @@ supervisor.start 2. **Dispatcher** routes notifications to a `WorkQueue`, mapping message types to interested reactors 3. **Workers** pop reactors from the queue, claim a partition via `Router#handle_next_for`, process messages, and ack 4. **CatchUpPoller** periodically pushes all reactors as a safety net (handles missed notifications) -5. **ScheduledMessagePoller** promotes due delayed messages into the main Sourced log -6. **StaleClaimReaper** releases claims held by dead workers +5. **Store#schedule_messages** persists delayed messages in a separate `sourced_scheduled_messages` table keyed by `available_at` +6. **ScheduledMessagePoller** runs on an interval and promotes any messages whose `available_at` is in the past into the main log +7. **StaleClaimReaper** releases claims held by dead workers ### Router (direct usage) @@ -633,6 +613,60 @@ router.handle_next_for(CourseDecider) router.drain ``` +## Failure handling and retries + +Sourced already supports consumer-group retries on failure. + +- On reactor errors, `Router#handle_next_for` calls the reactor's `on_exception` hook. +- If a batch fails mid-way, it is raised as `Sourced::PartialBatchError`. The error carries the already-processed `action_pairs` (which are still ack'd) and the `failed_message` the hook receives — so partial progress is not lost. +- By default, the hook uses `Sourced.config.error_strategy`. +- The default `Sourced::ErrorStrategy` marks the consumer group as failed immediately. +- If you configure a retrying error strategy, Sourced stores the next retry time in the consumer group's `retry_at` column and skips claiming work for that group until that time has passed. + +So retries are built in already, but they are opt-in via the error strategy configuration. + +### Example: exponential backoff retries + +```ruby +require 'sourced' + +Sourced.configure do |c| + c.store = Sequel.sqlite('my_app.db') + + c.error_strategy = Sourced::ErrorStrategy.new do |s| + s.retry( + times: 5, + after: 2, + backoff: ->(retry_after, retry_count) { retry_after * (2**(retry_count - 1)) } + ) + + s.on_retry do |retry_count, exception, message, later| + LOGGER.warn( + "Sourced retry ##{retry_count} for #{message.type} (#{message.id}) " \ + "at #{later}: #{exception.class}: #{exception.message}" + ) + end + + s.on_fail do |exception, message| + LOGGER.error( + "Sourced failing consumer group after retries for #{message.type} (#{message.id}): " \ + "#{exception.class}: #{exception.message}" + ) + end + end +end +``` + +With the configuration above, failures retry after: + +- retry 1: 2 seconds +- retry 2: 4 seconds +- retry 3: 8 seconds +- retry 4: 16 seconds +- retry 5: 32 seconds + +After the configured retries are exhausted, the consumer group is marked as failed. + ## Consumer groups Each reactor class is a consumer group. The store tracks per-partition offsets so multiple reactors process the same events independently. @@ -820,6 +854,37 @@ behind = store.read_offsets(limit: 50).to_enum.lazy.select { |o| offsets, total_count = store.read_offsets(group_id: 'CourseDecider') ``` +## Topology introspection + +`Sourced.topology` analyses all registered reactors (deciders and projectors) and returns a flat array of node structs describing the message-flow graph. Nodes are `CommandNode`, `EventNode`, `AutomationNode` (for reactions), and `ReadModelNode` (for projectors). Each node has at least `type`, `id`, `name`, `group_id`, and depending on kind, `consumes` / `produces` / `schema`. + +```ruby +Sourced.register(CourseDecider) +Sourced.register(EnrolmentDecider) +Sourced.register(CourseCatalogProjector) + +Sourced.topology.each do |node| + puts "#{node.type}: #{node.name}" +end +# command: CourseApp::CreateCourse +# event: CourseApp::CourseCreated +# command: CourseApp::EnrolStudent +# automation: reaction(CourseCreated) +# readmodel: CourseApp::CourseCatalogProjector +# ... +``` + +The graph is cached; `Sourced.register` invalidates the cache automatically, and you can force a rebuild with `Sourced.reset_topology`. Typical uses are generating [Event Modeling](https://eventmodeling.org/) diagrams, debugging "which reactor produces this event", or driving visual service-dependency tooling. + +```ruby +# Which commands produce the `courses.created` event? +Sourced.topology + .select { |n| n.type == 'command' && n.produces.include?('courses.created') } + .map(&:name) +``` + +`produces` / `consumes` are resolved statically by parsing reactor source with Prism, so they are only populated when the `prism` gem is available. + ## Testing Sourced ships with RSpec helpers for Given-When-Then testing of deciders and projectors. The helpers call `handle_batch` directly — no store, router, or consumer group setup needed. @@ -965,3 +1030,83 @@ See `examples/app/` for a complete Sinatra application with: - An event-sourced projector writing JSON files - Synchronous command handling via `Sourced.handle!` in HTTP endpoints - Background worker processing via Falcon + +## Setup & configuration + +### Configuration + +```ruby +require 'sourced' + +Sourced.configure do |c| + # Pass a Sequel SQLite connection or a Sourced::Store instance + c.store = Sequel.sqlite('my_app.db') + + # Optional settings + c.worker_count = 4 # background worker fibers (default: 2) + c.batch_size = 50 # messages per claim (default: 50) + c.catchup_interval = 5 # seconds between catch-up polls (default: 5) + c.max_drain_rounds = 10 # max drain iterations per pickup (default: 10) + c.claim_ttl_seconds = 120 # stale claim threshold (default: 120) + c.housekeeping_interval = 30 # heartbeat/reap cycle (default: 30) +end +``` + +### Database setup + +`Store#install!` creates all required tables directly (useful for scripts, tests, and quick prototyping). For production apps using Sequel migrations, the store can export a migration file instead. + +#### Quick setup (e.g. scripts, tests) + +```ruby +db = Sequel.sqlite('my_app.db') +store = Sourced::Store.new(db) +store.install! +``` + +#### Exporting a Sequel migration + +Use `Store#copy_migration_to` to generate a migration file compatible with `Sequel::Migrator`: + +```ruby +db = Sequel.sqlite('my_app.db') +store = Sourced::Store.new(db) + +# Option 1: pass a directory (uses a default filename) +store.copy_migration_to('db/migrations') + +# Option 2: pass a block for full control over the path +store.copy_migration_to do + "db/migrations/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_sourced_tables.rb" +end +``` + +Then run your migrations as usual: + +```bash +sequel -m db/migrations sqlite://my_app.db +``` + +#### Custom table prefix + +By default, tables are prefixed with `sourced_` (e.g. `sourced_messages`, `sourced_consumer_groups`). Pass a `prefix:` to `Store.new` to customise this — for example when running multiple Sourced stores in the same database: + +```ruby +store = Sourced::Store.new(db, prefix: 'billing') +store.install! +# Creates: billing_messages, billing_key_pairs, billing_consumer_groups, ... +``` + +The prefix is carried through to exported migrations automatically. + +#### Using the Installer directly + +The installer is also available as a standalone object, which is useful for Rake tasks or setup scripts: + +```ruby +installer = Sourced::Installer.new(db, logger: Logger.new($stdout), prefix: 'sourced') +installer.install # create tables +installer.installed? # check if tables exist +installer.uninstall # drop tables (test env only) +installer.copy_migration_to('db/migrations') +``` From 053a75c87f32c8041e75c57fbd78ddf5b61def69 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 14:20:15 +0100 Subject: [PATCH 106/115] Untrack examples/app Keep the files locally for experimentation but exclude them from the repo until they're ready. Co-Authored-By: Claude Opus 4.6 (1M context) --- examples/app/Gemfile | 9 -- examples/app/Gemfile.lock | 286 ---------------------------------- examples/app/README.md | 245 ----------------------------- examples/app/app.rb | 93 ----------- examples/app/config.ru | 19 --- examples/app/domain.rb | 142 ----------------- examples/app/falcon.rb | 14 -- examples/app/storage/.gitkeep | 0 8 files changed, 808 deletions(-) delete mode 100644 examples/app/Gemfile delete mode 100644 examples/app/Gemfile.lock delete mode 100644 examples/app/README.md delete mode 100644 examples/app/app.rb delete mode 100644 examples/app/config.ru delete mode 100644 examples/app/domain.rb delete mode 100644 examples/app/falcon.rb delete mode 100644 examples/app/storage/.gitkeep diff --git a/examples/app/Gemfile b/examples/app/Gemfile deleted file mode 100644 index 4eff7441..00000000 --- a/examples/app/Gemfile +++ /dev/null @@ -1,9 +0,0 @@ -source 'https://rubygems.org' - -gem 'sourced', path: '../..' -gem 'sourced-ui', path: '/Users/ismasan/code/personal/gems/sourced-ui' -gem 'sequel' -gem 'sqlite3' -gem 'sinatra' -gem 'falcon' -gem 'irb' diff --git a/examples/app/Gemfile.lock b/examples/app/Gemfile.lock deleted file mode 100644 index f3eab322..00000000 --- a/examples/app/Gemfile.lock +++ /dev/null @@ -1,286 +0,0 @@ -PATH - remote: ../.. - specs: - sourced (0.0.1) - async - plumb (>= 0.0.17) - -PATH - remote: /Users/ismasan/code/personal/gems/sourced-ui - specs: - sourced-ui (0.1.0) - datastar (= 1.0.2) - logger - phlex - rack - sourced - -GEM - remote: https://rubygems.org/ - specs: - async (2.36.0) - console (~> 1.29) - fiber-annotation - io-event (~> 1.11) - metrics (~> 0.12) - traces (~> 0.18) - async-container (0.34.2) - async (~> 2.22) - async-container-supervisor (0.10.0) - async-service - io-endpoint - memory (~> 0.7) - memory-leak (~> 0.10) - process-metrics - async-http (0.94.2) - async (>= 2.10.2) - async-pool (~> 0.11) - io-endpoint (~> 0.14) - io-stream (~> 0.6) - metrics (~> 0.12) - protocol-http (~> 0.58) - protocol-http1 (~> 0.36) - protocol-http2 (~> 0.22) - protocol-url (~> 0.2) - traces (~> 0.10) - async-http-cache (0.4.6) - async-http (~> 0.56) - async-pool (0.11.1) - async (>= 2.0) - async-service (0.20.1) - async - async-container (~> 0.34) - string-format (~> 0.2) - bake (0.24.1) - bigdecimal - samovar (~> 2.1) - base64 (0.3.0) - bigdecimal (4.0.1) - concurrent-ruby (1.3.6) - console (1.34.3) - fiber-annotation - fiber-local (~> 1.1) - json - datastar (1.0.2) - json - logger - rack (>= 3.2) - date (3.5.1) - erb (6.0.1) - falcon (0.54.2) - async - async-container (~> 0.20) - async-container-supervisor (~> 0.6) - async-http (~> 0.75) - async-http-cache (~> 0.4) - async-service (~> 0.19) - bundler - localhost (~> 1.1) - openssl (>= 3.0) - protocol-http (~> 0.31) - protocol-rack (~> 0.7) - samovar (~> 2.3) - fiber-annotation (0.2.0) - fiber-local (1.1.0) - fiber-storage - fiber-storage (1.0.1) - io-console (0.8.2) - io-endpoint (0.17.2) - io-event (1.14.2) - io-stream (0.11.1) - irb (1.17.0) - pp (>= 0.6.0) - prism (>= 1.3.0) - rdoc (>= 4.0.0) - reline (>= 0.4.2) - json (2.18.1) - localhost (1.7.0) - logger (1.7.0) - mapping (1.1.3) - memory (0.12.0) - bake (~> 0.15) - console - msgpack - memory-leak (0.10.2) - process-metrics (>= 0.10.1) - metrics (0.15.0) - msgpack (1.8.0) - mustermann (3.0.4) - ruby2_keywords (~> 0.0.1) - openssl (4.0.1) - phlex (2.4.1) - refract (~> 1.0) - zeitwerk (~> 2.7) - plumb (0.0.17) - bigdecimal - concurrent-ruby - pp (0.6.3) - prettyprint - prettyprint (0.2.0) - prism (1.9.0) - process-metrics (0.10.2) - console (~> 1.8) - json (~> 2) - samovar (~> 2.1) - protocol-hpack (1.5.1) - protocol-http (0.59.0) - protocol-http1 (0.37.0) - protocol-http (~> 0.58) - protocol-http2 (0.24.0) - protocol-hpack (~> 1.4) - protocol-http (~> 0.47) - protocol-rack (0.21.1) - io-stream (>= 0.10) - protocol-http (~> 0.58) - rack (>= 1.0) - protocol-url (0.4.0) - psych (5.3.1) - date - stringio - rack (3.2.5) - rack-protection (4.2.1) - base64 (>= 0.1.0) - logger (>= 1.6.0) - rack (>= 3.0.0, < 4) - rack-session (2.1.1) - base64 (>= 0.1.0) - rack (>= 3.0.0) - rdoc (7.2.0) - erb - psych (>= 4.0.0) - tsort - refract (1.1.0) - prism - zeitwerk - reline (0.6.3) - io-console (~> 0.5) - ruby2_keywords (0.0.5) - samovar (2.4.1) - console (~> 1.0) - mapping (~> 1.0) - sequel (5.101.0) - bigdecimal - sinatra (4.2.1) - logger (>= 1.6.0) - mustermann (~> 3.0) - rack (>= 3.0.0, < 4) - rack-protection (= 4.2.1) - rack-session (>= 2.0.0, < 3) - tilt (~> 2.0) - sqlite3 (2.9.0-aarch64-linux-gnu) - sqlite3 (2.9.0-aarch64-linux-musl) - sqlite3 (2.9.0-arm-linux-gnu) - sqlite3 (2.9.0-arm-linux-musl) - sqlite3 (2.9.0-arm64-darwin) - sqlite3 (2.9.0-x86-linux-gnu) - sqlite3 (2.9.0-x86-linux-musl) - sqlite3 (2.9.0-x86_64-darwin) - sqlite3 (2.9.0-x86_64-linux-gnu) - sqlite3 (2.9.0-x86_64-linux-musl) - string-format (0.2.0) - stringio (3.2.0) - tilt (2.7.0) - traces (0.18.2) - tsort (0.2.0) - zeitwerk (2.7.5) - -PLATFORMS - aarch64-linux-gnu - aarch64-linux-musl - arm-linux-gnu - arm-linux-musl - arm64-darwin - x86-linux-gnu - x86-linux-musl - x86_64-darwin - x86_64-linux-gnu - x86_64-linux-musl - -DEPENDENCIES - falcon - irb - sequel - sinatra - sourced! - sourced-ui! - sqlite3 - -CHECKSUMS - async (2.36.0) sha256=090623f4c65706664355c9efa6c7bfb86771a513e65cd681c51cb27747530550 - async-container (0.34.2) sha256=83ac767e74a832c42e156110c14e10b1fb3fd7583fa1beb3dfe45269e3554746 - async-container-supervisor (0.10.0) sha256=96a2b6048312e679993c116f0b29b70fc25696b0a6e8c68d8c4f9b8e666a8614 - async-http (0.94.2) sha256=c5ca94b337976578904a373833abe5b8dfb466a2946af75c4ae38c409c5c78b2 - async-http-cache (0.4.6) sha256=2038d1f093182f16b50b4db271c25085e3938da10bfcfc2904cadb0530fddfd6 - async-pool (0.11.1) sha256=98e1583e199a75f7dc70f8e65fc8d0d3b28636c3f256595d43e206642ad8fbda - async-service (0.20.1) sha256=d77ca8912e31c729f6e62c783ae3f364385ccc91a34c97998b761d7bda72b01b - bake (0.24.1) sha256=8bfac7e61514b17720e3b13cf6a5e122243f43123c6802707b150904bec5f4c7 - base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b - bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7 - concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab - console (1.34.3) sha256=869fbd74697efc4c606f102d2812b0b008e4e7fd738a91c591e8577140ec0dcc - datastar (1.0.2) sha256=3fd1430a279f0668142e9d5ec5f7f81087b1ea3defc75f55ea8aa2ac036f651c - date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 - erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 - falcon (0.54.2) sha256=ab35a702466f318d6b3189e638e4aee53290261a5404c9f681115f4a8aca153e - fiber-annotation (0.2.0) sha256=7abfadf1d119f508867d4103bf231c0354d019cc39a5738945dec2edadaf6c03 - fiber-local (1.1.0) sha256=c885f94f210fb9b05737de65d511136ea602e00c5105953748aa0f8793489f06 - fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1 - io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc - io-endpoint (0.17.2) sha256=3feaf766c116b35839c11fac68b6aaadc47887bb488902a57bf8e1d288fb3338 - io-event (1.14.2) sha256=b0a069190eafe86005c22f7464f744971b5bd82f153740d34e6ab49548d4f613 - io-stream (0.11.1) sha256=fa5f551fcff99581c1757b9d1cee2c37b124f07d2ca4f40b756a05ab9bd21b87 - irb (1.17.0) sha256=168c4ddb93d8a361a045c41d92b2952c7a118fa73f23fe14e55609eb7a863aae - json (2.18.1) sha256=fe112755501b8d0466b5ada6cf50c8c3f41e897fa128ac5d263ec09eedc9f986 - localhost (1.7.0) sha256=09b32819537f914ccdf0a7c595fab162517401b6ef644a2afd3708d943c4547f - logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 - mapping (1.1.3) sha256=2274931d20ecd46eaafdd1e00c58cc7472133b213bcac335cc7733d3c75f4da2 - memory (0.12.0) sha256=786a14d84cec8e5667a491da02ebbf492b9ec3d19d35161131ac9d47abb684b4 - memory-leak (0.10.2) sha256=c486973e3dd2339d837f050bbb0ffc5a72584f8553982c60f59d486a6ad04a5a - metrics (0.15.0) sha256=61ded5bac95118e995b1bc9ed4a5f19bc9814928a312a85b200abbdac9039072 - msgpack (1.8.0) sha256=e64ce0212000d016809f5048b48eb3a65ffb169db22238fb4b72472fecb2d732 - mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 - openssl (4.0.1) sha256=e27974136b7b02894a1bce46c5397ee889afafe704a839446b54dc81cb9c5f7d - phlex (2.4.1) sha256=e596717fbfe38b5271840266758779ebe75092e02629f0c170287e6290a70b12 - plumb (0.0.17) sha256=434138323bde29cefbec136bdd2f23f3e37d5ea9eca72fafb5bce6801d64a56d - pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 - prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 - prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 - process-metrics (0.10.2) sha256=c0593c7e6695d0e75a5e10fb9c13728a0a23c1ad3514b747dee2e7b19eed00d3 - protocol-hpack (1.5.1) sha256=6feca238b8078da1cd295677d6f306c6001af92d75fe0643d33e6956cbc3ad91 - protocol-http (0.59.0) sha256=90e20ad817cb3ffe947d4fd6194fe0651f385625dcce055386d1c356ee32547b - protocol-http1 (0.37.0) sha256=5bdd739e28792b341134596f6f5ab21a9d4b395f67bae69e153743eb0e69d123 - protocol-http2 (0.24.0) sha256=65327a019b7e36d2774e94050bf57a43bb60212775d2fcf02ae1d2ed4f01ef28 - protocol-rack (0.21.1) sha256=366ff16efbf4c2f8d2e3fad4e992effa2357610f70effbccfa2767d26fedc577 - protocol-url (0.4.0) sha256=64d4c03b6b51ad815ac6fdaf77a1d91e5baf9220d26becb846c5459dacdea9e1 - psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 - rack (3.2.5) sha256=4cbd0974c0b79f7a139b4812004a62e4c60b145cba76422e288ee670601ed6d3 - rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac - rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 - rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192 - refract (1.1.0) sha256=ee3b9627e39f7692831101e2fedd73e0d09a592ff5d5c05f171d14211fc7a9c7 - reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 - ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef - samovar (2.4.1) sha256=c3b91dd0580771e3bc600621c1111f29542529dcffafaac3b6bf068b3f309e80 - sequel (5.101.0) sha256=d2ae3fd997a7c4572e8357918e777869faf90dc19310fcd6332747122aed2b29 - sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 - sourced (0.0.1) - sourced-ui (0.1.0) - sqlite3 (2.9.0-aarch64-linux-gnu) sha256=cfe1e0216f46d7483839719bf827129151e6c680317b99d7b8fc1597a3e13473 - sqlite3 (2.9.0-aarch64-linux-musl) sha256=56a35cb2d70779afc2ac191baf2c2148242285ecfed72f9b021218c5c4917913 - sqlite3 (2.9.0-arm-linux-gnu) sha256=a19a21504b0d7c8c825fbbf37b358ae316b6bd0d0134c619874060b2eef05435 - sqlite3 (2.9.0-arm-linux-musl) sha256=fca5b26197c70e3363115d3faaea34d7b2ad9c7f5fa8d8312e31b64e7556ee07 - sqlite3 (2.9.0-arm64-darwin) sha256=a917bd9b84285766ff3300b7d79cd583f5a067594c8c1263e6441618c04a6ed3 - sqlite3 (2.9.0-x86-linux-gnu) sha256=47317ba230f6c2c361981aa5fc1bf9de1b99727317171393ba90abab092c5b5f - sqlite3 (2.9.0-x86-linux-musl) sha256=b627f3a2ca59aaaa5e10b8666cdbd7122469b49afa4bd895133cecb7b5c1368d - sqlite3 (2.9.0-x86_64-darwin) sha256=59fe51baa3cb33c36d27ce78b4ed9360cd33ccca09498c2ae63850c97c0a6026 - sqlite3 (2.9.0-x86_64-linux-gnu) sha256=72fff9bd750070ba3af695511ba5f0e0a2d8a9206f84869640b3e99dfaf3d5a5 - sqlite3 (2.9.0-x86_64-linux-musl) sha256=ef716ba7a66d7deb1ccc402ac3a6d7343da17fac862793b7f0be3d2917253c90 - string-format (0.2.0) sha256=bc981c14116b061f12134549f32fa2d61a17b5a35dd6fd36596c21722a789af6 - stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 - tilt (2.7.0) sha256=0d5b9ba69f6a36490c64b0eee9f6e9aad517e20dcc848800a06eb116f08c6ab3 - traces (0.18.2) sha256=80f1649cb4daace1d7174b81f3b3b7427af0b93047759ba349960cb8f315e214 - tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f - zeitwerk (2.7.5) sha256=d8da92128c09ea6ec62c949011b00ed4a20242b255293dd66bf41545398f73dd - -BUNDLED WITH - 4.0.3 diff --git a/examples/app/README.md b/examples/app/README.md deleted file mode 100644 index 9948f0d3..00000000 --- a/examples/app/README.md +++ /dev/null @@ -1,245 +0,0 @@ -# CCC Demo: Student Enrolment - -A demo app exercising CCC's core features: sync deciders with optimistic concurrency, multi-entity context validation, uniqueness checks, and an async projector for read models. - -## Setup - -```bash -cd examples/ccc_app -bundle install -``` - -## Running - -```bash -bundle exec falcon host -``` - -The app starts at `http://localhost:9292`. - -## Domain - -- **CourseDecider** -- enforces course name uniqueness (partitioned by `course_name`) -- **EnrolmentDecider** -- validates course exists, no duplicate students, max 20 per course (partitioned by `course_id`) -- **CourseCatalogProjector** -- async read model updated by background workers (partitioned by `course_id`) - -Deciders run synchronously in the request (via the resource endpoints) or asynchronously via the generic `/commands` endpoint. The projector runs in the background via CCC's Dispatcher, so reads are eventually consistent. - -## IRB - -```bash -irb -r ./domain -``` - -```ruby -CourseApp.setup! -# Now you can use CCC.load, CCC.store.append, etc. -``` - -## Endpoints - -### Generic command endpoint - -``` -POST /commands -``` - -Accepts any registered CCC message type. The command is appended to the store and processed asynchronously by background workers. Returns **202** on success. - -| Field | Type | Required | -|-------|------|----------| -| `type` | string | yes | -| `payload` | object | yes | - -```bash -curl -X POST http://localhost:9292/commands \ - -H 'Content-Type: application/json' \ - -d '{"type": "courses.create", "payload": {"course_id": "abc-123", "course_name": "Algebra"}}' -``` - -```json -{"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "type": "courses.create"} -``` - -Invalid payload returns **422** with field-level errors: - -```bash -curl -X POST http://localhost:9292/commands \ - -H 'Content-Type: application/json' \ - -d '{"type": "courses.enrol", "payload": {"course_id": "", "student_id": ""}}' -``` - -```json -{"payload": {"course_id": "must be present", "student_id": "must be present"}} -``` - -Unknown message type returns **422**: - -```json -{"error": "Unknown message type: foo.bar"} -``` - -### List courses - -``` -GET / -``` - -Returns the course catalog (populated by the async projector). - -```bash -curl http://localhost:9292/ -``` - -```json -[ - {"course_id": "abc-123", "course_name": "Algebra", "student_count": 2} -] -``` - -### Create a course - -``` -POST /courses -``` - -| Field | Type | Required | -|-------|------|----------| -| `course_name` | string | yes | - -```bash -curl -X POST http://localhost:9292/courses \ - -H 'Content-Type: application/json' \ - -d '{"course_name": "Algebra"}' -``` - -```json -{"course_id": "abc-123", "course_name": "Algebra"} -``` - -Duplicate name returns **422**: - -```bash -curl -X POST http://localhost:9292/courses \ - -H 'Content-Type: application/json' \ - -d '{"course_name": "Algebra"}' -``` - -```json -{"error": "Course 'Algebra' already exists"} -``` - -### Course detail - -``` -GET /courses/:id -``` - -```bash -curl http://localhost:9292/courses/abc-123 -``` - -```json -{ - "course_id": "abc-123", - "course_name": "Algebra", - "students": ["stu-1", "stu-2"], - "student_count": 1 -} -``` - -Returns **404** if the course hasn't been projected yet or doesn't exist. - -### Enrol a student - -``` -POST /courses/:id/enrolments -``` - -| Field | Type | Required | -|-------|------|----------| -| `student_id` | string | yes | - -```bash -curl -X POST http://localhost:9292/courses/abc-123/enrolments \ - -H 'Content-Type: application/json' \ - -d '{"student_id": "stu-1"}' -``` - -```json -{"course_id": "abc-123", "student_id": "stu-1"} -``` - -Errors return **422**: - -- Non-existent course: `{"error": "Course 'abc-123' does not exist"}` -- Duplicate student: `{"error": "Student 'stu-1' is already enrolled"}` -- Course full: `{"error": "Course is full (max 20 students)"}` - -Concurrent modification returns **409**: - -```json -{"error": "Concurrent modification — please retry"} -``` - -## Full walkthrough (resource endpoints) - -```bash -# Create a course -curl -s -X POST http://localhost:9292/courses \ - -H 'Content-Type: application/json' \ - -d '{"course_name": "Algebra"}' | jq . -# Note the course_id from the response - -# List courses (may take a moment for the projector to catch up) -curl -s http://localhost:9292/ | jq . - -# Enrol students (replace with the actual ID) -curl -s -X POST http://localhost:9292/courses//enrolments \ - -H 'Content-Type: application/json' \ - -d '{"student_id": "student-1"}' | jq . - -curl -s -X POST http://localhost:9292/courses//enrolments \ - -H 'Content-Type: application/json' \ - -d '{"student_id": "student-2"}' | jq . - -# View course detail -curl -s http://localhost:9292/courses/ | jq . - -# Try duplicate name (422) -curl -s -X POST http://localhost:9292/courses \ - -H 'Content-Type: application/json' \ - -d '{"course_name": "Algebra"}' | jq . - -# Try duplicate enrolment (422) -curl -s -X POST http://localhost:9292/courses//enrolments \ - -H 'Content-Type: application/json' \ - -d '{"student_id": "student-1"}' | jq . -``` - -## Full walkthrough (generic /commands endpoint) - -```bash -# Create a course (async — processed by background workers) -curl -s -X POST http://localhost:9292/commands \ - -H 'Content-Type: application/json' \ - -d '{"type": "courses.create", "payload": {"course_id": "abc-123", "course_name": "Algebra"}}' | jq . - -# Enrol a student -curl -s -X POST http://localhost:9292/commands \ - -H 'Content-Type: application/json' \ - -d '{"type": "courses.enrol", "payload": {"course_id": "abc-123", "student_id": "student-1"}}' | jq . - -# List courses (may take a moment for workers to process) -curl -s http://localhost:9292/ | jq . - -# Invalid payload (422) -curl -s -X POST http://localhost:9292/commands \ - -H 'Content-Type: application/json' \ - -d '{"type": "courses.create", "payload": {"course_id": "", "course_name": ""}}' | jq . - -# Unknown type (422) -curl -s -X POST http://localhost:9292/commands \ - -H 'Content-Type: application/json' \ - -d '{"type": "nope", "payload": {}}' | jq . -``` diff --git a/examples/app/app.rb b/examples/app/app.rb deleted file mode 100644 index b4be4fc4..00000000 --- a/examples/app/app.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -require_relative 'domain' -require 'sinatra/base' -require 'json' -require 'securerandom' - -class App < Sinatra::Base - set :default_content_type, 'application/json' - - helpers do - def json_body - JSON.parse(request.body.read, symbolize_names: true) - rescue JSON::ParserError - halt 400, { error: 'Invalid JSON' }.to_json - end - end - - # Generic command endpoint - post '/commands' do - data = json_body - message = CourseApp::Command.from(data.merge(metadata: { source: 'http' })) - - unless message.valid? - halt 422, message.errors.to_json - end - - Sourced.store.append([message]) - - status 202 - { id: message.id, type: message.type }.to_json - - rescue Sourced::UnknownMessageError => e - halt 422, { error: e.message }.to_json - end - - # List all courses - get '/' do - courses = CourseApp::CourseCatalogProjector.all_courses.map do |c| - { course_id: c[:course_id], course_name: c[:course_name], student_count: c[:student_count] } - end - courses.to_json - end - - # Create a course - post '/courses' do - body = json_body - course_id = SecureRandom.uuid - - cmd = CourseApp::CreateCourse.new(payload: { course_id: course_id, course_name: body[:course_name] }) - cmd, _decider, _events = Sourced.handle!(CourseApp::CourseDecider, cmd) - - halt 422, cmd.errors.to_json unless cmd.valid? - - status 201 - { course_id: course_id, course_name: cmd.payload.course_name }.to_json - - rescue RuntimeError => e - halt 422, { error: e.message }.to_json - rescue Sourced::ConcurrentAppendError - halt 409, { error: 'Concurrent modification — please retry' }.to_json - end - - # Course detail - get '/courses/:id' do - course = CourseApp::CourseCatalogProjector.read_course(params[:id]) - - halt 404, { error: 'Course not found' }.to_json unless course - - course.to_json - end - - # Enrol a student - post '/courses/:id/enrolments' do - body = json_body - - cmd = CourseApp::EnrolStudent.new(payload: { - course_id: params[:id], - student_id: body[:student_id] - }) - cmd, _decider, _events = Sourced.handle!(CourseApp::EnrolmentDecider, cmd) - - halt 422, cmd.errors.to_json unless cmd.valid? - - status 201 - { course_id: cmd.payload.course_id, student_id: cmd.payload.student_id }.to_json - - rescue RuntimeError => e - halt 422, { error: e.message }.to_json - rescue Sourced::ConcurrentAppendError - halt 409, { error: 'Concurrent modification — please retry' }.to_json - end -end diff --git a/examples/app/config.ru b/examples/app/config.ru deleted file mode 100644 index 425e4671..00000000 --- a/examples/app/config.ru +++ /dev/null @@ -1,19 +0,0 @@ -require_relative 'app' - -require 'sourced/ui/dashboard' -require 'datastar/async_executor' -Datastar.config.executor = Datastar::AsyncExecutor.new - -Sourced::UI::Dashboard.configure do |config| - config.header_links([ - { label: 'back to app', href: '/', url: false } - ]) -end - -map '/sourced' do - run Sourced::UI::Dashboard -end - -map '/' do - run App -end diff --git a/examples/app/domain.rb b/examples/app/domain.rb deleted file mode 100644 index 3e6cb31c..00000000 --- a/examples/app/domain.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true - -require 'bundler/setup' -require 'sourced' -require 'sourced' -require 'sequel' -require 'fileutils' -require 'json' - - -module CourseApp - # --- Messages --- - - class Event < Sourced::Message; end - class Command < Sourced::Message; end - - CreateCourse = Command.define('courses.create') { attribute :course_id, String; attribute :course_name, String } - CourseCreated = Event.define('courses.created') { attribute :course_id, String; attribute :course_name, String } - EnrolStudent = Command.define('courses.enrol') { attribute :course_id, String; attribute :student_id, String } - StudentEnrolled = Event.define('courses.enrolled') { attribute :course_id, String; attribute :student_id, String } - - # --- Deciders --- - - # Enforces course name uniqueness. - # Partition by :course_name so Sourced.load reads all CourseCreated with that name. - class CourseDecider < Sourced::Decider - partition_by :course_name - - state do |_partition_values| - { name_taken: false } - end - - evolve CourseCreated do |state, _event| - state[:name_taken] = true - end - - command CreateCourse do |state, cmd| - raise "Course '#{cmd.payload.course_name}' already exists" if state[:name_taken] - - event CourseCreated, course_id: cmd.payload.course_id, course_name: cmd.payload.course_name - end - end - - # Enforces enrolment rules: course must exist, no duplicates, max 20 students. - # Partition by :course_id so Sourced.load reads CourseCreated + StudentEnrolled for that course. - class EnrolmentDecider < Sourced::Decider - partition_by :course_id - - state do |_partition_values| - { course_exists: false, student_ids: [], student_count: 0 } - end - - evolve CourseCreated do |state, _event| - state[:course_exists] = true - end - - evolve StudentEnrolled do |state, event| - state[:student_ids] << event.payload.student_id - state[:student_count] += 1 - end - - command EnrolStudent do |state, cmd| - raise "Course '#{cmd.payload.course_id}' does not exist" unless state[:course_exists] - raise "Student '#{cmd.payload.student_id}' is already enrolled" if state[:student_ids].include?(cmd.payload.student_id) - raise "Course is full (max 20 students). Has #{state[:student_count]}" if state[:student_count] >= 20 - - event StudentEnrolled, - course_id: cmd.payload.course_id, - student_id: cmd.payload.student_id - end - end - - # --- Projector (async read model) --- - - # Builds a file-backed course catalog from events. - # Each course is written to a JSON file in storage/projections/. - # Registered with Sourced for background processing by workers. - class CourseCatalogProjector < Sourced::Projector::EventSourced - partition_by :course_id - - PROJECTIONS_DIR = File.join(__dir__, 'storage', 'projections') - - class << self - def projection_path(course_id) - File.join(PROJECTIONS_DIR, "#{course_id}.json") - end - - def read_course(course_id) - path = projection_path(course_id) - return nil unless File.exist?(path) - - JSON.parse(File.read(path), symbolize_names: true) - end - - def all_courses - Dir.glob(File.join(PROJECTIONS_DIR, '*.json')).filter_map do |path| - JSON.parse(File.read(path), symbolize_names: true) - rescue JSON::ParserError - nil - end - end - end - - state do |partition_values| - { course_id: nil, course_name: nil, students: [] } - end - - evolve CourseCreated do |state, event| - state[:course_id] = event.payload.course_id - state[:course_name] = event.payload.course_name - end - - evolve StudentEnrolled do |state, event| - state[:students] << event.payload.student_id - end - - sync do |state:, messages:, **| - next unless state[:course_id] - - FileUtils.mkdir_p(PROJECTIONS_DIR) - data = { - course_id: state[:course_id], - course_name: state[:course_name], - students: state[:students], - student_count: state[:students].size - } - File.write(self.class.projection_path(state[:course_id]), JSON.pretty_generate(data)) - end - end - - # --- Configuration --- - - DB_PATH = File.join(__dir__, 'storage', 'ccc_app.db') - - Sourced.configure do |c| - c.store = Sequel.sqlite(DB_PATH) - end - - Sourced.register(CourseDecider) - Sourced.register(EnrolmentDecider) - Sourced.register(CourseCatalogProjector) -end diff --git a/examples/app/falcon.rb b/examples/app/falcon.rb deleted file mode 100644 index 6084363f..00000000 --- a/examples/app/falcon.rb +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env falcon-host -# frozen_string_literal: true - -require_relative 'domain' -require_relative 'app' -require 'sourced/falcon' - -service "ccc-app" do - include Sourced::Falcon::Environment - include Falcon::Environment::Rackup - - # url "http://localhost:9292" - count 1 -end diff --git a/examples/app/storage/.gitkeep b/examples/app/storage/.gitkeep deleted file mode 100644 index e69de29b..00000000 From 931d53a442eb8e66b280653858663b71f524dfb2 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 14:21:48 +0100 Subject: [PATCH 107/115] Untrack bench files Keep benchmarks locally for experimentation but exclude them from the repo until they're ready. Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/append_bench.rb | 266 ------------------- bench/append_results.html | 402 ----------------------------- bench/scaling_bench.rb | 512 ------------------------------------- bench/scaling_results.html | 311 ---------------------- 4 files changed, 1491 deletions(-) delete mode 100644 bench/append_bench.rb delete mode 100644 bench/append_results.html delete mode 100644 bench/scaling_bench.rb delete mode 100644 bench/scaling_results.html diff --git a/bench/append_bench.rb b/bench/append_bench.rb deleted file mode 100644 index f8bd15b6..00000000 --- a/bench/append_bench.rb +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Benchmark: measures Store#append performance across message counts -# and payload sizes. Baseline before eager offset creation. -# -# Usage: -# bundle exec ruby bench/append_bench.rb -# bundle exec ruby bench/append_bench.rb --counts 1,10,100 -# bundle exec ruby bench/append_bench.rb --keys 1,3 -# bundle exec ruby bench/append_bench.rb --output bench/append_results.html - -require 'bundler/setup' -require 'sequel' -require 'sourced' -require 'sourced/ccc' -require 'sourced/ccc/store' -require 'optparse' -require 'json' - -Sourced.config.logger = Logger.new(File::NULL) -Console.logger.off! - -# --- CLI options ----------------------------------------------------------- - -options = { - counts: [1, 10, 50, 100], # messages per append call - keys: [1, 2, 3], # payload attributes used as partition keys - iterations: 5, # iterations per measurement - pre_existing: [0, 1_000, 10_000], # pre-existing messages in the store - output: 'bench/append_results.html' -} - -OptionParser.new do |opts| - opts.banner = "Usage: #{$0} [options]" - opts.on('--counts LIST', 'Messages per append call') { |v| options[:counts] = v.split(',').map(&:to_i) } - opts.on('--keys LIST', 'Key counts') { |v| options[:keys] = v.split(',').map(&:to_i) } - opts.on('--iterations N', Integer, 'Iterations') { |v| options[:iterations] = v } - opts.on('--pre LIST', 'Pre-existing message counts') { |v| options[:pre_existing] = v.split(',').map(&:to_i) } - opts.on('--output FILE', 'HTML output') { |v| options[:output] = v } -end.parse! - -# --- Message definitions -------------------------------------------------- - -AppendBenchEvent = Sourced::CCC::Message.define('append_bench.event') do - attribute :k0, String - attribute :k1, String - attribute :k2, String -end - -# --- Helpers --------------------------------------------------------------- - -def new_store - db = Sequel.sqlite - store = Sourced::CCC::Store.new(db) - store.install! - [db, store] -end - -def seed_messages(db, count) - return if count == 0 - now = Time.now.iso8601 - (0...count).each_slice(10_000) do |slice| - db.transaction do - db[:sourced_messages].multi_insert( - slice.map { |i| - { message_id: "seed-#{i}", message_type: 'append_bench.seed', payload: '{"x":"y"}', created_at: now } - } - ) - end - end -end - -def build_messages(count, key_count, offset: 0) - count.times.map do |i| - n = offset + i - payload = {} - payload[:k0] = "v#{n}" if key_count >= 1 - payload[:k1] = "v#{n}" if key_count >= 2 - payload[:k2] = "v#{n}" if key_count >= 3 - AppendBenchEvent.new(payload: payload) - end -end - -def measure - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - result = yield - [Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0, result] -end - -def median(values) - sorted = values.sort - mid = sorted.size / 2 - sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0 -end - -def fmt_scale(s) - return "#{s / 1_000_000}M" if s >= 1_000_000 - return "#{s / 1_000}K" if s >= 1_000 - s.to_s -end - -# --- Benchmark runner ------------------------------------------------------ - -results = [] - -options[:keys].each do |key_count| - options[:pre_existing].each do |pre| - options[:counts].each do |count| - label = "keys=#{key_count} pre=#{fmt_scale(pre)} count=#{count}" - $stderr.print " #{label}..." - - times = [] - per_msg_times = [] - - options[:iterations].times do - _db, store = new_store - seed_messages(_db, pre) - - msgs = build_messages(count, key_count) - - elapsed, _ = measure { store.append(msgs) } - times << elapsed - per_msg_times << elapsed / count - end - - row = { - keys: key_count, - pre_existing: pre, - count: count, - total_ms: (median(times) * 1000).round(3), - per_msg_ms: (median(per_msg_times) * 1000).round(3) - } - results << row - $stderr.puts " #{row[:total_ms]}ms total, #{row[:per_msg_ms]}ms/msg" - end - end -end - -# --- CSV output ------------------------------------------------------------ - -puts "keys,pre_existing,count,total_ms,per_msg_ms" -results.each do |r| - puts "#{r[:keys]},#{r[:pre_existing]},#{r[:count]},#{r[:total_ms]},#{r[:per_msg_ms]}" -end - -# --- HTML chart output ----------------------------------------------------- - -# Group by (keys, pre_existing) for charting -chart_data = {} -options[:keys].each do |k| - options[:pre_existing].each do |pre| - label = "#{k} key#{k > 1 ? 's' : ''}, #{fmt_scale(pre)} existing" - chart_data[label] = results.select { |r| r[:keys] == k && r[:pre_existing] == pre } - end -end - -colors = [ - '#2196F3', '#FF9800', '#4CAF50', '#9C27B0', '#F44336', - '#00BCD4', '#FF5722', '#8BC34A', '#3F51B5', '#CDDC39' -] - -html = <<~HTML - - - - CCC::Store#append — Benchmark - - - - -

CCC::Store#append — Benchmark

-

- Generated #{Time.now.strftime('%Y-%m-%d %H:%M')} · - Keys: #{options[:keys].join(', ')} · - Batch sizes: #{options[:counts].join(', ')} · - Pre-existing: #{options[:pre_existing].map { |s| fmt_scale(s) }.join(', ')} · - Iterations: #{options[:iterations]} -

- -
-
-

Total append time — Wall-clock time for a single - store.append(messages) call. Includes message insertion, key_pair - extraction/dedup, and message_key_pairs indexing. Varies with batch size - and number of payload attributes (keys).

- -
-
-

Per-message cost — Total time divided by message count. - Shows the marginal cost of each additional message in the batch. Should be - roughly constant if append scales linearly with batch size.

- -
-
- -

Raw Data

- - - - - - #{results.map { |r| - pre_fmt = r[:pre_existing].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse - "" - }.join("\n ")} - -
KeysPre-existingBatch sizeTotal (ms)Per-msg (ms)
#{r[:keys]}#{pre_fmt}#{r[:count]}#{r[:total_ms]}#{r[:per_msg_ms]}
- - - - -HTML - -File.write(options[:output], html) -$stderr.puts "\nChart written to #{options[:output]}" diff --git a/bench/append_results.html b/bench/append_results.html deleted file mode 100644 index 0167c234..00000000 --- a/bench/append_results.html +++ /dev/null @@ -1,402 +0,0 @@ - - - - CCC::Store#append — Benchmark - - - - -

CCC::Store#append — Benchmark

-

- Generated 2026-03-16 11:27 · - Keys: 1, 2, 3 · - Batch sizes: 1, 10, 50, 100 · - Pre-existing: 0, 1K, 10K · - Iterations: 5 -

- -
-
-

Total append time — Wall-clock time for a single - store.append(messages) call. Includes message insertion, key_pair - extraction/dedup, and message_key_pairs indexing. Varies with batch size - and number of payload attributes (keys).

- -
-
-

Per-message cost — Total time divided by message count. - Shows the marginal cost of each additional message in the batch. Should be - roughly constant if append scales linearly with batch size.

- -
-
- -

Raw Data

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KeysPre-existingBatch sizeTotal (ms)Per-msg (ms)
1010.1590.159
10100.4390.044
10502.1210.042
101003.8140.038
11,00010.1350.135
11,000100.5260.053
11,000502.3790.048
11,0001004.7990.048
110,00010.1470.147
110,000100.5420.054
110,000502.2210.044
110,0001004.5760.046
2010.110.11
20100.5750.057
20502.8650.057
201005.7280.057
21,00010.120.12
21,000100.6930.069
21,000503.1430.063
21,0001006.4010.064
210,00010.1790.179
210,000100.6980.07
210,000503.1180.062
210,0001005.8530.059
3010.1420.142
30100.8410.084
30504.0550.081
301007.1390.071
31,00010.120.12
31,000100.7340.073
31,000503.4650.069
31,0001007.6050.076
310,00010.2030.203
310,000100.8720.087
310,000503.9050.078
310,0001007.310.073
- - - - diff --git a/bench/scaling_bench.rb b/bench/scaling_bench.rb deleted file mode 100644 index bfa880ed..00000000 --- a/bench/scaling_bench.rb +++ /dev/null @@ -1,512 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Benchmark: measures claim_next performance across orders of magnitude. -# Seeds data directly via SQL for large scales, then measures a sample. -# -# Usage: -# bundle exec ruby bench/scaling_bench.rb -# bundle exec ruby bench/scaling_bench.rb --scales 10,100,1000 -# bundle exec ruby bench/scaling_bench.rb --keys 1,2 -# bundle exec ruby bench/scaling_bench.rb --output results.html - -require 'bundler/setup' -require 'sequel' -require 'sourced' -require 'sourced/ccc' -require 'sourced/ccc/store' -require 'optparse' -require 'json' - -Sourced.config.logger = Logger.new(File::NULL) -Console.logger.off! - -# --- CLI options ----------------------------------------------------------- - -options = { - scales: [10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000], - keys: [1, 2, 3], - iterations: 2, - sample_size: 50, - output: 'bench/scaling_results.html' -} - -OptionParser.new do |opts| - opts.banner = "Usage: #{$0} [options]" - opts.on('--scales LIST', 'Comma-separated partition counts') { |v| options[:scales] = v.split(',').map(&:to_i) } - opts.on('--keys LIST', 'Comma-separated key counts') { |v| options[:keys] = v.split(',').map(&:to_i) } - opts.on('--iterations N', Integer, 'Iterations per measurement') { |v| options[:iterations] = v } - opts.on('--sample N', Integer, 'Claims to measure per scenario') { |v| options[:sample_size] = v } - opts.on('--output FILE', 'HTML output file') { |v| options[:output] = v } -end.parse! - -GROUP_ID = 'bench-group' -HANDLED_TYPES = ['scaling_bench.event'] -INCR_MAX_SCALE = 10_000 # skip incremental discovery above this - -BenchEvent = Sourced::CCC::Message.define('scaling_bench.event') do - attribute :k0, String - attribute :k1, String - attribute :k2, String - attribute :k3, String -end - -# --- Fast SQL seeding ------------------------------------------------------ - -def seed(count, key_count, caught_up: false) - db = Sequel.sqlite - db.run('PRAGMA cache_size = -64000') - db.run('PRAGMA synchronous = OFF') - db.run('PRAGMA journal_mode = MEMORY') - store = Sourced::CCC::Store.new(db) - store.install! - store.register_consumer_group(GROUP_ID) - - cg_id = db[:sourced_consumer_groups].where(group_id: GROUP_ID).get(:id) - now = Time.now.iso8601 - batch = [10_000, count].min - - (0...count).each_slice(batch) do |slice| - db.transaction do - # Messages - db[:sourced_messages].multi_insert( - slice.map { |i| - payload = (0...4).map { |k| "\"k#{k}\":\"v#{i}\"" }.join(',') - { message_id: "m-#{i}", message_type: 'scaling_bench.event', payload: "{#{payload}}", created_at: now } - } - ) - - # Key pairs - slice.each { |i| - (0...key_count).each { |k| db.run("INSERT OR IGNORE INTO sourced_key_pairs (name, value) VALUES ('k#{k}', 'v#{i}')") } - } - - # Message key pairs - slice.each { |i| - (0...key_count).each { |k| - db.run("INSERT INTO sourced_message_key_pairs (message_position, key_pair_id) SELECT #{i + 1}, id FROM sourced_key_pairs WHERE name = 'k#{k}' AND value = 'v#{i}'") - } - } - - if caught_up - # Offsets - db[:sourced_offsets].multi_insert( - slice.map { |i| - pk = (0...key_count).map { |k| "k#{k}:v#{i}" }.join('|') - { consumer_group_id: cg_id, partition_key: pk, last_position: i + 1, claimed: 0 } - } - ) - - # Offset key pairs — bulk via INSERT...SELECT - (0...key_count).each { |k| - db.run(<<~SQL) - INSERT OR IGNORE INTO sourced_offset_key_pairs (offset_id, key_pair_id) - SELECT o.id, kp.id - FROM sourced_offsets o - JOIN sourced_key_pairs kp ON kp.name = 'k#{k}' - AND kp.value = SUBSTR(o.partition_key, #{k > 0 ? "INSTR(o.partition_key, 'k#{k}:') + #{k.to_s.length + 2}" : '4'}, LENGTH(o.partition_key)) - WHERE o.consumer_group_id = #{cg_id} - AND o.id NOT IN (SELECT offset_id FROM sourced_offset_key_pairs) - SQL - } - end - end - $stderr.print '.' - end - - if caught_up - db[:sourced_consumer_groups].where(id: cg_id).update( - highest_position: count, discovery_position: count, updated_at: now - ) - end - - # Restore WAL mode for benchmarking - db.run('PRAGMA synchronous = FULL') - db.run('PRAGMA journal_mode = WAL') - - [db, store] -end - -# Simpler caught_up seeding: parse partition_key to match key_pairs -# For the bulk offset_key_pairs insert, extract the value from partition_key -# which has format "k0:v123" or "k0:v123|k1:v123" -def seed_caught_up_fast(count, key_count, eager: false) - db = Sequel.sqlite - db.run('PRAGMA cache_size = -64000') - db.run('PRAGMA synchronous = OFF') - db.run('PRAGMA journal_mode = MEMORY') - store = Sourced::CCC::Store.new(db) - store.install! - if eager - store.register_consumer_group(GROUP_ID, partition_by: partition_keys(key_count)) - else - store.register_consumer_group(GROUP_ID) - end - - cg_id = db[:sourced_consumer_groups].where(group_id: GROUP_ID).get(:id) - now = Time.now.iso8601 - batch = [10_000, count].min - - (0...count).each_slice(batch) do |slice| - db.transaction do - # Messages - db[:sourced_messages].multi_insert( - slice.map { |i| - payload = (0...4).map { |k| "\"k#{k}\":\"v#{i}\"" }.join(',') - { message_id: "m-#{i}", message_type: 'scaling_bench.event', payload: "{#{payload}}", created_at: now } - } - ) - - # Key pairs + message_key_pairs - slice.each { |i| - (0...key_count).each { |k| - db.run("INSERT OR IGNORE INTO sourced_key_pairs (name, value) VALUES ('k#{k}', 'v#{i}')") - db.run("INSERT INTO sourced_message_key_pairs (message_position, key_pair_id) SELECT #{i + 1}, id FROM sourced_key_pairs WHERE name = 'k#{k}' AND value = 'v#{i}'") - } - } - - # Offsets - db[:sourced_offsets].multi_insert( - slice.map { |i| - pk = (0...key_count).map { |k| "k#{k}:v#{i}" }.join('|') - { consumer_group_id: cg_id, partition_key: pk, last_position: i + 1, claimed: 0 } - } - ) - - # Offset key pairs — per-row but using subquery - slice.each { |i| - pk = (0...key_count).map { |k| "k#{k}:v#{i}" }.join('|') - (0...key_count).each { |k| - db.run("INSERT OR IGNORE INTO sourced_offset_key_pairs (offset_id, key_pair_id) SELECT o.id, kp.id FROM sourced_offsets o, sourced_key_pairs kp WHERE o.partition_key = '#{pk}' AND o.consumer_group_id = #{cg_id} AND kp.name = 'k#{k}' AND kp.value = 'v#{i}'") - } - } - end - $stderr.print '.' - end - - db[:sourced_consumer_groups].where(id: cg_id).update( - highest_position: count, discovery_position: count, updated_at: now - ) - db.run('PRAGMA synchronous = FULL') - db.run('PRAGMA journal_mode = WAL') - - [db, store] -end - -# --- Measurement helpers --------------------------------------------------- - -def measure - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - result = yield - [Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0, result] -end - -def median(values) - sorted = values.sort - mid = sorted.size / 2 - sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0 -end - -def partition_keys(n) = (0...n).map { |i| "k#{i}" } - -def claim_once(store, key_count) - store.claim_next(GROUP_ID, partition_by: partition_keys(key_count), handled_types: HANDLED_TYPES, worker_id: 'w-1') -end - -# Adaptive iterations — fewer for large scales to keep total time reasonable -def effective_iterations(scale, base) - case scale - when 0..10_000 then base - when 10_001..100_000 then [base, 2].min - else 1 - end -end - -# --- Benchmark runner ------------------------------------------------------ - -results = [] - -options[:keys].each do |key_count| - options[:scales].each do |scale| - label = "keys=#{key_count} scale=#{format('%10d', scale)}" - $stderr.print "\n#{label}" - iters = effective_iterations(scale, options[:iterations]) - - row = { keys: key_count, scale: scale } - - # --- 1. Idle poll (all caught up) --- - $stderr.print " [idle" - idle_times = [] - iters.times do - _db, store = seed_caught_up_fast(scale, key_count) - polls = 5.times.map { measure { claim_once(store, key_count) }.first } - idle_times << median(polls) - end - row[:idle_poll_ms] = (median(idle_times) * 1000).round(4) - $stderr.print "=#{row[:idle_poll_ms]}ms]" - - # --- 2. Cold drain (sample first N claims) --- - $stderr.print " [cold" - sample = [options[:sample_size], scale].min - drain_times = [] - iters.times do - _db, store = seed(scale, key_count, caught_up: false) - times = [] - sample.times do - t, r = measure { claim_once(store, key_count) } - break unless r - store.ack(GROUP_ID, offset_id: r.offset_id, position: r.messages.last.position) - times << t - end - drain_times << median(times) if times.any? - end - row[:per_claim_cold_ms] = drain_times.any? ? (median(drain_times) * 1000).round(4) : 0 - $stderr.print "=#{row[:per_claim_cold_ms]}ms]" - - # --- 3. Warm claim (sample N claims with new messages) --- - $stderr.print " [warm" - warm_times = [] - iters.times do - _db, store = seed_caught_up_fast(scale, key_count) - msgs = (0...sample).map { |i| BenchEvent.new(payload: { k0: "v#{i}", k1: "v#{i}", k2: "v#{i}", k3: "v#{i}" }) } - store.append(msgs) - - times = [] - sample.times do - t, r = measure { claim_once(store, key_count) } - break unless r - store.ack(GROUP_ID, offset_id: r.offset_id, position: r.messages.last.position) - times << t - end - warm_times << median(times) if times.any? - end - row[:per_claim_warm_ms] = warm_times.any? ? (median(warm_times) * 1000).round(4) : 0 - $stderr.print "=#{row[:per_claim_warm_ms]}ms]" - - # --- 4. Incremental discovery (1 new partition) --- - if scale <= INCR_MAX_SCALE - $stderr.print " [incr" - incr_times = [] - iters.times do - _db, store = seed_caught_up_fast(scale, key_count) - store.append(BenchEvent.new(payload: { k0: 'vnew', k1: 'vnew', k2: 'vnew', k3: 'vnew' })) - t, _ = measure { claim_once(store, key_count) } - incr_times << t - end - row[:incremental_ms] = (median(incr_times) * 1000).round(4) - $stderr.print "=#{row[:incremental_ms]}ms]" - else - row[:incremental_ms] = nil - end - - # --- 5. Eager warm claim (offsets created by append, no discovery) --- - $stderr.print " [eager-warm" - eager_warm_times = [] - iters.times do - _db, store = seed_caught_up_fast(scale, key_count, eager: true) - msgs = (0...sample).map { |i| BenchEvent.new(payload: { k0: "v#{i}", k1: "v#{i}", k2: "v#{i}", k3: "v#{i}" }) } - store.append(msgs) - - times = [] - sample.times do - t, r = measure { claim_once(store, key_count) } - break unless r - store.ack(GROUP_ID, offset_id: r.offset_id, position: r.messages.last.position) - times << t - end - eager_warm_times << median(times) if times.any? - end - row[:eager_warm_ms] = eager_warm_times.any? ? (median(eager_warm_times) * 1000).round(4) : 0 - $stderr.print "=#{row[:eager_warm_ms]}ms]" - - # --- 6. Eager incremental (1 new partition, offset created by append) --- - if scale <= INCR_MAX_SCALE - $stderr.print " [eager-incr" - eager_incr_times = [] - iters.times do - _db, store = seed_caught_up_fast(scale, key_count, eager: true) - store.append(BenchEvent.new(payload: { k0: 'vnew', k1: 'vnew', k2: 'vnew', k3: 'vnew' })) - t, _ = measure { claim_once(store, key_count) } - eager_incr_times << t - end - row[:eager_incremental_ms] = (median(eager_incr_times) * 1000).round(4) - $stderr.print "=#{row[:eager_incremental_ms]}ms]" - else - row[:eager_incremental_ms] = nil - end - - results << row - end -end - -# --- CSV output ------------------------------------------------------------ - -$stderr.puts "\n" -puts "keys,scale,idle_poll_ms,per_claim_cold_ms,per_claim_warm_ms,incremental_ms,eager_warm_ms,eager_incremental_ms" -results.each do |r| - incr = r[:incremental_ms] ? r[:incremental_ms].to_s : '' - eager_incr = r[:eager_incremental_ms] ? r[:eager_incremental_ms].to_s : '' - puts "#{r[:keys]},#{r[:scale]},#{r[:idle_poll_ms]},#{r[:per_claim_cold_ms]},#{r[:per_claim_warm_ms]},#{incr},#{r[:eager_warm_ms]},#{eager_incr}" -end - -# --- HTML chart output ----------------------------------------------------- - -chart_data = {} -options[:keys].each { |k| chart_data[k] = results.select { |r| r[:keys] == k } } - -def fmt_scale(s) - return "#{s / 1_000_000}M" if s >= 1_000_000 - return "#{s / 1_000}K" if s >= 1_000 - s.to_s -end - -html = <<~HTML - - - - CCC::Store#claim_next — Scaling Benchmark - - - - -

CCC::Store#claim_next — Scaling Benchmark

-

- Generated #{Time.now.strftime('%Y-%m-%d %H:%M')} · - Keys: #{options[:keys].join(', ')} · - Scales: #{options[:scales].map { |s| fmt_scale(s) }.join(', ')} · - Iterations: #{options[:iterations]} (adaptive) · - Sample: #{options[:sample_size]} claims -

- -
-
-

Idle Poll — All partitions are fully caught up. No new messages exist. - This is what happens on every catch-up poll interval when the system is quiet. - Measures the cost of determining “nothing to do”.

- -
-
-

Cold Drain — A new consumer group starts processing an existing log from scratch. - No offsets exist yet — each claim_next call must discover new partitions, create offsets, - and claim work. This is the per-claim cost during initial catch-up (sampled over #{options[:sample_size]} claims).

- -
-
-

Warm Claim — All partitions have existing offsets (previously caught up), - then new messages arrive for some partitions. Legacy path runs discovery to advance watermark. - This is the steady-state cost when the notifier or catch-up poller triggers processing.

- -
-
-

Eager Warm Claim — Same as Warm Claim, but with partition_by - registered. Offsets are created during append, so claim_next goes straight - to the fast path — no discovery CTE needed.

- -
-
-

Incremental Discovery — All existing partitions are caught up. - One message arrives for a brand-new partition (never seen before). Measures the cost of discovering - and claiming that single new partition against a backdrop of N existing offsets. - Skipped for scales > #{fmt_scale(INCR_MAX_SCALE)} due to prohibitive NOT EXISTS cost.

- -
-
-

Eager Incremental — Same scenario as Incremental, but with eager offset - creation. The offset is created during append, so claim_next finds it - on the fast path without running the discovery CTE. - Skipped for scales > #{fmt_scale(INCR_MAX_SCALE)}.

- -
-
- -

Raw Data

- - - - - - #{results.map { |r| - scale_fmt = r[:scale].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse - incr_fmt = r[:incremental_ms] ? r[:incremental_ms].to_s : '—' - eager_incr_fmt = r[:eager_incremental_ms] ? r[:eager_incremental_ms].to_s : '—' - "" - }.join("\n ")} - -
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm (ms)Eager Warm (ms)Incr (ms)Eager Incr (ms)
#{r[:keys]}#{scale_fmt}#{r[:idle_poll_ms]}#{r[:per_claim_cold_ms]}#{r[:per_claim_warm_ms]}#{r[:eager_warm_ms]}#{incr_fmt}#{eager_incr_fmt}
- - - - -HTML - -File.write(options[:output], html) -$stderr.puts "Chart written to #{options[:output]}" diff --git a/bench/scaling_results.html b/bench/scaling_results.html deleted file mode 100644 index a70f70ca..00000000 --- a/bench/scaling_results.html +++ /dev/null @@ -1,311 +0,0 @@ - - - - CCC::Store#claim_next — Scaling Benchmark - - - - -

CCC::Store#claim_next — Scaling Benchmark

-

- Generated 2026-03-16 12:09 · - Keys: 1, 2, 3 · - Scales: 100, 1K, 10K, 100K, 1M · - Iterations: 2 (adaptive) · - Sample: 50 claims -

- -
-
-

Idle Poll — All partitions are fully caught up. No new messages exist. - This is what happens on every catch-up poll interval when the system is quiet. - Measures the cost of determining “nothing to do”.

- -
-
-

Cold Drain — A new consumer group starts processing an existing log from scratch. - No offsets exist yet — each claim_next call must discover new partitions, create offsets, - and claim work. This is the per-claim cost during initial catch-up (sampled over 50 claims).

- -
-
-

Warm Claim — All partitions have existing offsets (previously caught up), - then new messages arrive for some partitions. Legacy path runs discovery to advance watermark. - This is the steady-state cost when the notifier or catch-up poller triggers processing.

- -
-
-

Eager Warm Claim — Same as Warm Claim, but with partition_by - registered. Offsets are created during append, so claim_next goes straight - to the fast path — no discovery CTE needed.

- -
-
-

Incremental Discovery — All existing partitions are caught up. - One message arrives for a brand-new partition (never seen before). Measures the cost of discovering - and claiming that single new partition against a backdrop of N existing offsets. - Skipped for scales > 10K due to prohibitive NOT EXISTS cost.

- -
-
-

Eager Incremental — Same scenario as Incremental, but with eager offset - creation. The offset is created during append, so claim_next finds it - on the fast path without running the discovery CTE. - Skipped for scales > 10K.

- -
-
- -

Raw Data

- - - - - - - - - - - - - - - - - - - - - -
KeysPartitionsIdle Poll (ms)Cold /claim (ms)Warm (ms)Eager Warm (ms)Incr (ms)Eager Incr (ms)
11000.15450.4430.48650.49250.80350.8365
11,0000.1131.34781.33231.3181.8661.438
110,0000.118516.40279.74959.65410.1729.476
1100,0000.1205133.7683102.8572102.696——
11,000,0000.1111377.7551090.8711093.1885——
21000.11250.56730.5760.57050.9880.62
21,0000.11851.96351.98282.02482.6192.2775
210,0000.1227.249716.44116.270516.916516.084
2100,0000.1155253.194179.9335179.7775——
21,000,0000.1162594.0711874.92851883.2475——
31000.1220.55830.64330.61851.220.6865
31,0000.11652.35582.39782.40682.91452.5075
310,0000.11435.423720.486720.383320.981520.2955
3100,0000.136356.1052226.6478226.9365——
31,000,0000.1183647.8922357.1292351.4825——
- - - - From e1a6a7816fa80cfc0ab7b32567a93cad21297348 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 14:22:25 +0100 Subject: [PATCH 108/115] Ignore bench/ for now --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6f5b75ea..298fbaeb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ logs/ pkg .DS_Store .claude +bench/ + From d95dd5771f28367f213593d16761504e23d71e09 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 15:43:32 +0100 Subject: [PATCH 109/115] Update CLAUDE.md to reflect CCC APIs --- CLAUDE.md | 241 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 124 insertions(+), 117 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2855e28d..e99edd8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,167 +4,174 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -Sourced is an Event Sourcing / CQRS library for Ruby built around the "Decide, Evolve, React" pattern. It provides eventual consistency by default with an actor-like execution model for building event-sourced applications. +Sourced is a Ruby library for **aggregateless, stream-less event sourcing**. Messages go into a flat, globally-ordered log (SQLite via Sequel). Consistency context is assembled dynamically by querying relevant facts via key-value pairs extracted from event payloads, rather than being pre-assigned to fixed streams. Reactors declare partition keys which the store uses to build query conditions and claim work. ## Core Architecture -### Key Components -- **Actors**: Classes that hold state, handle commands, produce events, and react to events (lib/sourced/actor.rb) -- **Commands**: Intents to effect change in the system -- **Events**: Facts describing state changes that have occurred -- **Projectors**: React to events to build views, caches, or other representations (lib/sourced/projector.rb) -- **Backends**: Storage adapters (ActiveRecord, Sequel, test backend) in lib/sourced/backends/ -- **Router**: Routes commands and events to appropriate handlers (lib/sourced/router.rb) -- **Supervisor**: Manages background worker processes (lib/sourced/supervisor.rb) +### Key abstractions -### Message Flow -Commands → Actors (Decide) → Events → Storage → Reactors (React) → New Commands +- **Message** (`lib/sourced/message.rb`) — base class for all commands/events. No `stream_id` or `seq`; gets a global `position` when stored. Provides `causation_id` / `correlation_id`, `#correlate`, `#extracted_keys`, and a `Registry`. Subclasses: `Sourced::Command`, `Sourced::Event`. +- **Store** (`lib/sourced/store.rb`) — SQLite-backed append-only log with key-pair indexing, consumer groups, scheduled messages, and stale-claim reaping. Returns `ReadResult`, `ClaimResult`, `ConsistencyGuard`, `PositionedMessage`, `Stats`, `OffsetsResult`, `ReadAllResult`. +- **Reactor base classes** — all `extend Sourced::Consumer` and declare `partition_by :key` (+ other keys) to define their consistency boundary: + - `Decider` (`lib/sourced/decider.rb`) — handles commands, produces events via `event` helper. + - `Projector` (`lib/sourced/projector.rb`) — builds read models. Two flavors: `Projector::StateStored` and `Projector::EventSourced`. + - `DurableWorkflow` (`lib/sourced/durable_workflow.rb`) — long-running workflows with step memoisation via `durable`/`wait`/`context`/`execute` and `catch(:halt)`. + - Plain `Consumer` reactors (extend `Sourced::Consumer` directly) for side-effect-only handlers. +- **Mixins**: `Sourced::Evolve` (state evolution from history), `Sourced::React` (event → command/event reactions), `Sourced::Sync` (post-append side effects). +- **Router** (`lib/sourced/router.rb`) — registers reactors, dispatches claimed batches, manages consumer-group lifecycle hooks. +- **Dispatcher / Worker / WorkQueue** (`lib/sourced/{dispatcher,worker,work_queue}.rb`) — claim-and-drain processing; signal-driven via `InlineNotifier` + `CatchUpPoller`. +- **StaleClaimReaper** (`lib/sourced/stale_claim_reaper.rb`) — releases abandoned partition claims from dead workers via heartbeats. +- **ScheduledMessagePoller** (`lib/sourced/scheduled_message_poller.rb`) — promotes due scheduled messages into the main log. +- **Supervisor** (`lib/sourced/supervisor.rb`) — top-level process entry point wiring Dispatcher, executor, and reactors. +- **CommandContext** (`lib/sourced/command_context.rb`) — builds commands from raw attributes; supports per-message and `any` hooks. +- **Topology** (`lib/sourced/topology.rb`) — graph of reactors / message flows. +- **Installer + migrations** (`lib/sourced/installer.rb`, `lib/sourced/migrations/`) — Sequel migration template for installing store tables. +- **Falcon integration** (`lib/sourced/falcon/`) — `Environment` + `Service` for deferred post-fork setup. -### Concurrency Model -Sourced processes events by acquiring locks on `[reactor_group_id][stream_id]` combinations, ensuring sequential processing within streams while allowing concurrent processing across different streams. +### Message flow + +`Command → Decider.decide → Events → Store.append → Router claims → Reactor.handle_claim → (Projections / Reactions / Sync actions)` + +All reactors implement `.handle_claim(claim, history:)` and/or `.handle_batch(partition_values, new_messages, history:, replaying:)` with a uniform signature so GWT helpers and partial-ack logic work across types. + +### Partition-based consistency + +Reactors declare `partition_by :key1, :key2`. The store indexes every payload attribute into `sourced_key_pairs` at append time, and reads use AND-filtered conditions over these keys. `ConsistencyGuard` (returned by `read` / `claim_next`) detects conflicting appends via `messages_since(conditions, position)`. ## Development Commands ### Testing -```bash -# Run all tests (default rake task) -rake - -# Run specific test file -bundle exec rspec spec/actor_spec.rb -# Run backend tests -bundle exec rspec spec/backends/ +```bash +# Full suite +bundle exec rake -# Run with specific database (PostgreSQL required for some tests) -DATABASE_URL=postgres://localhost/sourced_test bundle exec rspec +# Specific file +bundle exec rspec spec/store_spec.rb +bundle exec rspec spec/decider_spec.rb ``` -### Database Setup for Tests -The gem supports multiple backends: -- PostgreSQL (via Sequel or ActiveRecord) -- SQLite (via Sequel or ActiveRecord) -- In-memory test backend +Tests use in-memory SQLite by default. `spec/store_spec.rb` is the central integration suite; `spec/testing/rspec_spec.rb` covers the GWT helpers in `lib/sourced/testing/rspec.rb`. -Test databases are automatically created/cleared by the test suite. +### Console -### Console/IRB ```bash -# Interactive console for experimentation bin/console ``` -## Configuration Patterns +## Configuration -### Backend Configuration ```ruby -# PostgreSQL via Sequel (default production setup) -Sourced.configure do |config| - config.backend = Sequel.connect(ENV.fetch('DATABASE_URL')) -end - -# Test backend (default, in-memory) Sourced.configure do |config| - config.backend = Sourced::Backends::TestBackend.new + config.store = Sequel.sqlite('my_app.db') # auto-wraps in Sourced::Store + # or: config.store = Sourced::Store.new(db) + # or: any object matching Configuration::StoreInterface + config.worker_count = 2 + config.batch_size = 50 + config.catchup_interval = 5 + config.claim_ttl_seconds = 120 end -``` -### Registering Components -```ruby -# Register actors and projectors for background processing -Sourced.register(SomeActor) +Sourced.register(SomeDecider) Sourced.register(SomeProjector) ``` -## Key DSL Patterns +- `Sourced.configure` stores the block and calls `setup!`; re-runnable post-fork to re-establish DB connections (used by Falcon integration). +- `Sourced.store`, `Sourced.router`, `Sourced.topology`, `Sourced.reset!` — module-level accessors. +- `Sourced.handle!(ReactorClass, command)` — synchronous command dispatch (for web controllers): validates, loads history via partition read, decides, appends with guard, advances registered offsets. Returns `HandleResult(command, reactor, events)`. +- `Sourced.load(ReactorClass, **partition_values)` — loads a reactor instance by evolving over AND-filtered partition history. Returns `[instance, read_result]`. + +## DSL Patterns + +### Decider -### Actor Definition ```ruby -class SomeActor < Sourced::Actor - # Initial state factory - state do |id| - { id: id, status: 'new' } - end - - # Command handler - command :create_something, name: String do |state, cmd| - event :something_created, cmd.payload +class Courses < Sourced::Decider + partition_by :course_id + + command CreateCourse do |_state, cmd| + event CourseCreated, course_id: cmd.payload.course_id, course_name: cmd.payload.course_name end - - # Event handler (state evolution) - event :something_created, name: String do |state, event| - state[:name] = event.payload.name + + event CourseCreated do |state, evt| + state[:course_id] = evt.payload.course_id + state[:name] = evt.payload.course_name end - - # Reaction (workflow orchestration) - reaction :something_created do |event| - stream_for(event).command :next_step + + reaction CourseCreated do |evt| + dispatch SendWelcomeEmail, course_id: evt.payload.course_id end end ``` -### Message Definitions +### Message definition + ```ruby -# Expanded syntax for complex validation/coercion -CreateLead = Sourced::Command.define('leads.create') do - attribute :name, Types::String.present - attribute :email, Types::Email.present +CreateCourse = Sourced::Command.define('courses.create') do + attribute :course_id, Types::String.present + attribute :course_name, Types::String.present end -LeadCreated = Sourced::Event.define('leads.created') do - attribute :name, String - attribute :email, String +CourseCreated = Sourced::Event.define('courses.created') do + attribute :course_id, String + attribute :course_name, String end ``` -## Backend Implementation Notes +`Sourced::Command` and `Sourced::Event` each have their own `Registry` (both reachable from `Sourced::Message.registry` via recursive lookup). + +### Projector flavors -- All backends must implement the BackendInterface defined in lib/sourced/configuration.rb -- SequelBackend is the main production backend (lib/sourced/backends/sequel_backend.rb) -- ActiveRecordBackend provides Rails integration (lib/sourced/backends/active_record_backend.rb) -- TestBackend provides in-memory storage for testing (lib/sourced/backends/test_backend.rb) +- `Projector::StateStored` — evolves only the claimed batch on top of the stored state snapshot. +- `Projector::EventSourced` — evolves from full history every claim (via `context_for`). + +### Scheduled / delayed messages + +```ruby +cmd = SendReminder.new(payload: { course_id: 'c1' }).at(Time.now + 3600) +store.schedule_messages([cmd]) +store.update_schedule! # manual promotion (normally done by ScheduledMessagePoller) +``` -## Testing Considerations +In reactions: `dispatch(Cmd, ...).at(time)`. -- Use shared examples from spec/shared_examples/backend_examples.rb when testing backends -- Time manipulation available via Timecop gem -- Database isolation handled automatically per test -- Concurrent testing patterns available for testing race conditions +## Store API highlights + +- `append(messages, guard: nil)` — writes + auto-indexes payload keys; raises `ConcurrentAppendError` if guard is violated. +- `read(conditions, after_position:, limit:)` → `ReadResult(messages, guard)`. +- `read_partition(partition_attrs, handled_types:)` — AND-filtered read for loading reactor state. +- `read_all(after_position:, limit:, order: :asc, conditions: nil)` → `ReadAllResult` (lazy pagination via `to_enum`). +- `claim_next(reactor, worker_id:)` → `ClaimResult` with partition batch + guard. Supports compound partitions and replaying flag. +- `ack(claim, last_position:)` / `release(claim)` / `advance_offset(group_id, partition:, position:)`. +- `register_consumer_group`, `start_consumer_group`, `stop_consumer_group`, `reset_consumer_group`. +- `read_offsets(group_id:, limit:, from_id:)` → `OffsetsResult` (cursor-paginated, `to_enum`). +- `stats` → `Stats(max_position, groups)` including `error_context`. +- `worker_heartbeat` / `release_stale_claims` — claim liveness. + +## Testing + +- `lib/sourced/testing/rspec.rb` provides GWT helpers (`given`/`when_`/`then_`) usable across all reactor types since `#handle_batch` has a uniform signature. +- Shared store behaviour concentrated in `spec/store_spec.rb` (2400+ lines). +- Durable workflow specs demonstrate the step-memoisation pattern. ## Error Handling -- Default error strategy logs exceptions and stops consumer groups -- Configurable retry/backoff strategies available -- Consumer groups can be stopped/started programmatically via backend API - -## CCC Module — Stream-less Event Sourcing (Experimental) - -`Sourced::CCC` is a prototype for aggregateless, stream-less event sourcing inspired by "Context-driven Consistency Checks". Events go into a flat, globally-ordered log. Consistency context is assembled dynamically by querying relevant facts via normalized key-value pairs extracted from event payloads. - -### Files -- `lib/sourced/ccc.rb` — module entrypoint -- `lib/sourced/ccc/message.rb` — `CCC::Message` base class, `QueryCondition` -- `lib/sourced/ccc/store.rb` — `CCC::Store` (SQLite), `PositionedMessage` wrapper -- `spec/sourced/ccc/` — specs (40 examples) - -### CCC::Message -- Extends `Types::Data` like `Sourced::Message`, but without `stream_id`, `seq`, `causation_id`, `correlation_id` -- Own `Registry`, separate from `Sourced::Message.registry` -- `.define(type_str, &block)` — creates subclass with typed payload (same DSL as `Sourced::Message`) -- `.from(hash)` — instantiate correct subclass from type string -- `#extracted_keys` — auto-extracts `[[name, value], ...]` from all top-level payload attributes (skips nils) - -### CCC::Store -- Accepts a `Sequel::SQLite::Database` connection -- 3 tables: `ccc_messages` (append-only log with auto-increment position), `ccc_key_pairs` (deduplicated name/value pairs), `ccc_message_key_pairs` (join table) -- `append(messages)` — writes messages + auto-indexes all payload keys, returns last position -- `read(conditions, from_position:, limit:)` — queries by `QueryCondition` array (message_type + key_name + key_value), OR semantics -- `messages_since(conditions, position)` — conflict detection (messages matching conditions after position) -- `PositionedMessage` — `SimpleDelegator` wrapper adding `#position` to frozen `Types::Data` instances - -### CCC::QueryCondition -- `Data.define(:message_type, :key_name, :key_value)` — used to query the store - -### Design Reference -- Article: `plans/ccc/ccc.md` -- TypeScript reference: [Boundless SQLite storage](https://github.com/SBortz/boundless) \ No newline at end of file +- `error_strategy` on `Configuration` — configurable retry / backoff / fail. See `lib/sourced/error_strategy.rb`. +- Consumer groups have `running` / `stopped` / `failed` states. `on_fail` fires on terminal failures. +- `PartialBatchError` carries successfully-processed `action_pairs` plus the failing message so batches can be partially acked. + +## Key Files + +- Entrypoint: `lib/sourced.rb` (top-level API, `handle!`, `load`) +- Store: `lib/sourced/store.rb` + `lib/sourced/installer.rb` + `lib/sourced/migrations/` +- Reactors: `lib/sourced/{decider,projector,durable_workflow,consumer}.rb` +- Mixins: `lib/sourced/{evolve,react,sync}.rb` +- Dispatch: `lib/sourced/{dispatcher,worker,work_queue,stale_claim_reaper,scheduled_message_poller,inline_notifier}.rb` +- Router/topology: `lib/sourced/{router,topology}.rb` +- Messages: `lib/sourced/message.rb` (includes `QueryCondition`, `ConsistencyGuard`) +- Falcon: `lib/sourced/falcon/{environment,service}.rb` +- Testing: `lib/sourced/testing/rspec.rb` + +## Local scratch (untracked) + +`examples/app/` and `bench/*` are intentionally untracked (see commits `Untrack examples/app` / `Untrack bench files`). Files may exist locally for experimentation but must not be re-added to the repo without explicit approval. From 42135c330e0f0b984418a9c1453695a4b22227c2 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Wed, 15 Apr 2026 23:54:37 +0100 Subject: [PATCH 110/115] Update README --- README.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 61d5cbda..901dfca0 100644 --- a/README.md +++ b/README.md @@ -476,7 +476,7 @@ end ### Reactions -Both deciders and projectors can react to events to produce new commands or events, enabling workflow orchestration. +Both deciders and projectors can react to events to produce follow-up messages, enabling workflow orchestration. Reaction blocks queue messages via the `dispatch` helper rather than returning them — `dispatch` correlates each message with the triggering event, tags it with the reactor's `group_id` as `metadata[:producer]`, and returns a chainable `Dispatcher`. ```ruby class EnrolmentDecider < Sourced::Decider @@ -484,13 +484,42 @@ class EnrolmentDecider < Sourced::Decider # ... evolve and command handlers ... - # React to an event by producing new messages + # Dispatch a follow-up message by class reaction StudentEnrolled do |state, event| - NotifyStudent.new(payload: { student_id: event.payload.student_id }) + dispatch(NotifyStudent, student_id: event.payload.student_id) + end + + # Dispatch multiple messages from one block + reaction OrderPlaced do |_state, event| + dispatch(ReserveInventory, order_id: event.payload.order_id) + dispatch(ChargePayment, order_id: event.payload.order_id) + end + + # Chain .with_metadata and .at for metadata/delay + reaction StudentEnrolled do |_state, event| + dispatch(SendReminder, student_id: event.payload.student_id) + .with_metadata(channel: 'email') + .at(Time.now + 300) + end + + # Dispatch by symbol (resolved via the reactor's message registry) + reaction :student_enrolled do |_state, event| + dispatch(:notify_student, student_id: event.payload.student_id) + end + + # Wildcard: react to every evolve type without an explicit handler. + # Useful for side-channel pipelines (audit logs, outbox, etc.). + reaction do |_state, event| + dispatch(ForwardToOutbox, event_id: event.id) end end ``` +`dispatch` accepts either a message class or a symbol. Symbols are looked up via `self.class[symbol]` against the reactor's command/event registry and raise if unresolved. The returned `Dispatcher` supports: + +- `.with_metadata(hash)` — merges into the message metadata (the `producer` key is already set). +- `.at(time)` — stamps the message for delayed delivery (promoted by the `ScheduledMessagePoller`). + Reactions are skipped during replay (when `replaying: true`), so side effects don't re-fire. ### Sync and After-Sync Blocks From e76d3b12ab80045246436d04fcc198a0d5bf73f9 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 16 Apr 2026 11:25:17 +0100 Subject: [PATCH 111/115] Style tweaks --- lib/sourced/actions.rb | 5 +++-- lib/sourced/projector.rb | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/sourced/actions.rb b/lib/sourced/actions.rb index 48eae73e..e93ed749 100644 --- a/lib/sourced/actions.rb +++ b/lib/sourced/actions.rb @@ -18,6 +18,7 @@ def self.build_for(messages, guard: nil, source: nil, correlated: false) messages = Array(messages) return actions if messages.empty? + # TODO: review use of Time.now now = Time.now to_schedule, to_append = messages.partition { |message| message.created_at > now } @@ -58,13 +59,13 @@ def correlated? = @correlated # @param source_message [Sourced::Message] default message to correlate from # @return [Array] correlated messages that were appended def execute(store, source_message) - to_append = if @correlated + to_append = if correlated? messages else correlate_from = @source || source_message messages.map { |m| correlate_from.correlate(m) } end - store.append(to_append, guard: guard) + store.append(to_append, guard:) to_append end end diff --git a/lib/sourced/projector.rb b/lib/sourced/projector.rb index a62541a6..07214e4f 100644 --- a/lib/sourced/projector.rb +++ b/lib/sourced/projector.rb @@ -20,7 +20,7 @@ def handled_messages def build_action_pairs(instance, messages, replaying:) sync_actions = instance.collect_actions( - state: instance.state, messages: messages, replaying: replaying + state: instance.state, messages:, replaying: ) reaction_pairs = if replaying @@ -51,7 +51,7 @@ class << self def handle_batch(partition_values, new_messages, history: nil, replaying: false) instance = new(partition_values) instance.evolve(new_messages) - build_action_pairs(instance, new_messages, replaying: replaying) + build_action_pairs(instance, new_messages, replaying:) end # @param claim [ClaimResult] claimed partition batch @@ -69,7 +69,7 @@ class << self def handle_batch(partition_values, new_messages, history:, replaying: false) instance = new(partition_values) instance.evolve(history.messages) - build_action_pairs(instance, new_messages, replaying: replaying) + build_action_pairs(instance, new_messages, replaying:) end # @param claim [ClaimResult] claimed partition batch From f621f7b2829a4f581e7c721cf6c6d9444a64812b Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 16 Apr 2026 15:25:44 +0200 Subject: [PATCH 112/115] Defer reactions in Decider.handle_batch Reactions no longer run inline with the command that produced the triggering event. The Decider's own subscription (via handled_messages_for_react) re-claims the event on the next cycle and runs react() in a separate handle_batch invocation. - The originating command's after_sync fires as soon as events commit instead of waiting for every reaction to finish. - Command and reactions are now in separate transactions; a failing reaction does not roll back the command. - handle_claim forwards claim.replaying to handle_batch; the reaction branch skips on replay, matching Projector. Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 4 +- lib/sourced/decider.rb | 69 ++++++++++++++++++++++++++----- spec/decider_spec.rb | 83 +++++++++++++++++++++++++++++++------- spec/dispatcher_spec.rb | 6 +++ spec/testing/rspec_spec.rb | 16 ++++++-- 5 files changed, 148 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e99edd8f..f49d5e09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,9 @@ Sourced is a Ruby library for **aggregateless, stream-less event sourcing**. Mes ### Message flow -`Command → Decider.decide → Events → Store.append → Router claims → Reactor.handle_claim → (Projections / Reactions / Sync actions)` +`Command → Decider.decide → Events → Store.append → Router claims → Reactor.handle_claim → (Projections / Sync actions)` + +Reactions are **deferred**: a Decider's `react` blocks don't run inline with the command that produced the triggering event. When the Decider appends events, its own subscription (`handled_messages_for_react`) picks them up on the next claim cycle and runs the reaction in a separate `handle_batch`. Consequence: the originating command's `after_sync` commits as soon as its events commit, not after reactions finish. Trade-off: command and reactions are no longer in the same transaction — a failing reaction does not roll back the command. All reactors implement `.handle_claim(claim, history:)` and/or `.handle_batch(partition_values, new_messages, history:, replaying:)` with a uniform signature so GWT helpers and partial-ack logic work across types. diff --git a/lib/sourced/decider.rb b/lib/sourced/decider.rb index 917ab4db..9ec994f0 100644 --- a/lib/sourced/decider.rb +++ b/lib/sourced/decider.rb @@ -34,6 +34,53 @@ def command(message_class, &block) define_method(Sourced.message_method_name(PREFIX, message_class.to_s), &block) end + # Build executable actions for a batch of claimed messages. + # + # Each message in +new_messages+ routes through one of three branches: + # + # 1. **Command** — if the message class is in {.handled_commands}, the + # decider runs {Decider#decide}, produces new events, and wraps them + # in an {Actions::Append} (guarded by +history.guard+). Per-message + # +sync+ / +after_sync+ actions are collected. + # + # 2. **Reaction-triggering event** — if the decider + # {React#reacts_to? reacts to} this message (and the batch is not + # replaying), its state is evolved with the event, {React#react} is + # invoked, and any produced messages are wrapped in + # {Actions::Append} / {Actions::Schedule} with +source: msg+ so + # the infra layer correlates reaction messages against the event. + # + # 3. **Anything else** — yields +[Actions::OK, msg]+ (the message is + # acked with no side effects). In practice this is unreachable + # because claim filtering uses {.handled_messages}, but it's kept + # as a safety net. + # + # ### Reactions are deferred + # + # Reactions do **not** run inline with the command that produced the + # triggering event. A command's batch only appends its events; the + # decider's own subscription (via +handled_messages_for_react+) picks + # those events up on the next claim cycle and runs the reaction in a + # separate +handle_batch+ invocation. + # + # Consequences: + # + # - The originating command's +after_sync+ fires as soon as its + # events commit — no longer blocked by slow reactions. + # - Command and its reactions are in **separate transactions**. A + # failing reaction does not roll back the command. + # - There is one extra claim-cycle of latency before reactions run + # (sub-ms via +InlineNotifier+; up to +catchup_interval+ as + # fallback). + # - +sync+ / +after_sync+ blocks also fire on reaction claims + # (+messages: [evt]+, +events: []+) — inspectors that assume the + # primary message is always a command will see different shapes. + # + # @param partition_values [Hash{Symbol => String}] partition key-value pairs + # @param new_messages [Array] claimed messages to process + # @param history [ReadResult] prior partition history (evolved into state) + # @param replaying [Boolean] when +true+, the reaction branch is skipped + # @return [Array, PositionedMessage)>] action/source pairs def handle_batch(partition_values, new_messages, history:, replaying: false) instance = new(partition_values) instance.evolve(history.messages) @@ -41,20 +88,20 @@ def handle_batch(partition_values, new_messages, history:, replaying: false) each_with_partial_ack(new_messages) do |msg| if handled_commands.include?(msg.class) raw_events = instance.decide(msg) + # TODO: correlation should be handled by infra layer, not reactors. correlated_events = raw_events.map { |e| msg.correlate(e) } - actions = [] - actions.concat( - Actions.build_for(correlated_events, guard: history.guard, correlated: true) + actions = Actions.build_for(correlated_events, guard: history.guard, correlated: true) + actions += instance.collect_actions( + state: instance.state, messages: [msg], events: raw_events ) - correlated_events.each do |evt| - next unless instance.reacts_to?(evt) - reaction_msgs = Array(instance.react(evt)) - actions.concat(Actions.build_for(reaction_msgs, source: evt)) - end - + [actions, msg] + elsif !replaying && instance.reacts_to?(msg) + instance.evolve([msg]) + reaction_msgs = Array(instance.react(msg)) + actions = Actions.build_for(reaction_msgs, source: msg) actions += instance.collect_actions( - state: instance.state, messages: [msg], events: raw_events + state: instance.state, messages: [msg], events: [] ) [actions, msg] @@ -71,7 +118,7 @@ def handle_batch(partition_values, new_messages, history:, replaying: false) # @return [Array, PositionedMessage)>] action/source pairs def handle_claim(claim, history:) values = partition_keys.to_h { |k| [k, claim.partition_value[k.to_s]] } - handle_batch(values, claim.messages, history:) + handle_batch(values, claim.messages, history:, replaying: claim.replaying) end # Copy registered command handlers into subclasses. diff --git a/spec/decider_spec.rb b/spec/decider_spec.rb index 06c1e2ae..543b9ef0 100644 --- a/spec/decider_spec.rb +++ b/spec/decider_spec.rb @@ -179,21 +179,73 @@ class TestDelayedReactionDecider < Sourced::Decider actions, source_msg = pairs.first expect(source_msg).to eq(cmd_positioned) - # Actions: Append(events with guard), Append(reactions), possibly sync + # Reactions are deferred: command claim produces only the event Append (+ after_sync). + # The reaction runs when DeviceBound is re-claimed by this Decider. append_actions = Array(actions).select { |a| a.is_a?(Sourced::Actions::Append) } - expect(append_actions.size).to be >= 1 + expect(append_actions.size).to eq(1) - # First append has the events with guard event_append = append_actions.first expect(event_append.messages.first).to be_a(DeciderTestMessages::DeviceBound) expect(event_append.guard).to eq(guard) - # Second append has the reactions (no guard) - if append_actions.size > 1 - reaction_append = append_actions[1] - expect(reaction_append.messages.first).to be_a(DeciderTestMessages::NotifyBound) - expect(reaction_append.guard).to be_nil - end + # No reaction Append in this pass. + reaction_types = append_actions.flat_map { |a| a.messages.map(&:class) } + expect(reaction_types).not_to include(DeciderTestMessages::NotifyBound) + end + + it 'runs deferred reactions when the triggering event is re-claimed' do + # History: device was already registered and bound (state reflects both) + reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + bound = DeciderTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }) + history_msgs = [Sourced::PositionedMessage.new(reg, 1)] + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 2) + history = Sourced::ReadResult.new(messages: history_msgs, guard: guard) + + bound_positioned = Sourced::PositionedMessage.new(bound, 2) + claim = Sourced::ClaimResult.new( + offset_id: 2, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [bound_positioned], replaying: false, guard: guard + ) + + pairs = TestDeviceDecider.handle_claim(claim, history: history) + expect(pairs.size).to eq(1) + + actions, source_msg = pairs.first + expect(source_msg).to eq(bound_positioned) + + append_actions = Array(actions).select { |a| a.is_a?(Sourced::Actions::Append) } + expect(append_actions.size).to eq(1) + + reaction_append = append_actions.first + expect(reaction_append.messages.first).to be_a(DeciderTestMessages::NotifyBound) + expect(reaction_append.source).to eq(bound_positioned) + expect(reaction_append.guard).to be_nil + + after_sync = Array(actions).find { |a| a.is_a?(Sourced::Actions::AfterSync) } + expect(after_sync).not_to be_nil + end + + it 'skips reactions on replaying claims' do + reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + bound = DeciderTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }) + history_msgs = [Sourced::PositionedMessage.new(reg, 1)] + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 2) + history = Sourced::ReadResult.new(messages: history_msgs, guard: guard) + + bound_positioned = Sourced::PositionedMessage.new(bound, 2) + claim = Sourced::ClaimResult.new( + offset_id: 2, key_pair_ids: [], partition_key: 'device_id:d1', + partition_value: { 'device_id' => 'd1' }, + messages: [bound_positioned], replaying: true, guard: guard + ) + + pairs = TestDeviceDecider.handle_claim(claim, history: history) + expect(pairs.size).to eq(1) + + actions, source_msg = pairs.first + expect(actions).to eq(Sourced::Actions::OK) + expect(source_msg).to eq(bound_positioned) end it 'includes after_sync actions in action pairs' do @@ -239,17 +291,20 @@ class TestDelayedReactionDecider < Sourced::Decider expect(source_msg).to eq(reg_positioned) end - it 'returns schedule actions for delayed reaction dispatches' do + it 'returns schedule actions for delayed reaction dispatches on reaction-event re-claim' do + # Reactions are deferred, so the delayed-dispatch only appears when + # DeviceBound is re-claimed by the Decider, not on the command claim. reg = DeciderTestMessages::DeviceRegistered.new(payload: { device_id: 'd1', name: 'Sensor' }) + bound = DeciderTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }) history_msgs = [Sourced::PositionedMessage.new(reg, 1)] - guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 1) + guard = Sourced::ConsistencyGuard.new(conditions: [], last_position: 2) history = Sourced::ReadResult.new(messages: history_msgs, guard: guard) - cmd = DeciderTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) + bound_positioned = Sourced::PositionedMessage.new(bound, 2) claim = Sourced::ClaimResult.new( - offset_id: 1, key_pair_ids: [], partition_key: 'device_id:d1', + offset_id: 2, key_pair_ids: [], partition_key: 'device_id:d1', partition_value: { 'device_id' => 'd1' }, - messages: [Sourced::PositionedMessage.new(cmd, 2)], + messages: [bound_positioned], replaying: false, guard: guard ) diff --git a/spec/dispatcher_spec.rb b/spec/dispatcher_spec.rb index eb3b493b..76933a0f 100644 --- a/spec/dispatcher_spec.rb +++ b/spec/dispatcher_spec.rb @@ -360,6 +360,12 @@ def task.async; end DispatcherTestMessages::BindDevice.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) + # First claim: process the command, append DeviceBound (reaction deferred). + expect(router.handle_next_for(DispatchTestDecider)).to be true + expect(db[:sourced_scheduled_messages].count).to eq(0) + + # Second claim: the Decider re-claims DeviceBound and runs its reaction, + # which schedules DelayedNotify. expect(router.handle_next_for(DispatchTestDecider)).to be true expect(db[:sourced_scheduled_messages].count).to eq(1) diff --git a/spec/testing/rspec_spec.rb b/spec/testing/rspec_spec.rb index 871520cb..079a7848 100644 --- a/spec/testing/rspec_spec.rb +++ b/spec/testing/rspec_spec.rb @@ -152,12 +152,21 @@ class GWTTestEventSourcedProjector < Sourced::Projector::EventSourced include Sourced::Testing::RSpec describe 'Decider' do - it 'given history + when command → then expected messages (event + reaction)' do + it 'given history + when command → then event (reactions are deferred)' do with_reactor(GWTTestDecider, device_id: 'd1') .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') .then( - GWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), + GWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }) + ) + end + + it 'when reaction-triggering event → then reaction message' do + # Reactions run on a separate claim cycle from the originating command. + with_reactor(GWTTestDecider, device_id: 'd1') + .given(GWTTestMessages::DeviceRegistered, device_id: 'd1', name: 'Sensor') + .when(GWTTestMessages::DeviceBound, device_id: 'd1', asset_id: 'a1') + .then( GWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) ) end @@ -210,8 +219,7 @@ class GWTTestEventSourcedProjector < Sourced::Projector::EventSourced .given(reg) .when(GWTTestMessages::BindDevice, device_id: 'd1', asset_id: 'a1') .then( - GWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }), - GWTTestMessages::NotifyBound.new(payload: { device_id: 'd1' }) + GWTTestMessages::DeviceBound.new(payload: { device_id: 'd1', asset_id: 'a1' }) ) end From 03fd28f418fd13bd1b537ce68025934530c1b997 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 16 Apr 2026 15:32:57 +0200 Subject: [PATCH 113/115] Move correlation into Actions::Append/Schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Decider's command branch no longer pre-correlates events with msg.correlate + correlated: true. Both command and reaction branches now delegate correlation to Actions::Append#execute via source:, matching Projector and DurableWorkflow. Drops the correlated: parameter from Actions.build_for, Append, and Schedule — Decider was the only caller that set it. Correlation now always runs at execute time against source: || source_message. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/sourced/actions.rb | 42 ++++++++++-------------------------------- lib/sourced/decider.rb | 4 +--- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/lib/sourced/actions.rb b/lib/sourced/actions.rb index e93ed749..b62d3a44 100644 --- a/lib/sourced/actions.rb +++ b/lib/sourced/actions.rb @@ -11,9 +11,8 @@ module Actions # @param messages [Sourced::Message, Array] messages produced by a reactor # @param guard [ConsistencyGuard, nil] optional concurrency guard for immediate appends # @param source [Sourced::Message, nil] source message used for correlation when executing - # @param correlated [Boolean] whether +messages+ are already correlated # @return [Array] executable actions in append/schedule groups - def self.build_for(messages, guard: nil, source: nil, correlated: false) + def self.build_for(messages, guard: nil, source: nil) actions = [] messages = Array(messages) return actions if messages.empty? @@ -22,49 +21,37 @@ def self.build_for(messages, guard: nil, source: nil, correlated: false) now = Time.now to_schedule, to_append = messages.partition { |message| message.created_at > now } - actions << Append.new(to_append, guard:, source:, correlated:) if to_append.any? + actions << Append.new(to_append, guard:, source:) if to_append.any? to_schedule.group_by(&:created_at).each do |at, scheduled_messages| - actions << Schedule.new(scheduled_messages, at:, source:, correlated:) + actions << Schedule.new(scheduled_messages, at:, source:) end actions end # Append messages to the store with optional consistency guard. - # Auto-correlates messages with the source message at execution time. + # Auto-correlates messages at execution time. # # When +source:+ is provided, it overrides the runtime's source_message # for correlation (e.g. reactions correlated with the event, not the command). - # - # When +correlated: true+, messages are assumed to be already correlated - # and are appended as-is without re-correlation. class Append attr_reader :messages, :guard, :source # @param messages [Sourced::Message, Array] messages to append # @param guard [ConsistencyGuard, nil] optional optimistic concurrency guard # @param source [Sourced::Message, nil] explicit correlation source - # @param correlated [Boolean] whether +messages+ are already correlated - def initialize(messages, guard: nil, source: nil, correlated: false) + def initialize(messages, guard: nil, source: nil) @messages = Array(messages) @guard = guard @source = source - @correlated = correlated end - # @return [Boolean] whether messages should be appended without re-correlation - def correlated? = @correlated - # @param store [Sourced::Store] # @param source_message [Sourced::Message] default message to correlate from # @return [Array] correlated messages that were appended def execute(store, source_message) - to_append = if correlated? - messages - else - correlate_from = @source || source_message - messages.map { |m| correlate_from.correlate(m) } - end + correlate_from = @source || source_message + to_append = messages.map { |m| correlate_from.correlate(m) } store.append(to_append, guard:) to_append end @@ -77,27 +64,18 @@ class Schedule # @param messages [Sourced::Message, Array] messages to schedule # @param at [Time] when the messages should become available for promotion # @param source [Sourced::Message, nil] explicit correlation source - # @param correlated [Boolean] whether +messages+ are already correlated - def initialize(messages, at:, source: nil, correlated: false) + def initialize(messages, at:, source: nil) @messages = Array(messages) @at = at @source = source - @correlated = correlated end - # @return [Boolean] whether messages should be scheduled without re-correlation - def correlated? = @correlated - # @param store [Sourced::Store] # @param source_message [Sourced::Message] default message to correlate from # @return [Array] correlated messages that were scheduled def execute(store, source_message) - to_schedule = if @correlated - messages - else - correlate_from = @source || source_message - messages.map { |message| correlate_from.correlate(message) } - end + correlate_from = @source || source_message + to_schedule = messages.map { |m| correlate_from.correlate(m) } store.schedule_messages(to_schedule, at: at) to_schedule end diff --git a/lib/sourced/decider.rb b/lib/sourced/decider.rb index 9ec994f0..e92208cd 100644 --- a/lib/sourced/decider.rb +++ b/lib/sourced/decider.rb @@ -88,9 +88,7 @@ def handle_batch(partition_values, new_messages, history:, replaying: false) each_with_partial_ack(new_messages) do |msg| if handled_commands.include?(msg.class) raw_events = instance.decide(msg) - # TODO: correlation should be handled by infra layer, not reactors. - correlated_events = raw_events.map { |e| msg.correlate(e) } - actions = Actions.build_for(correlated_events, guard: history.guard, correlated: true) + actions = Actions.build_for(raw_events, guard: history.guard, source: msg) actions += instance.collect_actions( state: instance.state, messages: [msg], events: raw_events ) From 1ae600e5a3b27650dd9eb6bb6fa48c24e14d5ea8 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Fri, 24 Apr 2026 12:21:51 +0100 Subject: [PATCH 114/115] Sourced.load(upto:) loads history up to specific position number position number is relative to queried partition, not global position --- lib/sourced.rb | 6 ++-- lib/sourced/store.rb | 28 +++++++++++----- spec/load_spec.rb | 77 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/lib/sourced.rb b/lib/sourced.rb index e64cc6be..f6709636 100644 --- a/lib/sourced.rb +++ b/lib/sourced.rb @@ -141,11 +141,13 @@ def self.handle!(reactor_class, command, store: nil) end # Load a reactor instance from its event history using AND-filtered partition reads. - def self.load(reactor_class, store: nil, **values) + # Pass +upto:+ (partition-local rank) to evolve over at most the first N matching + # messages in the partition — useful for time-travel, deterministic replay, or debugging. + def self.load(reactor_class, store: nil, upto: nil, **values) store ||= self.store partition_attrs = reactor_class.partition_keys.to_h { |k| [k, values[k]] } handled_types = reactor_class.handled_messages_for_evolve.map(&:type).uniq - read_result = store.read_partition(partition_attrs, handled_types:) + read_result = store.read_partition(partition_attrs, handled_types:, upto: upto) instance = reactor_class.new(values) instance.evolve(read_result.messages) diff --git a/lib/sourced/store.rb b/lib/sourced/store.rb index d334a191..dbd97e9a 100644 --- a/lib/sourced/store.rb +++ b/lib/sourced/store.rb @@ -376,8 +376,13 @@ def read(conditions, after_position: nil, limit: nil) # @param partition_attrs [Hash{Symbol|String => String}] partition attribute values # @param handled_types [Array] message type strings to include # @param after_position [Integer] fetch messages after this position (exclusive, default 0) + # @param upto [Integer, nil] partition-local cutoff: return at most the first +upto+ + # matching messages (SQL LIMIT). Counts ranks within the partition, not global positions. + # When the partition has more messages than +upto+, the guard's last_position is set to + # the global position of the last returned message so further partition writes are + # detected as conflicts. # @return [ReadResult] messages and a guard for optimistic concurrency - def read_partition(partition_attrs, handled_types:, after_position: 0) + def read_partition(partition_attrs, handled_types:, after_position: 0, upto: nil) # Resolve key_pair_ids for each partition attribute key_pair_ids = partition_attrs.filter_map do |name, value| db[@key_pairs_table].where(name: name.to_s, value: value.to_s).get(:id) @@ -389,7 +394,7 @@ def read_partition(partition_attrs, handled_types:, after_position: 0) return ReadResult.new(messages: [], guard:) end - messages = fetch_partition_messages(key_pair_ids, after_position, handled_types) + messages = fetch_partition_messages(key_pair_ids, after_position, handled_types, upto: upto) # Build guard conditions from handled_types, scoped to partition attrs. # These use OR semantics so the guard detects any concurrent write @@ -400,10 +405,16 @@ def read_partition(partition_attrs, handled_types:, after_position: 0) klass&.to_conditions(**partition_sym) end.flatten - # The guard's last_position must cover the full OR-context, not just - # the AND-filtered messages. Otherwise a message that passes the OR - # conditions but was excluded by AND filtering would look like a conflict. - last_pos = max_position_for(guard_conditions, after_position: after_position) + # When upto truncates the partition (we returned exactly `upto` messages and + # there may be more), anchor the guard at the last returned message's global + # position so later partition writes register as conflicts. + # Otherwise, use the broader OR-context max so a message that passes the OR + # conditions but was excluded by AND filtering doesn't look like a conflict. + last_pos = if upto && messages.size == upto && messages.any? + messages.last.position + else + max_position_for(guard_conditions, after_position: after_position) + end guard = ConsistencyGuard.new(conditions: guard_conditions, last_position: last_pos) ReadResult.new(messages: messages, guard: guard) @@ -1136,7 +1147,7 @@ def find_and_claim_partition(cg_id, handled_types, worker_id) # @param handled_types [Array] message type strings # @param limit [Integer, nil] max messages to return (nil = unlimited) # @return [Array] - def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: nil) + def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: nil, upto: nil) return [] if key_pair_ids.empty? kp_ids_list = key_pair_ids.map { |id| db.literal(id) }.join(', ') @@ -1172,7 +1183,8 @@ def fetch_partition_messages(key_pair_ids, last_position, handled_types, limit: ) ORDER BY m.position ASC SQL - sql += " LIMIT #{db.literal(limit)}" if limit + effective_limit = [limit, upto].compact.min + sql += " LIMIT #{db.literal(effective_limit)}" if effective_limit db.fetch(sql).map { |row| deserialize(row) } end diff --git a/spec/load_spec.rb b/spec/load_spec.rb index 6bd1302c..3f3a8869 100644 --- a/spec/load_spec.rb +++ b/spec/load_spec.rb @@ -191,6 +191,83 @@ class LoadTestProjector < Sourced::Projector::StateStored store.append(new_msg, guard: read_result.guard) }.to raise_error(Sourced::ConcurrentAppendError) end + + describe 'with upto: (partition-local rank)' do + # joe's partition has 3 matching messages (StudentEnrolled, two AssignmentSubmitted). + # jane's StudentEnrolled is excluded by AND filtering. + it 'loads only the first N partition-matching messages' do + instance, _read_result = Sourced.load( + LoadTestDecider, store: store, upto: 1, + course_id: 'algebra', student_id: 'joe' + ) + + expect(instance.state[:enrolled]).to be true + expect(instance.state[:grades]).to eq([]) + end + + it 'counts partition messages, not global positions (upto: 2 → first 2 matches)' do + instance, read_result = Sourced.load( + LoadTestDecider, store: store, upto: 2, + course_id: 'algebra', student_id: 'joe' + ) + + expect(read_result.messages.size).to eq(2) + expect(instance.state[:grades]).to eq(%w[A]) + end + + it 'returns all partition messages when upto equals the partition size' do + instance, read_result = Sourced.load( + LoadTestDecider, store: store, upto: 3, + course_id: 'algebra', student_id: 'joe' + ) + + expect(read_result.messages.size).to eq(3) + expect(instance.state[:grades]).to eq(%w[A B]) + end + + it 'returns all partition messages when upto exceeds the partition size' do + instance, read_result = Sourced.load( + LoadTestDecider, store: store, upto: 9999, + course_id: 'algebra', student_id: 'joe' + ) + + expect(read_result.messages.size).to eq(3) + expect(instance.state[:grades]).to eq(%w[A B]) + end + + it 'returns empty when upto is 0' do + instance, read_result = Sourced.load( + LoadTestDecider, store: store, upto: 0, + course_id: 'algebra', student_id: 'joe' + ) + + expect(instance.state[:enrolled]).to be false + expect(read_result.messages).to be_empty + end + + it 'guard last_position anchors at the last returned message when truncated' do + _instance, read_result = Sourced.load( + LoadTestDecider, store: store, upto: 2, + course_id: 'algebra', student_id: 'joe' + ) + + expect(read_result.guard.last_position).to eq(read_result.messages.last.position) + end + + it 'guard detects unseen partition messages as conflicts when truncated' do + _instance, read_result = Sourced.load( + LoadTestDecider, store: store, upto: 2, + course_id: 'algebra', student_id: 'joe' + ) + + new_msg = LoadTestMessages::AssignmentSubmitted.new( + payload: { course_id: 'algebra', student_id: 'joe', grade: 'C' } + ) + expect { + store.append(new_msg, guard: read_result.guard) + }.to raise_error(Sourced::ConcurrentAppendError) + end + end end describe 'loading a Projector' do From cda8e6a7f9d63ae398f92017fabe6c9b4a3430c0 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Thu, 30 Apr 2026 11:03:01 +0100 Subject: [PATCH 115/115] Dispatcher.spawn_into(task) => Dispatcher.start(task) --- lib/sourced/dispatcher.rb | 10 +++++----- lib/sourced/falcon/service.rb | 2 +- lib/sourced/supervisor.rb | 2 +- spec/dispatcher_spec.rb | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/sourced/dispatcher.rb b/lib/sourced/dispatcher.rb index 651b6618..a8d24c11 100644 --- a/lib/sourced/dispatcher.rb +++ b/lib/sourced/dispatcher.rb @@ -11,12 +11,12 @@ module Sourced # {WorkQueue}, {NotificationQueuer}, {CatchUpPoller}, store notifier, and {Worker}s. # # Does not own the process lifecycle — the caller provides the task/fiber - # context via {#spawn_into}, and triggers shutdown via {#stop}. + # context via {#start}, and triggers shutdown via {#stop}. # # @example Usage with a task runner # dispatcher = Sourced::Dispatcher.new(router: router, worker_count: 4) # executor.start do |task| - # dispatcher.spawn_into(task) + # dispatcher.start(task) # end # dispatcher.stop # @@ -86,7 +86,7 @@ def build_group_id_lookup(reactors) # @return [Array] worker instances managed by this dispatcher attr_reader :workers - def self.spawn_into(task) + def self.start(task) config = Sourced.config dispatcher = Sourced::Dispatcher.new( router: Sourced.router, @@ -97,7 +97,7 @@ def self.spawn_into(task) housekeeping_interval: config.housekeeping_interval, claim_ttl_seconds: config.claim_ttl_seconds, logger: config.logger - ).spawn_into(task) + ).start(task) end # @param router [Sourced::Router] the router providing reactors and store @@ -172,7 +172,7 @@ def initialize( # # @param task [Object] an executor task or Async::Task to spawn fibers into # @return [void] - def spawn_into(task) + def start(task) return if @workers.empty? s = task.respond_to?(:spawn) ? :spawn : :async diff --git a/lib/sourced/falcon/service.rb b/lib/sourced/falcon/service.rb index 0135d0a2..d17e82f6 100644 --- a/lib/sourced/falcon/service.rb +++ b/lib/sourced/falcon/service.rb @@ -20,7 +20,7 @@ def run(instance, evaluator) Async do |task| server.run - Sourced::Dispatcher.spawn_into(task) + Sourced::Dispatcher.start(task) task.children.each(&:wait) end diff --git a/lib/sourced/supervisor.rb b/lib/sourced/supervisor.rb index e4b620e9..d93d3564 100644 --- a/lib/sourced/supervisor.rb +++ b/lib/sourced/supervisor.rb @@ -71,7 +71,7 @@ def start ) @executor.start do |task| - @dispatcher.spawn_into(task) + @dispatcher.start(task) end end diff --git a/spec/dispatcher_spec.rb b/spec/dispatcher_spec.rb index 76933a0f..ad517aea 100644 --- a/spec/dispatcher_spec.rb +++ b/spec/dispatcher_spec.rb @@ -292,14 +292,14 @@ class DispatchTestProjector < Sourced::Projector::StateStored task = double('Task') # 1 notifier + 1 catchup_poller + 1 scheduled_message_poller + 1 stale_claim_reaper + 2 workers = 6 spawns expect(task).to receive(:spawn).exactly(6).times - dispatcher.spawn_into(task) + dispatcher.start(task) end it 'spawns via #async when task does not respond to spawn' do task = Object.new def task.async; end expect(task).to receive(:async).exactly(6).times - dispatcher.spawn_into(task) + dispatcher.start(task) end it '#stop stops all components' do