diff --git a/cmd/serve.go b/cmd/serve.go index b887c41f75..d3e5a6ece6 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -60,6 +60,8 @@ type ServeCommandConfig struct { MaxPageSize uint64 `mapstructure:"max-page-size"` WorkerEnabled bool `mapstructure:"worker"` WorkerAddress string `mapstructure:"worker-grpc-address"` + + ExperimentalTransactionCandidatePagination bool `mapstructure:"experimental-transaction-candidate-pagination"` } const ( @@ -75,6 +77,8 @@ const ( WorkerEnabledFlag = "worker" SemconvMetricsNames = "semconv-metrics-names" SchemaEnforcementMode = "schema-enforcement-mode" + + ExperimentalTransactionCandidatePaginationFlag = "experimental-transaction-candidate-pagination" ) func NewServeCommand() *cobra.Command { @@ -106,6 +110,7 @@ func NewServeCommand() *cobra.Command { storagefx.BunConnectModule(*connectionOptions, service.IsDebug(cmd)), storage.NewFXModule(storage.ModuleConfig{ AutoUpgrade: cfg.AutoUpgrade, + ExperimentalTransactionCandidatePagination: cfg.ExperimentalTransactionCandidatePagination, }), drivers.NewFXModule(), fx.Invoke(alldrivers.Register), @@ -192,6 +197,7 @@ func NewServeCommand() *cobra.Command { cmd.Flags().Uint64(DefaultPageSizeFlag, 15, "Default page size") cmd.Flags().Bool(WorkerEnabledFlag, false, "Enable worker") cmd.Flags().Bool(ExperimentalFeaturesFlag, false, "Enable features configurability") + cmd.Flags().Bool(ExperimentalTransactionCandidatePaginationFlag, false, "Enable experimental candidate-key pagination for selective transaction queries") cmd.Flags().Bool(NumscriptInterpreterFlag, false, "Enable experimental numscript rewrite") cmd.Flags().StringSlice(NumscriptInterpreterFlagsToPass, nil, "Feature flags to pass to the experimental numscript interpreter") cmd.Flags().String(WorkerGRPCAddressFlag, "localhost:8081", "GRPC address") diff --git a/internal/storage/common/resource.go b/internal/storage/common/resource.go index a0e1f00b04..1314b2d374 100644 --- a/internal/storage/common/resource.go +++ b/internal/storage/common/resource.go @@ -97,6 +97,11 @@ type RepositoryHandler[Opts any] interface { Expand(query ResourceQuery[Opts], property string) (*bun.SelectQuery, *JoinCondition, error) } +type CandidateKeyPaginationHandler[Opts any] interface { + UseCandidateKeyPagination(ResourceQuery[Opts], ColumnPaginatedQuery[Opts]) bool + CandidateKeyColumns(ResourceQuery[Opts], ColumnPaginatedQuery[Opts]) []string +} + type ResourceRepository[ResourceType, OptionsType any] struct { resourceHandler RepositoryHandler[OptionsType] } @@ -310,8 +315,10 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType]) Paginate( } var ( - paginator Paginator[ResourceType] - resourceQuery ResourceQuery[OptionsType] + paginator Paginator[ResourceType] + resourceQuery ResourceQuery[OptionsType] + columnPaginationQuery *ColumnPaginatedQuery[OptionsType] + columnPaginationStrategy *columnPaginator[ResourceType, OptionsType] ) switch v := any(paginationQuery).(type) { case OffsetPaginatedQuery[OptionsType]: @@ -322,12 +329,46 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType]) Paginate( if field == nil { return nil, fmt.Errorf("invalid property '%s' for pagination", v.Column) } - paginator = newColumnPaginator[ResourceType, OptionsType](v, fieldName, field.Type) + cp := newColumnPaginator[ResourceType, OptionsType](v, fieldName, field.Type) + paginator = cp + columnPaginationQuery = &v + columnPaginationStrategy = &cp resourceQuery = v.Options default: panic("should not happen") } + if columnPaginationQuery != nil { + if candidateHandler, ok := r.resourceHandler.(CandidateKeyPaginationHandler[OptionsType]); ok && + candidateHandler.UseCandidateKeyPagination(resourceQuery, *columnPaginationQuery) { + finalQuery, err := r.buildCandidateKeyPaginatedDataset( + resourceQuery, + *columnPaginationStrategy, + candidateHandler.CandidateKeyColumns(resourceQuery, *columnPaginationQuery), + ) + if err != nil { + return nil, err + } + + finalQuery, err = r.expand(finalQuery, resourceQuery) + if err != nil { + return nil, fmt.Errorf("expanding results: %w", err) + } + + orderExpr := paginator.OrderExpression() + col, dir, _ := strings.Cut(orderExpr, " ") + finalQuery = finalQuery.Order(fmt.Sprintf("dataset.%s %s", col, dir)) + + ret := make([]ResourceType, 0) + err = finalQuery.Model(&ret).Scan(ctx) + if err != nil { + return nil, fmt.Errorf("scanning results: %w", err) + } + + return paginator.BuildCursor(ret) + } + } + finalQuery, err := r.buildFilteredDataset(resourceQuery) if err != nil { return nil, fmt.Errorf("building filtered dataset: %w", err) @@ -358,6 +399,56 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType]) Paginate( return paginator.BuildCursor(ret) } +func (r *PaginatedResourceRepository[ResourceType, OptionsType]) buildCandidateKeyPaginatedDataset( + resourceQuery ResourceQuery[OptionsType], + paginator columnPaginator[ResourceType, OptionsType], + keyColumns []string, +) (*bun.SelectQuery, error) { + if len(keyColumns) == 0 { + return nil, fmt.Errorf("candidate key pagination requires at least one key column") + } + + filteredDataset, err := r.buildFilteredDataset(resourceQuery) + if err != nil { + return nil, fmt.Errorf("building filtered candidate dataset: %w", err) + } + + candidateKeys := filteredDataset.NewSelect(). + ModelTableExpr("(?) candidate_source", filteredDataset) + for _, column := range keyColumns { + candidateKeys = candidateKeys.ColumnExpr("candidate_source." + column) + } + + page := filteredDataset.NewSelect(). + ModelTableExpr("candidate_keys") + for _, column := range keyColumns { + page = page.Column(column) + } + page, err = paginator.Paginate(page) + if err != nil { + return nil, fmt.Errorf("paginating candidate keys: %w", err) + } + + unfilteredResourceQuery := resourceQuery + unfilteredResourceQuery.Builder = nil + unfilteredDataset, err := r.buildFilteredDataset(unfilteredResourceQuery) + if err != nil { + return nil, fmt.Errorf("building unfiltered page dataset: %w", err) + } + + joinConditions := make([]string, 0, len(keyColumns)) + for _, column := range keyColumns { + joinConditions = append(joinConditions, fmt.Sprintf("page.%s = dataset.%s", column, column)) + } + + return unfilteredDataset.NewSelect(). + WithQuery(bun.NewWithQuery("candidate_keys", candidateKeys).Materialized()). + With("page", page). + ModelTableExpr("(?) dataset", unfilteredDataset). + ColumnExpr("dataset.*"). + Join("join page on " + strings.Join(joinConditions, " and ")), nil +} + func NewPaginatedResourceRepository[ResourceType, OptionsType any]( handler RepositoryHandler[OptionsType], defaultPaginationColumn string, diff --git a/internal/storage/driver/module.go b/internal/storage/driver/module.go index 744425e50e..3dc549fafc 100644 --- a/internal/storage/driver/module.go +++ b/internal/storage/driver/module.go @@ -16,7 +16,11 @@ import ( systemstore "github.com/formancehq/ledger/internal/storage/system" ) -func NewFXModule() fx.Option { +type ModuleConfig struct { + ExperimentalTransactionCandidatePagination bool +} + +func NewFXModule(config ModuleConfig) fx.Option { return fx.Options( fx.Provide(fx.Annotate(func(tracerProvider trace.TracerProvider) bucket.Factory { return bucket.NewDefaultFactory(bucket.WithTracer(tracerProvider.Tracer("store"))) @@ -44,6 +48,9 @@ func NewFXModule() fx.Option { if params.MeterProvider != nil { options = append(options, ledgerstore.WithMeter(params.MeterProvider.Meter("store"))) } + if config.ExperimentalTransactionCandidatePagination { + options = append(options, ledgerstore.WithExperimentalTransactionCandidatePagination(true)) + } return ledgerstore.NewFactory(params.DB, options...) }), fx.Provide(func( diff --git a/internal/storage/ledger/resource_transactions.go b/internal/storage/ledger/resource_transactions.go index f1ee9a2761..6477c74206 100644 --- a/internal/storage/ledger/resource_transactions.go +++ b/internal/storage/ledger/resource_transactions.go @@ -6,6 +6,8 @@ import ( "github.com/uptrace/bun" + "github.com/formancehq/go-libs/v5/pkg/storage/bun/paginate" + "github.com/formancehq/ledger/internal/queries" "github.com/formancehq/ledger/internal/storage/common" "github.com/formancehq/ledger/pkg/features" @@ -186,4 +188,31 @@ func (h transactionsResourceHandler) Expand(_ common.ResourceQuery[any], propert }, nil } +func (h transactionsResourceHandler) UseCandidateKeyPagination( + q common.ResourceQuery[any], + paginationQuery common.ColumnPaginatedQuery[any], +) bool { + if !h.store.experimentalTransactionCandidatePagination { + return false + } + if paginationQuery.Column != "id" { + return false + } + if paginationQuery.Order == nil || *paginationQuery.Order != paginate.OrderDesc { + return false + } + if paginationQuery.Reverse { + return false + } + return hasPositiveSelectiveTransactionFilter(q.Builder) +} + +func (h transactionsResourceHandler) CandidateKeyColumns( + common.ResourceQuery[any], + common.ColumnPaginatedQuery[any], +) []string { + return []string{"ledger", "id"} +} + var _ common.RepositoryHandler[any] = transactionsResourceHandler{} +var _ common.CandidateKeyPaginationHandler[any] = transactionsResourceHandler{} diff --git a/internal/storage/ledger/store.go b/internal/storage/ledger/store.go index e10915268f..b58f78f92f 100644 --- a/internal/storage/ledger/store.go +++ b/internal/storage/ledger/store.go @@ -34,6 +34,8 @@ type Store struct { // (e.g. when a new ledger is created) immediately affects all stores. aloneInBucket *atomic.Bool + experimentalTransactionCandidatePagination bool + tracer trace.Tracer meter metric.Meter checkBucketSchemaHistogram metric.Int64Histogram @@ -346,6 +348,12 @@ func WithTracer(tracer trace.Tracer) Option { } } +func WithExperimentalTransactionCandidatePagination(enabled bool) Option { + return func(s *Store) { + s.experimentalTransactionCandidatePagination = enabled + } +} + var defaultOptions = []Option{ WithMeter(noopmetrics.Meter{}), WithTracer(nooptracer.Tracer{}), diff --git a/internal/storage/ledger/transactions_test.go b/internal/storage/ledger/transactions_test.go index 51478d8ec1..9612847ff0 100644 --- a/internal/storage/ledger/transactions_test.go +++ b/internal/storage/ledger/transactions_test.go @@ -9,14 +9,18 @@ import ( "fmt" "math/big" "slices" + "strings" + "sync" "testing" libtime "time" "github.com/alitto/pond" "github.com/stretchr/testify/require" + "github.com/uptrace/bun" logging "github.com/formancehq/go-libs/v5/pkg/observe/log" "github.com/formancehq/go-libs/v5/pkg/query" + "github.com/formancehq/go-libs/v5/pkg/storage/bun/paginate" "github.com/formancehq/go-libs/v5/pkg/storage/postgres" "github.com/formancehq/go-libs/v5/pkg/types/metadata" "github.com/formancehq/go-libs/v5/pkg/types/pointer" @@ -947,3 +951,192 @@ func TestTransactionsList(t *testing.T) { }) } } + +func TestTransactionsCandidateKeyPaginationMatchesDefault(t *testing.T) { + t.Parallel() + + store := newLedgerStore(t) + candidateStore := ledgerstore.New( + defaultBunDB.GetValue(), + store.GetBucket(), + store.GetLedger(), + ledgerstore.WithExperimentalTransactionCandidatePagination(true), + ) + + ctx := logging.TestingContext() + now := time.Now() + + for i := 0; i < 12; i++ { + source := fmt.Sprintf("users:%d", i) + destination := "treasury" + txMetadata := metadata.Metadata{ + "transaction_currency": "EUR", + } + + switch i { + case 1, 4, 7, 10: + source = "wallet:target" + txMetadata["transaction_currency"] = "USD" + } + if i == 7 { + txMetadata["wallet_transaction_method"] = "hold_settlement" + } + if i == 10 { + delete(txMetadata, "transaction_currency") + txMetadata["destination_currency"] = "USD" + } + + tx := ledger.NewTransaction(). + WithPostings( + ledger.NewPosting(source, destination, "USD", big.NewInt(int64(i+1))), + ). + WithMetadata(txMetadata). + WithReference(fmt.Sprintf("tx-%02d", i)). + WithTimestamp(now.Add(time.Duration(i) * time.Minute)) + require.NoError(t, commitTransactionAndUpsertAccounts(ctx, store, &tx)) + } + + builder := query.And( + query.Match("account", "wallet:target"), + query.Or( + query.Match("metadata[transaction_currency]", "USD"), + query.Match("metadata[destination_currency]", "USD"), + ), + query.Not(query.Match("metadata[wallet_transaction_method]", "hold_settlement")), + ) + initialQuery := common.InitialPaginatedQuery[any]{ + PageSize: 2, + Options: common.ResourceQuery[any]{ + Builder: builder, + Expand: []string{"volumes", "effectiveVolumes"}, + }, + } + + defaultPage, err := store.Transactions().Paginate(ctx, initialQuery) + require.NoError(t, err) + candidatePage, err := candidateStore.Transactions().Paginate(ctx, initialQuery) + require.NoError(t, err) + requireTransactionsCursorEqual(t, defaultPage, candidatePage) + require.True(t, defaultPage.HasMore) + + defaultNextQuery, err := common.UnmarshalCursor[any](defaultPage.Next) + require.NoError(t, err) + candidateNextQuery, err := common.UnmarshalCursor[any](candidatePage.Next) + require.NoError(t, err) + + defaultNextPage, err := store.Transactions().Paginate(ctx, defaultNextQuery) + require.NoError(t, err) + candidateNextPage, err := candidateStore.Transactions().Paginate(ctx, candidateNextQuery) + require.NoError(t, err) + requireTransactionsCursorEqual(t, defaultNextPage, candidateNextPage) + require.False(t, defaultNextPage.HasMore) + require.NotEmpty(t, defaultNextPage.Previous) + + defaultPreviousQuery, err := common.UnmarshalCursor[any](defaultNextPage.Previous) + require.NoError(t, err) + candidatePreviousQuery, err := common.UnmarshalCursor[any](candidateNextPage.Previous) + require.NoError(t, err) + + defaultPreviousPage, err := store.Transactions().Paginate(ctx, defaultPreviousQuery) + require.NoError(t, err) + candidatePreviousPage, err := candidateStore.Transactions().Paginate(ctx, candidatePreviousQuery) + require.NoError(t, err) + requireTransactionsCursorEqual(t, defaultPreviousPage, candidatePreviousPage) + requireTransactionsCursorEqual(t, defaultPage, defaultPreviousPage) +} + +func TestTransactionsCandidateKeyPaginationSQLShape(t *testing.T) { + <-defaultBunDB.Done() + + hook := &captureQueriesHook{} + defaultBunDB.GetValue().AddQueryHook(hook) + + store := newLedgerStore(t) + candidateStore := ledgerstore.New( + defaultBunDB.GetValue(), + store.GetBucket(), + store.GetLedger(), + ledgerstore.WithExperimentalTransactionCandidatePagination(true), + ) + + ctx := context.WithValue(logging.TestingContext(), captureQueriesContextKey{}, true) + tx := ledger.NewTransaction(). + WithPostings(ledger.NewPosting("wallet:target", "treasury", "USD", big.NewInt(1))). + WithMetadata(metadata.Metadata{"transaction_currency": "USD"}). + WithTimestamp(time.Now()) + require.NoError(t, commitTransactionAndUpsertAccounts(ctx, store, &tx)) + + paginatedQuery := common.InitialPaginatedQuery[any]{ + PageSize: 2, + Options: common.ResourceQuery[any]{ + Builder: query.And( + query.Match("account", "wallet:target"), + query.Match("metadata[transaction_currency]", "USD"), + ), + }, + } + + hook.Reset() + _, err := store.Transactions().Paginate(ctx, paginatedQuery) + require.NoError(t, err) + require.False(t, hook.Contains("candidate_keys")) + + hook.Reset() + _, err = candidateStore.Transactions().Paginate(ctx, paginatedQuery) + require.NoError(t, err) + require.True(t, hook.Contains("candidate_keys")) + require.True(t, hook.Contains("AS MATERIALIZED")) +} + +func requireTransactionsCursorEqual( + t *testing.T, + expected *paginate.Cursor[ledger.Transaction], + actual *paginate.Cursor[ledger.Transaction], +) { + t.Helper() + + require.Equal(t, expected.PageSize, actual.PageSize) + require.Equal(t, expected.HasMore, actual.HasMore) + require.Equal(t, expected.Previous, actual.Previous) + require.Equal(t, expected.Next, actual.Next) + require.Len(t, actual.Data, len(expected.Data)) + RequireEqual(t, expected.Data, actual.Data) +} + +type captureQueriesContextKey struct{} + +type captureQueriesHook struct { + mu sync.Mutex + queries []string +} + +func (h *captureQueriesHook) BeforeQuery(ctx context.Context, _ *bun.QueryEvent) context.Context { + return ctx +} + +func (h *captureQueriesHook) AfterQuery(ctx context.Context, event *bun.QueryEvent) { + if ctx.Value(captureQueriesContextKey{}) == nil { + return + } + + h.mu.Lock() + defer h.mu.Unlock() + h.queries = append(h.queries, event.Query) +} + +func (h *captureQueriesHook) Reset() { + h.mu.Lock() + defer h.mu.Unlock() + h.queries = nil +} + +func (h *captureQueriesHook) Contains(value string) bool { + h.mu.Lock() + defer h.mu.Unlock() + for _, query := range h.queries { + if strings.Contains(query, value) { + return true + } + } + return false +} diff --git a/internal/storage/ledger/utils.go b/internal/storage/ledger/utils.go index bc93ab2b47..a04623f411 100644 --- a/internal/storage/ledger/utils.go +++ b/internal/storage/ledger/utils.go @@ -161,6 +161,65 @@ func nodeContainsAddressFilter(node map[string]any) bool { return false } +func hasPositiveSelectiveTransactionFilter(builder query.Builder) bool { + if builder == nil { + return false + } + data, err := json.Marshal(builder) + if err != nil { + return false + } + var node map[string]any + if err := json.Unmarshal(data, &node); err != nil { + return false + } + return nodeHasPositiveSelectiveTransactionFilter(node, false) +} + +func nodeHasPositiveSelectiveTransactionFilter(node map[string]any, insideNot bool) bool { + for op, value := range node { + switch { + case op == "$match": + if insideNot { + continue + } + if m, ok := value.(map[string]any); ok { + for key := range m { + if isSelectiveTransactionFilterKey(key) { + return true + } + } + } + + case op == "$not": + if child, ok := value.(map[string]any); ok { + if nodeHasPositiveSelectiveTransactionFilter(child, true) { + return true + } + } + + case op == "$and" || op == "$or": + if items, ok := value.([]any); ok { + for _, item := range items { + if child, ok := item.(map[string]any); ok { + if nodeHasPositiveSelectiveTransactionFilter(child, insideNot) { + return true + } + } + } + } + } + } + return false +} + +func isSelectiveTransactionFilterKey(key string) bool { + return key == "account" || + key == "source" || + key == "destination" || + strings.HasPrefix(key, "metadata[") +} + // isNodeSafeForLateral walks the JSON AST and checks whether pushing the // address filter into the LATERAL join would produce correct results. // insideNot tracks whether we are inside a $not ancestor. diff --git a/internal/storage/ledger/utils_test.go b/internal/storage/ledger/utils_test.go index e5c333fe94..201c819b93 100644 --- a/internal/storage/ledger/utils_test.go +++ b/internal/storage/ledger/utils_test.go @@ -108,6 +108,71 @@ func TestCanPushAddressFilterToLateral(t *testing.T) { } } +func TestHasPositiveSelectiveTransactionFilter(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + builder query.Builder + expected bool + }{ + { + name: "nil builder", + builder: nil, + expected: false, + }, + { + name: "positive account match", + builder: query.Match("account", "wallet:1"), + expected: true, + }, + { + name: "positive metadata match", + builder: query.Match("metadata[transaction_currency]", "USD"), + expected: true, + }, + { + name: "negative metadata only", + builder: query.Not(query.Match("metadata[wallet_transaction_method]", "hold_settlement")), + expected: false, + }, + { + name: "positive and negative metadata", + builder: query.And( + query.Match("metadata[transaction_currency]", "USD"), + query.Not(query.Match("metadata[wallet_transaction_method]", "hold_settlement")), + ), + expected: true, + }, + { + name: "positive account in or", + builder: query.Or( + query.Match("account", "wallet:1"), + query.Match("reference", "tx1"), + ), + expected: true, + }, + { + name: "reference only", + builder: query.Match("reference", "tx1"), + expected: false, + }, + { + name: "metadata exists only", + builder: query.Exists("metadata", "category"), + expected: false, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, test.expected, hasPositiveSelectiveTransactionFilter(test.builder)) + }) + } +} + func TestBuildAddressFilterForLateral(t *testing.T) { t.Parallel() diff --git a/internal/storage/module.go b/internal/storage/module.go index c7d5608995..d0a3e2f116 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -19,7 +19,8 @@ import ( const HealthCheckName = `storage-driver-up-to-date` type ModuleConfig struct { - AutoUpgrade bool + AutoUpgrade bool + ExperimentalTransactionCandidatePagination bool } func NewFXModule(config ModuleConfig) fx.Option { @@ -28,7 +29,9 @@ func NewFXModule(config ModuleConfig) fx.Option { fx.Provide(func(store *systemstore.DefaultStore) driver.SystemStore { return store }), - driver.NewFXModule(), + driver.NewFXModule(driver.ModuleConfig{ + ExperimentalTransactionCandidatePagination: config.ExperimentalTransactionCandidatePagination, + }), servicefx.ProvideHealthCheck(func(driver *driver.Driver, tracer trace.TracerProvider) health.NamedCheck { hasReachedMinimalVersion := false return health.NewNamedCheck(HealthCheckName, health.CheckFn(func(ctx context.Context) error {