From bd495fd99435b3cddc1c5803344581ee5b586399 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Wed, 2 Jul 2025 15:20:57 +0200 Subject: [PATCH 1/8] get objects by ids --- publish/publishrepo/publishrepo.go | 33 +++++++++++++++++++++++++ publish/publishrepo/publishrepo_test.go | 27 ++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/publish/publishrepo/publishrepo.go b/publish/publishrepo/publishrepo.go index dadaa53..acea2da 100644 --- a/publish/publishrepo/publishrepo.go +++ b/publish/publishrepo/publishrepo.go @@ -36,6 +36,7 @@ type PublishRepo interface { DeletePublish(ctx context.Context, id primitive.ObjectID) (err error) DeleteOutdatedPublishes(ctx context.Context, before time.Time) (deletedCount int, err error) DeleteOutdatedObjects(ctx context.Context, before time.Time) (deletedCount int, err error) + GetPublishesByObjectIds(ctx context.Context, objectIds []string) (publishes []domain.ObjectWithPublish, err error) app.ComponentRunnable } @@ -377,6 +378,38 @@ func (p *publishRepo) DeleteOutdatedObjects(ctx context.Context, before time.Tim return int(res.DeletedCount), nil } +func (p *publishRepo) GetPublishesByObjectIds(ctx context.Context, objectIds []string) ([]domain.ObjectWithPublish, error) { + filter := bson.D{ + {"objectId", bson.D{ + {"$in", objectIds}, + }}, + } + + cur, err := p.objectsColl.Find(ctx, filter) + if err != nil { + return nil, err + } + defer func() { + _ = cur.Close(ctx) + }() + var publishes []domain.ObjectWithPublish + for cur.Next(ctx) { + var publish domain.ObjectWithPublish + if err = cur.Decode(&publish.Object); err != nil { + return nil, err + } + if publish.ActivePublishId != nil { + _ = p.publishColl.FindOne(ctx, bson.D{{"_id", *publish.ActivePublishId}}).Decode(&publish.Publish) + } + if publish.Publish.Status == domain.PublishStatusPublished { + publishes = append(publishes, publish) + } + + } + return publishes, nil + +} + func (p *publishRepo) Close(ctx context.Context) (err error) { return } diff --git a/publish/publishrepo/publishrepo_test.go b/publish/publishrepo/publishrepo_test.go index 521bb96..0e65241 100644 --- a/publish/publishrepo/publishrepo_test.go +++ b/publish/publishrepo/publishrepo_test.go @@ -111,6 +111,33 @@ func TestPublishRepo_ObjectPublishStatus(t *testing.T) { }) } +func TestPublishRepo_GetPublishes(t *testing.T) { + t.Run("publishes by objectid", func(t *testing.T) { + fx := newFixture(t) + obj := newTestObj() + publishObj, _, err := fx.ObjectCreate(ctx, obj, "v1") + require.NoError(t, err) + uploadKey := publishObj.Publish.UploadKey + publish, err := fx.GetPublish(ctx, publishObj.Publish.Id) + require.NoError(t, err) + assert.Equal(t, publish.Publish.UploadKey, uploadKey) + publish.Publish.Size = 123 + publish.Publish.Status = domain.PublishStatusPublished + require.NoError(t, fx.FinalizePublish(ctx, publish)) + publishObj, err = fx.ObjectPublishStatus(ctx, obj) + require.NoError(t, err) + require.NotNil(t, publishObj.Publish) + assert.Equal(t, domain.PublishStatusPublished, publishObj.Publish.Status) + assert.Equal(t, int64(123), publishObj.Publish.Size) + + publishes, err := fx.GetPublishesByObjectIds(ctx, []string{obj.ObjectId}) + require.NoError(t, err) + require.Len(t, publishes, 1) + assert.Equal(t, "o1", publishes[0].ObjectId) + assert.Equal(t, domain.PublishStatusPublished, publishes[0].Publish.Status) + }) +} + func TestPublishRepo_IterateReadyToDeleteIds(t *testing.T) { fx := newFixture(t) docs := []any{ From 3a40a59e398c2588d3f5ecf9ff5b2ad0985ba205 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Wed, 2 Jul 2025 16:14:11 +0200 Subject: [PATCH 2/8] renderer, SetUrlRewriteMap --- gateway/gateway.go | 29 +++++++++++++++++++++++++++++ go.mod | 5 ++++- go.sum | 12 ++++++++++++ publish/service.go | 5 +++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/gateway/gateway.go b/gateway/gateway.go index f52bc44..2fac29c 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -280,6 +280,12 @@ func (g *gateway) renderPage(ctx context.Context, id cacheId) (*pageObject, erro return nil, err } + linkObjectIds := rend.GetLinkObjectIds() + objectIdToUrl, err := g.getObjectIdToUrl(ctx, linkObjectIds) + if err == nil { + rend.SetUrlRewriteMap(objectIdToUrl) + } + var buf = bytes.NewBuffer(make([]byte, 0, 5*1024)) if err = rend.Render(buf); err != nil { return nil, err @@ -290,6 +296,29 @@ func (g *gateway) renderPage(ctx context.Context, id cacheId) (*pageObject, erro }, nil } +func (g *gateway) getObjectIdToUrl(ctx context.Context, linkObjectIds []string) (map[string]string, error) { + var objectIdToUrl map[string]string + if len(linkObjectIds) > 0 { + publishes, err := g.publish.GetPublishesByObjectIds(ctx, linkObjectIds) + if err != nil { + return nil, err + } + objectIdToUrl = make(map[string]string, len(publishes)) + for _, publish := range publishes { + var url string + anyname, err := g.nameService.ResolveIdentity(ctx, publish.Identity) + // TODO: move to config + if err != nil { + url = fmt.Sprintf("https://any.coop/%s/%s", publish.Identity, publish.Uri) + } else { + url = fmt.Sprintf("https://%s.any.org/%s", anyname, publish.Uri) + } + objectIdToUrl[publish.ObjectId] = url + } + } + return objectIdToUrl, nil + +} func (g *gateway) invalidateCache(identity, uri string) { withName := "{" + string(newCacheId(identity, uri, true)) + "}" withoutName := "{" + string(newCacheId(identity, uri, false)) + "}" diff --git a/go.mod b/go.mod index 5dc2518..2a1b00b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.2 require ( github.com/ahmetb/govvv v0.3.0 github.com/anyproto/any-sync v0.8.5 - github.com/anyproto/anytype-publish-renderer v0.3.17 + github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250702140802-bfadc2e55c97 github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a github.com/aws/aws-sdk-go-v2 v1.36.5 github.com/aws/aws-sdk-go-v2/config v1.29.14 @@ -61,6 +61,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect github.com/hashicorp/yamux v0.1.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect @@ -91,6 +92,8 @@ require ( github.com/quic-go/quic-go v0.52.0 // indirect github.com/samber/lo v1.49.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 66b925a..7ca80fb 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,10 @@ github.com/anyproto/anytype-heart v0.40.22 h1:2a3YH3kmqGMgRQhoSoFqoqgmtnfUYdiqgD github.com/anyproto/anytype-heart v0.40.22/go.mod h1:KU4OeCQF5yGG13dygVdHXNwgygrOP6Xz4fzdBFpXe5I= github.com/anyproto/anytype-publish-renderer v0.3.17 h1:1U7+LtN4QEWyR65QckWD6fHVPq8i2eOL2rgMUY2K2U8= github.com/anyproto/anytype-publish-renderer v0.3.17/go.mod h1:PKed72gb5gUXc+VWcF4hxl3FxolqzWSHMYgYIT3d9ZI= +github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250701120025-f1444d976e5d h1:IrkfPpdLICAz4/nzEvld6GSN93NBFgjmM+VljHts7yQ= +github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250701120025-f1444d976e5d/go.mod h1:PKed72gb5gUXc+VWcF4hxl3FxolqzWSHMYgYIT3d9ZI= +github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250702140802-bfadc2e55c97 h1:B8DTKkWD3+eZL/oSkU8MO2SrufmbacUYL1S+lIVQkYM= +github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250702140802-bfadc2e55c97/go.mod h1:PKed72gb5gUXc+VWcF4hxl3FxolqzWSHMYgYIT3d9ZI= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY= github.com/anyproto/go-gelf v0.0.0-20210418191311-774bd5b016e7 h1:SyEu5uxZ5nKHEJ6TPKQqjM+T00SYi0MW1VaLzqZtZ9E= @@ -101,6 +105,7 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/mb/v3 v3.0.2 h1:jd1Xx0zzihZlXL6HmnRXVCI1BHuXz/kY+VzX9WbvNDU= github.com/cheggaaa/mb/v3 v3.0.2/go.mod h1:zCt2QeYukhd/g0bIdNqF+b/kKz1hnLFNDkP49qN5kqI= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -151,6 +156,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= @@ -243,6 +250,7 @@ github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= @@ -250,6 +258,10 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/publish/service.go b/publish/service.go index cbfac7f..d79461e 100644 --- a/publish/service.go +++ b/publish/service.go @@ -50,6 +50,7 @@ func New() Service { type Service interface { ResolveUriWithIdentity(ctx context.Context, name, uri string) (publish domain.Object, err error) SetInvalidateCacheCallback(f func(identity, uri string)) + GetPublishesByObjectIds(ctx context.Context, objectIds []string) ([]domain.ObjectWithPublish, error) app.ComponentRunnable } @@ -164,6 +165,10 @@ func (p *publishService) ListPublishes(ctx context.Context, spaceId string) (lis return p.repo.ListPublishes(ctx, identity, spaceId) } +func (p *publishService) GetPublishesByObjectIds(ctx context.Context, objectIds []string) ([]domain.ObjectWithPublish, error) { + return p.repo.GetPublishesByObjectIds(ctx, objectIds) +} + func (p *publishService) UploadTar(ctx context.Context, publishId, uploadKey string, reader io.Reader) (resultUrl string, err error) { id, err := primitive.ObjectIDFromHex(publishId) if err != nil { From e1dd6d231b173956c28b46d0cb340fb888170412 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Wed, 2 Jul 2025 17:09:59 +0200 Subject: [PATCH 3/8] fix anyname url --- gateway/gateway.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gateway/gateway.go b/gateway/gateway.go index 2fac29c..66450b5 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -280,6 +280,8 @@ func (g *gateway) renderPage(ctx context.Context, id cacheId) (*pageObject, erro return nil, err } + // TODO: url maps are not updated in cache, + // invalidate cache somehow linkObjectIds := rend.GetLinkObjectIds() objectIdToUrl, err := g.getObjectIdToUrl(ctx, linkObjectIds) if err == nil { @@ -311,7 +313,7 @@ func (g *gateway) getObjectIdToUrl(ctx context.Context, linkObjectIds []string) if err != nil { url = fmt.Sprintf("https://any.coop/%s/%s", publish.Identity, publish.Uri) } else { - url = fmt.Sprintf("https://%s.any.org/%s", anyname, publish.Uri) + url = fmt.Sprintf("https://%s.org/%s", anyname, publish.Uri) } objectIdToUrl[publish.ObjectId] = url } From c781a6da12e38819302d4c48b5b75f0b5cfec6b1 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Wed, 9 Jul 2025 17:13:48 +0200 Subject: [PATCH 4/8] small tmp adjustments for testing --- gateway/gateway.go | 2 +- gateway/gatewayconfig/config.go | 1 + go.mod | 9 ++++----- go.sum | 14 -------------- redisprovider/redisprovider.go | 4 +++- 5 files changed, 9 insertions(+), 21 deletions(-) diff --git a/gateway/gateway.go b/gateway/gateway.go index 66450b5..cb80776 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -311,7 +311,7 @@ func (g *gateway) getObjectIdToUrl(ctx context.Context, linkObjectIds []string) anyname, err := g.nameService.ResolveIdentity(ctx, publish.Identity) // TODO: move to config if err != nil { - url = fmt.Sprintf("https://any.coop/%s/%s", publish.Identity, publish.Uri) + url = fmt.Sprintf("%s/%s/%s",g.config.SelfURL, publish.Identity, publish.Uri) } else { url = fmt.Sprintf("https://%s.org/%s", anyname, publish.Uri) } diff --git a/gateway/gatewayconfig/config.go b/gateway/gatewayconfig/config.go index 12a248e..a935e86 100644 --- a/gateway/gatewayconfig/config.go +++ b/gateway/gatewayconfig/config.go @@ -8,6 +8,7 @@ type Config struct { Addr string `yaml:"addr"` Domain string `yaml:"domain"` StaticFilesURL string `yaml:"staticFilesUrl"` + SelfURL string `yaml:"selfUrl"` PublishFilesURL string `yaml:"publishFilesUrl"` ServeStatic bool `yaml:"serveStatic"` ServePublish bool `yaml:"servePublish"` diff --git a/go.mod b/go.mod index 2a1b00b..21b592e 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,13 @@ go 1.23.2 require ( github.com/ahmetb/govvv v0.3.0 github.com/anyproto/any-sync v0.8.5 - github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250702140802-bfadc2e55c97 + github.com/anyproto/anytype-publish-renderer v0.3.18 github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a github.com/aws/aws-sdk-go-v2 v1.36.5 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 + github.com/aws/smithy-go v1.22.4 github.com/golang/snappy v1.0.0 github.com/google/uuid v1.6.0 github.com/redis/go-redis/v9 v9.7.0 @@ -42,7 +43,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect - github.com/aws/smithy-go v1.22.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd v0.24.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect @@ -61,7 +61,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect github.com/hashicorp/yamux v0.1.2 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect @@ -92,8 +91,6 @@ require ( github.com/quic-go/quic-go v0.52.0 // indirect github.com/samber/lo v1.49.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect @@ -125,3 +122,5 @@ replace github.com/anyproto/anytype-publish-server/publishclient => ./publishcli replace gopkg.in/Graylog2/go-gelf.v2 => github.com/anyproto/go-gelf v0.0.0-20210418191311-774bd5b016e7 replace github.com/btcsuite/btcutil => github.com/btcsuite/btcd/btcutil v1.1.5 + +replace github.com/anyproto/anytype-publish-renderer => ../anytype-publish-renderer diff --git a/go.sum b/go.sum index 7ca80fb..0f6f9e4 100644 --- a/go.sum +++ b/go.sum @@ -13,12 +13,6 @@ github.com/anyproto/any-sync v0.8.5 h1:9vaFUI4mgO8pec72EXl5/3A/ZhARnH47nOAdlF56z github.com/anyproto/any-sync v0.8.5/go.mod h1:Ka5tDxpTrkepPL4/KPSXiy1GOZtIbxhEOoi26FPG6C4= github.com/anyproto/anytype-heart v0.40.22 h1:2a3YH3kmqGMgRQhoSoFqoqgmtnfUYdiqgDVoghjh8BE= github.com/anyproto/anytype-heart v0.40.22/go.mod h1:KU4OeCQF5yGG13dygVdHXNwgygrOP6Xz4fzdBFpXe5I= -github.com/anyproto/anytype-publish-renderer v0.3.17 h1:1U7+LtN4QEWyR65QckWD6fHVPq8i2eOL2rgMUY2K2U8= -github.com/anyproto/anytype-publish-renderer v0.3.17/go.mod h1:PKed72gb5gUXc+VWcF4hxl3FxolqzWSHMYgYIT3d9ZI= -github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250701120025-f1444d976e5d h1:IrkfPpdLICAz4/nzEvld6GSN93NBFgjmM+VljHts7yQ= -github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250701120025-f1444d976e5d/go.mod h1:PKed72gb5gUXc+VWcF4hxl3FxolqzWSHMYgYIT3d9ZI= -github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250702140802-bfadc2e55c97 h1:B8DTKkWD3+eZL/oSkU8MO2SrufmbacUYL1S+lIVQkYM= -github.com/anyproto/anytype-publish-renderer v0.3.18-0.20250702140802-bfadc2e55c97/go.mod h1:PKed72gb5gUXc+VWcF4hxl3FxolqzWSHMYgYIT3d9ZI= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY= github.com/anyproto/go-gelf v0.0.0-20210418191311-774bd5b016e7 h1:SyEu5uxZ5nKHEJ6TPKQqjM+T00SYi0MW1VaLzqZtZ9E= @@ -105,7 +99,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/mb/v3 v3.0.2 h1:jd1Xx0zzihZlXL6HmnRXVCI1BHuXz/kY+VzX9WbvNDU= github.com/cheggaaa/mb/v3 v3.0.2/go.mod h1:zCt2QeYukhd/g0bIdNqF+b/kKz1hnLFNDkP49qN5kqI= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -156,8 +149,6 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= @@ -250,7 +241,6 @@ github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= @@ -258,10 +248,6 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/redisprovider/redisprovider.go b/redisprovider/redisprovider.go index 3e94b8c..f851229 100644 --- a/redisprovider/redisprovider.go +++ b/redisprovider/redisprovider.go @@ -49,7 +49,9 @@ func (r *redisProvider) Name() (name string) { } func (r *redisProvider) Run(ctx context.Context) (err error) { - return r.redis.Ping(ctx).Err() + // return r.redis.Ping(ctx).Err() + // disable cache for now + return nil } func (r *redisProvider) Redis() redis.UniversalClient { From e7653cadc9b20c08d8bcccbcde256e4c994a965d Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Wed, 6 Aug 2025 18:25:35 +0200 Subject: [PATCH 5/8] invalidate backlinks, init --- flake.lock | 59 +++++++++++++++++++ flake.nix | 25 ++++++++ publish/handler.go | 2 +- publish/service.go | 3 +- .../publishapi/protos/publisher.proto | 1 + publishclient/publishapi/publisher.pb.go | 13 +++- .../publishapi/publisher_vtproto.pb.go | 49 ++++++++++++++- 7 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..1644737 --- /dev/null +++ b/flake.lock @@ -0,0 +1,59 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1753939845, + "narHash": "sha256-K2ViRJfdVGE8tpJejs8Qpvvejks1+A4GQej/lBk5y7I=", + "rev": "94def634a20494ee057c76998843c015909d6311", + "revCount": 837094, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.837094%2Brev-94def634a20494ee057c76998843c015909d6311/019866ba-4140-7e21-9b2b-75f13bfb023e/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.1.0.tar.gz" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ff93b09 --- /dev/null +++ b/flake.nix @@ -0,0 +1,25 @@ +{ + description = ""; + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1.0.tar.gz"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: let + pkgs = import nixpkgs { + inherit system; + config = { allowUnfree = true; }; + }; + in { + devShell = pkgs.mkShell { + name = "anytype-publish-server"; + nativeBuildInputs = [ + pkgs.go_1_24 + pkgs.gox + pkgs.protobuf3_21 + pkgs.pkg-config + pkgs.pre-commit + # todo: govvv, not packaged + ]; + }; + }); +} diff --git a/publish/handler.go b/publish/handler.go index 7f892c0..976ae17 100644 --- a/publish/handler.go +++ b/publish/handler.go @@ -72,7 +72,7 @@ func (r rpcHandler) Publish(ctx context.Context, req *publishapi.PublishRequest) ) }() - uploadUrl, err := r.s.Publish(ctx, domain.Object{SpaceId: req.SpaceId, ObjectId: req.ObjectId, Uri: req.Uri}, req.Version) + uploadUrl, err := r.s.Publish(ctx, domain.Object{SpaceId: req.SpaceId, ObjectId: req.ObjectId, Uri: req.Uri}, req.Version, req.Backlinks) if err != nil { return nil, err } diff --git a/publish/service.go b/publish/service.go index d79461e..34f24af 100644 --- a/publish/service.go +++ b/publish/service.go @@ -131,7 +131,7 @@ func (p *publishService) GetPublishStatus(ctx context.Context, spaceId string, o return p.repo.ObjectPublishStatus(ctx, obj) } -func (p *publishService) Publish(ctx context.Context, object domain.Object, version string) (uploadUrl string, err error) { +func (p *publishService) Publish(ctx context.Context, object domain.Object, version string, backlinks []string) (uploadUrl string, err error) { if object.Identity, err = p.checkIdentity(ctx); err != nil { return } @@ -140,6 +140,7 @@ func (p *publishService) Publish(ctx context.Context, object domain.Object, vers return } if prevUri != "" { + // TODO: invalidate backlinks, check identity p.invalidateCache(object.Identity, prevUri) } return url.JoinPath(p.config.UploadUrlPrefix, publish.Publish.Id.Hex(), publish.Publish.UploadKey) diff --git a/publishclient/publishapi/protos/publisher.proto b/publishclient/publishapi/protos/publisher.proto index 0114f2f..9853f0b 100644 --- a/publishclient/publishapi/protos/publisher.proto +++ b/publishclient/publishapi/protos/publisher.proto @@ -60,6 +60,7 @@ message PublishRequest { string objectId = 2; string uri = 3; string version = 4; + repeated string backlinks = 5; } message PublishResponse { diff --git a/publishclient/publishapi/publisher.pb.go b/publishclient/publishapi/publisher.pb.go index fe6d6ed..eed9952 100644 --- a/publishclient/publishapi/publisher.pb.go +++ b/publishclient/publishapi/publisher.pb.go @@ -442,6 +442,7 @@ type PublishRequest struct { ObjectId string `protobuf:"bytes,2,opt,name=objectId,proto3" json:"objectId,omitempty"` Uri string `protobuf:"bytes,3,opt,name=uri,proto3" json:"uri,omitempty"` Version string `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` + Backlinks []string `protobuf:"bytes,5,rep,name=backlinks,proto3" json:"backlinks,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -504,6 +505,13 @@ func (x *PublishRequest) GetVersion() string { return "" } +func (x *PublishRequest) GetBacklinks() []string { + if x != nil { + return x.Backlinks + } + return nil +} + type PublishResponse struct { state protoimpl.MessageState `protogen:"open.v1"` UploadUrl string `protobuf:"bytes,1,opt,name=uploadUrl,proto3" json:"uploadUrl,omitempty"` @@ -710,12 +718,13 @@ const file_publishclient_publishapi_protos_publisher_proto_rawDesc = "" + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + "\bobjectId\x18\x02 \x01(\tR\bobjectId\"E\n" + "\x18GetPublishStatusResponse\x12)\n" + - "\apublish\x18\x01 \x01(\v2\x0f.client.PublishR\apublish\"r\n" + + "\apublish\x18\x01 \x01(\v2\x0f.client.PublishR\apublish\"\x90\x01\n" + "\x0ePublishRequest\x12\x18\n" + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + "\bobjectId\x18\x02 \x01(\tR\bobjectId\x12\x10\n" + "\x03uri\x18\x03 \x01(\tR\x03uri\x12\x18\n" + - "\aversion\x18\x04 \x01(\tR\aversion\"/\n" + + "\aversion\x18\x04 \x01(\tR\aversion\x12\x1c\n" + + "\tbacklinks\x18\x05 \x03(\tR\tbacklinks\"/\n" + "\x0fPublishResponse\x12\x1c\n" + "\tuploadUrl\x18\x01 \x01(\tR\tuploadUrl\"H\n" + "\x10UnPublishRequest\x12\x18\n" + diff --git a/publishclient/publishapi/publisher_vtproto.pb.go b/publishclient/publishapi/publisher_vtproto.pb.go index a7e6c66..95fd85c 100644 --- a/publishclient/publishapi/publisher_vtproto.pb.go +++ b/publishclient/publishapi/publisher_vtproto.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go-vtproto. DO NOT EDIT. -// protoc-gen-go-vtproto version: v0.6.0 +// protoc-gen-go-vtproto version: v0.6.1-0.20250313105119-ba97887b0a25 // source: publishclient/publishapi/protos/publisher.proto package publishapi @@ -330,6 +330,15 @@ func (m *PublishRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Backlinks) > 0 { + for iNdEx := len(m.Backlinks) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Backlinks[iNdEx]) + copy(dAtA[i:], m.Backlinks[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Backlinks[iNdEx]))) + i-- + dAtA[i] = 0x2a + } + } if len(m.Version) > 0 { i -= len(m.Version) copy(dAtA[i:], m.Version) @@ -660,6 +669,12 @@ func (m *PublishRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if len(m.Backlinks) > 0 { + for _, s := range m.Backlinks { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } n += len(m.unknownFields) return n } @@ -1542,6 +1557,38 @@ func (m *PublishRequest) UnmarshalVT(dAtA []byte) error { } m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Backlinks", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Backlinks = append(m.Backlinks, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) From 775f5f6feae9b77112e6da5866cc8110967bf283 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Tue, 12 Aug 2025 19:37:48 +0200 Subject: [PATCH 6/8] backlinks invalidate, wip --- gateway/gateway.go | 24 ++++++++++++++++++++---- publish/service.go | 18 +++++++++++------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/gateway/gateway.go b/gateway/gateway.go index cb80776..e956405 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -280,8 +280,6 @@ func (g *gateway) renderPage(ctx context.Context, id cacheId) (*pageObject, erro return nil, err } - // TODO: url maps are not updated in cache, - // invalidate cache somehow linkObjectIds := rend.GetLinkObjectIds() objectIdToUrl, err := g.getObjectIdToUrl(ctx, linkObjectIds) if err == nil { @@ -311,7 +309,7 @@ func (g *gateway) getObjectIdToUrl(ctx context.Context, linkObjectIds []string) anyname, err := g.nameService.ResolveIdentity(ctx, publish.Identity) // TODO: move to config if err != nil { - url = fmt.Sprintf("%s/%s/%s",g.config.SelfURL, publish.Identity, publish.Uri) + url = fmt.Sprintf("%s/%s/%s", g.config.SelfURL, publish.Identity, publish.Uri) } else { url = fmt.Sprintf("https://%s.org/%s", anyname, publish.Uri) } @@ -321,7 +319,25 @@ func (g *gateway) getObjectIdToUrl(ctx context.Context, linkObjectIds []string) return objectIdToUrl, nil } -func (g *gateway) invalidateCache(identity, uri string) { +func (g *gateway) invalidateCache(identity, uri string, backlinks []string) { + // for all backlinks: + // find this object in db + // get identity, uri + // create keys, add to (for) + keysToDel := make([]string, len(backlinks)) + publishedObjects, err := g.publish.GetPublishesByObjectIds(context.Background(), backlinks) + if err != nil { + log.Error("failed to GetPublishesByObjectIds:", zap.Error(err)) + } else { + for _, publishedObject := range publishedObjects { + identity := publishedObject.Identity + uri := publishedObject.Uri + withName := "{" + string(newCacheId(identity, uri, true)) + "}" + withoutName := "{" + string(newCacheId(identity, uri, false)) + "}" + keysToDel = append(keysToDel, withName, withoutName) + } + } + withName := "{" + string(newCacheId(identity, uri, true)) + "}" withoutName := "{" + string(newCacheId(identity, uri, false)) + "}" for _, key := range []string{withName, withoutName} { diff --git a/publish/service.go b/publish/service.go index 34f24af..f72378a 100644 --- a/publish/service.go +++ b/publish/service.go @@ -49,7 +49,7 @@ func New() Service { type Service interface { ResolveUriWithIdentity(ctx context.Context, name, uri string) (publish domain.Object, err error) - SetInvalidateCacheCallback(f func(identity, uri string)) + SetInvalidateCacheCallback(f func(identity, uri string, backlinks []string)) GetPublishesByObjectIds(ctx context.Context, objectIds []string) ([]domain.ObjectWithPublish, error) app.ComponentRunnable } @@ -62,7 +62,7 @@ type publishService struct { ticker periodicsync.PeriodicSync nameService nameservice.NameService metric metric.Metric - invalidateFunc func(identity string, uri string) + invalidateFunc func(identity string, uri string, backlinks []string) } func (p *publishService) Init(a *app.App) (err error) { @@ -100,13 +100,13 @@ func (p *publishService) Name() (name string) { return CName } -func (p *publishService) SetInvalidateCacheCallback(f func(identity, uri string)) { +func (p *publishService) SetInvalidateCacheCallback(f func(identity, uri string, backlinks []string)) { p.invalidateFunc = f } -func (p *publishService) invalidateCache(identity, uri string) { +func (p *publishService) invalidateCache(identity, uri string, backlinks []string) { if p.invalidateFunc != nil { - p.invalidateFunc(identity, uri) + p.invalidateFunc(identity, uri, backlinks) } } @@ -141,7 +141,8 @@ func (p *publishService) Publish(ctx context.Context, object domain.Object, vers } if prevUri != "" { // TODO: invalidate backlinks, check identity - p.invalidateCache(object.Identity, prevUri) + // im + p.invalidateCache(object.Identity, prevUri, backlinks) } return url.JoinPath(p.config.UploadUrlPrefix, publish.Publish.Id.Hex(), publish.Publish.UploadKey) } @@ -154,7 +155,10 @@ func (p *publishService) UnPublish(ctx context.Context, object domain.Object) (e if err != nil { return err } - p.invalidateCache(object.Identity, uri) + // TODO: with empty backlinks here, we keep existing backlinks as-is, + // which means links will point to 404 -- at least until cache is expired. + // It is ok, but we can desire other behavior + p.invalidateCache(object.Identity, uri, []string{}) return } From 2d5342c0e871c6d8e42d57d42bfc9ade48c87195 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Wed, 13 Aug 2025 17:59:23 +0200 Subject: [PATCH 7/8] redis, use {identity} for sharding --- gateway/gateway.go | 30 +++++++++++++++++------------- gateway/gateway_test.go | 2 +- publish/service.go | 3 ++- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/gateway/gateway.go b/gateway/gateway.go index e956405..7a25433 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -157,7 +157,7 @@ func (g *gateway) handlePage(ctx context.Context, w http.ResponseWriter, identit func (g *gateway) cacheGet(ctx context.Context, key cacheId) (res *pageObject, err error) { var results = make([]*redis.StringCmd, 3) _, err = g.redisClient.Pipelined(ctx, func(pipe redis.Pipeliner) error { - redisKey := "{" + string(key) + "}" + redisKey := string(key) results[0] = pipe.GetEx(ctx, redisKey+":rver", time.Hour) results[1] = pipe.GetEx(ctx, redisKey+":notfound", time.Hour) results[2] = pipe.GetEx(ctx, redisKey+":body", time.Hour) @@ -208,7 +208,7 @@ func (g *gateway) cacheSet(ctx context.Context, key cacheId, data *pageObject) ( if data.IsNotFound { isNotFound = "1" } - redisKey := "{" + string(key) + "}" + redisKey := string(key) pipe.SetEx(ctx, redisKey+":rver", data.RenderVer, time.Hour) pipe.SetEx(ctx, redisKey+":notfound", isNotFound, time.Hour) @@ -320,27 +320,28 @@ func (g *gateway) getObjectIdToUrl(ctx context.Context, linkObjectIds []string) } func (g *gateway) invalidateCache(identity, uri string, backlinks []string) { - // for all backlinks: - // find this object in db - // get identity, uri - // create keys, add to (for) - keysToDel := make([]string, len(backlinks)) + var keysToDel []string + publishedObjects, err := g.publish.GetPublishesByObjectIds(context.Background(), backlinks) if err != nil { + keysToDel = make([]string, 2) log.Error("failed to GetPublishesByObjectIds:", zap.Error(err)) } else { + keysToDel = make([]string, len(backlinks)+2) for _, publishedObject := range publishedObjects { identity := publishedObject.Identity uri := publishedObject.Uri - withName := "{" + string(newCacheId(identity, uri, true)) + "}" - withoutName := "{" + string(newCacheId(identity, uri, false)) + "}" + withName := string(newCacheId(identity, uri, true)) + withoutName := string(newCacheId(identity, uri, false)) keysToDel = append(keysToDel, withName, withoutName) } } - withName := "{" + string(newCacheId(identity, uri, true)) + "}" - withoutName := "{" + string(newCacheId(identity, uri, false)) + "}" - for _, key := range []string{withName, withoutName} { + withName := string(newCacheId(identity, uri, true)) + withoutName := string(newCacheId(identity, uri, false)) + keysToDel = append(keysToDel, withName, withoutName) + + for _, key := range keysToDel { err := g.redisClient.Del( context.Background(), key+":rver", @@ -363,7 +364,9 @@ var cacheIdSep = string([]byte{0}) func newCacheId(identity, uri string, withName bool) cacheId { var res strings.Builder + res.WriteString("{") res.WriteString(identity) + res.WriteString("}") res.WriteString(cacheIdSep) res.WriteString(uri) res.WriteString(cacheIdSep) @@ -378,7 +381,8 @@ func newCacheId(identity, uri string, withName bool) cacheId { type cacheId string func (c cacheId) Identity() string { - return c.getElement(1) + id := c.getElement(1) + return id[1 : len(id)-1] } func (c cacheId) Uri() string { diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index bf6985a..a19f092 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -11,5 +11,5 @@ func Test_cacheId_getElement(t *testing.T) { assert.Equal(t, "identity", id.Identity()) assert.Equal(t, "uri/a/b", id.Uri()) assert.True(t, id.WithName()) - assert.Equal(t, "identity/uri/a/b/1", id.String()) + assert.Equal(t, "{identity}/uri/a/b/1", id.String()) } diff --git a/publish/service.go b/publish/service.go index f72378a..0b4fb3d 100644 --- a/publish/service.go +++ b/publish/service.go @@ -211,7 +211,8 @@ func (p *publishService) UploadTar(ctx context.Context, publishId, uploadKey str if err = p.repo.FinalizePublish(ctx, objWithPub); err != nil { return } - p.invalidateCache(objWithPub.Identity, objWithPub.Uri) + // TODO: invalidate backlinks? maybe not, we do it upon Publish. Double-check. + p.invalidateCache(objWithPub.Identity, objWithPub.Uri, []string{}) return url.JoinPath("https://", p.gatewayConfig.Domain, publish.ObjectId) } From f08d7b0296455f5a486c5427752c62130a0d0517 Mon Sep 17 00:00:00 2001 From: Anatolii Smolianinov Date: Thu, 14 Aug 2025 17:46:45 +0200 Subject: [PATCH 8/8] bump renderer, multipublish --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8337c28..d707d4b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.0 require ( github.com/ahmetb/govvv v0.3.0 github.com/anyproto/any-sync v0.9.2 - github.com/anyproto/anytype-publish-renderer v0.3.21 + github.com/anyproto/anytype-publish-renderer v0.4.0-alpha github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250716122732-cdcfe3a126bb github.com/aws/aws-sdk-go-v2 v1.36.5 github.com/aws/aws-sdk-go-v2/config v1.29.14 @@ -129,5 +129,3 @@ replace github.com/anyproto/anytype-publish-server/publishclient => ./publishcli replace gopkg.in/Graylog2/go-gelf.v2 => github.com/anyproto/go-gelf v0.0.0-20210418191311-774bd5b016e7 replace github.com/btcsuite/btcutil => github.com/btcsuite/btcd/btcutil v1.1.5 - -replace github.com/anyproto/anytype-publish-renderer => ../anytype-publish-renderer diff --git a/go.sum b/go.sum index 9e15838..b47fb6d 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/anyproto/any-sync v0.9.2 h1:nqKatxhHVnnKYk05dEYTaKbW2+oiMvZxpYvnZtwHc github.com/anyproto/any-sync v0.9.2/go.mod h1:egr+ItXzDs4awlDbNA8UGxWJZWiBnuBA8cUYazAkERk= github.com/anyproto/anytype-heart v0.42.0-rc30 h1:cCDzDmbPxgtuPc5qzTiz2QRZ/UCIfBoeRj/Q55dOgD4= github.com/anyproto/anytype-heart v0.42.0-rc30/go.mod h1:dlMltJFcvGrJUQssEhV3Ffx3MndWny+aYFlN/j9RgPU= +github.com/anyproto/anytype-publish-renderer v0.4.0-alpha h1:Nbe3RrAhJ1WLWJbMKK9SfKb6gXX3MGNKxMhMajMwZks= +github.com/anyproto/anytype-publish-renderer v0.4.0-alpha/go.mod h1:Gk8WeVz7ZiLkktvzAmqog6uRt9ztovAH2q9PwDAkwj8= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY= github.com/anyproto/go-gelf v0.0.0-20210418191311-774bd5b016e7 h1:SyEu5uxZ5nKHEJ6TPKQqjM+T00SYi0MW1VaLzqZtZ9E=