Skip to content

client: remove meta revision #10543

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bufferflies wants to merge 5 commits into
base: master
Choose a base branch
from
Open

client: remove meta revision #10543

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4ad4e84
client: remove meta revision
bufferflies Apr 1, 2026
daef237
client: reload RM snapshot before watch retry
bufferflies Apr 1, 2026
64409c5
client: sync new RM groups from snapshot reload
bufferflies Apr 1, 2026
daf78dc
Revert "client: sync new RM groups from snapshot reload"
bufferflies Apr 8, 2026
1228f9d
Revert "client: reload RM snapshot before watch retry"
bufferflies Apr 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 72 additions & 14 deletions client/resource_group/controller/global_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,10 +322,6 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
stateUpdateTicker.Reset(time.Millisecond * 100)
})

_, metaRevision, err := c.provider.LoadResourceGroups(ctx)
if err != nil {
log.Warn("load resource group revision failed", zap.Error(err))
}
resp, err := c.provider.Get(ctx, []byte(controllerConfigPath))
if err != nil {
log.Warn("load resource group revision failed", zap.Error(err))
Expand All @@ -335,7 +331,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
if !c.ruConfig.isSingleGroupByKeyspace {
// Use WithPrevKV() to get the previous key-value pair when get Delete Event.
prefix := pd.GroupSettingsPathPrefixBytes(c.keyspaceID)
watchMetaChannel, err = c.provider.Watch(ctx, prefix, opt.WithRev(metaRevision), opt.WithPrefix(), opt.WithPrevKV())
watchMetaChannel, err = c.provider.Watch(ctx, prefix, opt.WithPrefix(), opt.WithPrevKV())
Copy link
Copy Markdown
Contributor

@lhy1024 lhy1024 Apr 2, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The initial watch here no longer uses a revision, which leaves a startup window. If a resource group changes after the controller starts but before the watch is actually established, the local cache can miss that update. The reconnect path already rebuilds from a snapshot/revision, but the initial startup path still needs the same barrier.

Copy link
Copy Markdown
Contributor Author

@bufferflies bufferflies Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but we just need the latest event and can ignore all the mvvc events

Copy link
Copy Markdown
Contributor

@lhy1024 lhy1024 Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree we do not need the whole MVCC history. The issue is that without a startup barrier, we may miss the latest update itself if it happens before the watch is established and no later event arrives. In that case the local cache can stay stale indefinitely.

if err != nil {
log.Warn("watch resource group meta failed", zap.Error(err))
}
Expand All @@ -361,9 +357,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
}
case <-watchRetryTimer.C:
if !c.ruConfig.isSingleGroupByKeyspace && watchMetaChannel == nil {
// Use WithPrevKV() to get the previous key-value pair when get Delete Event.
prefix := pd.GroupSettingsPathPrefixBytes(c.keyspaceID)
watchMetaChannel, err = c.provider.Watch(ctx, prefix, opt.WithRev(metaRevision), opt.WithPrefix(), opt.WithPrevKV())
watchMetaChannel, err = c.reloadResourceGroupMetaWatch(ctx)
if err != nil {
log.Warn("watch resource group meta failed", zap.Error(err))
watchRetryTimer.Reset(watchRetryInterval)
Expand Down Expand Up @@ -417,7 +411,6 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
continue
}
for _, item := range resp {
metaRevision = item.Kv.ModRevision
group := &rmpb.ResourceGroup{}
switch item.Type {
case meta_storagepb.Event_PUT:
Expand Down Expand Up @@ -496,6 +489,72 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
}()
}

func (c *ResourceGroupsController) reloadResourceGroupMetaWatch(
ctx context.Context,
) (chan []*meta_storagepb.Event, error) {
groups, revision, err := c.provider.LoadResourceGroups(ctx)
if err != nil {
return nil, err
}
c.syncResourceGroupSnapshot(groups)
// Start from the next revision after the freshly loaded snapshot so reconnects
// keep the cache in sync without replaying the snapshot itself as watch events.
return c.provider.Watch(
ctx,
pd.GroupSettingsPathPrefixBytes(c.keyspaceID),
opt.WithRev(revision+1),
opt.WithPrefix(),
opt.WithPrevKV(),
)
}

func (c *ResourceGroupsController) syncResourceGroupSnapshot(groups []*rmpb.ResourceGroup) {
groupMap := make(map[string]*rmpb.ResourceGroup, len(groups))
for _, group := range groups {
groupMap[group.GetName()] = group
}

c.groupsController.Range(func(key, value any) bool {
name := key.(string)
group, exists := groupMap[name]
if !exists {
c.tombstoneGroupCostController(name)
return true
}
delete(groupMap, name)
gc := value.(*groupCostController)
if !gc.tombstone.Load() {
gc.modifyMeta(group)
return true
}
newGC, err := newGroupCostController(group, c.ruConfig, c.lowTokenNotifyChan, c.tokenBucketUpdateChan)
if err != nil {
log.Warn("[resource group controller] re-create resource group cost controller from snapshot failed",
zap.String("name", name), zap.Error(err))
return true
}
if c.groupsController.CompareAndSwap(name, gc, newGC) {
log.Info("[resource group controller] re-create resource group cost controller from snapshot",
zap.String("name", name))
}
return true
})

for name, group := range groupMap {
gc, err := newGroupCostController(group, c.ruConfig, c.lowTokenNotifyChan, c.tokenBucketUpdateChan)
if err != nil {
log.Warn("[resource group controller] create resource group cost controller from snapshot failed",
zap.String("name", name), zap.Error(err))
continue
}
if loaded := c.loadOrStoreGroupController(name, gc); !loaded {
metrics.ResourceGroupStatusGauge.WithLabelValues(name, group.Name).Set(1)
log.Info("[resource group controller] create resource group cost controller from snapshot",
zap.String("name", name))
}
}
}

// Stop stops ResourceGroupController service.
func (c *ResourceGroupsController) Stop() error {
if c.loopCancel == nil {
Expand All @@ -516,9 +575,9 @@ func (c *ResourceGroupsController) loadGroupController(name string) (*groupCostC
}

// loadOrStoreGroupController just wraps the `LoadOrStore` method of `sync.Map`.
func (c *ResourceGroupsController) loadOrStoreGroupController(name string, gc *groupCostController) (*groupCostController, bool) {
tmp, loaded := c.groupsController.LoadOrStore(name, gc)
return tmp.(*groupCostController), loaded
func (c *ResourceGroupsController) loadOrStoreGroupController(name string, gc *groupCostController) bool {
_, loaded := c.groupsController.LoadOrStore(name, gc)
return loaded
}

// NewResourceGroupNotExistErr returns a new error that indicates the resource group does not exist.
Expand Down Expand Up @@ -576,8 +635,7 @@ func (c *ResourceGroupsController) tryGetResourceGroupController(
}
if !isUseDegradedResourceGroup {
// Check again to prevent initializing the same resource group concurrently.
_, loaded := c.loadOrStoreGroupController(name, gc)
if !loaded {
if loaded := c.loadOrStoreGroupController(name, gc); !loaded {
metrics.ResourceGroupStatusGauge.WithLabelValues(name, group.Name).Set(1)
log.Info("[resource group controller] create resource group cost controller", zap.String("name", name))
}
Expand Down
73 changes: 73 additions & 0 deletions client/resource_group/controller/global_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,79 @@ func TestGetResourceGroup(t *testing.T) {
re.Nil(gc02)
}

func TestReloadResourceGroupMetaWatch(t *testing.T) {
re := require.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

mockProvider := &MockResourceGroupProvider{}
mockProvider.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(&meta_storagepb.GetResponse{}, nil)
controller, err := NewResourceGroupController(ctx, 1, mockProvider, nil, constants.NullKeyspaceID)
re.NoError(err)

defaultGroup := &rmpb.ResourceGroup{
Name: defaultResourceGroupName,
Mode: rmpb.GroupMode_RUMode,
RUSettings: &rmpb.GroupRequestUnitSettings{RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 1000000}}},
}
staleGroup := &rmpb.ResourceGroup{
Name: "stale-group",
Mode: rmpb.GroupMode_RUMode,
RUSettings: &rmpb.GroupRequestUnitSettings{RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 100}}},
}
newGroup := &rmpb.ResourceGroup{
Name: "new-group",
Mode: rmpb.GroupMode_RUMode,
RUSettings: &rmpb.GroupRequestUnitSettings{RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 300}}},
}
updatedDefaultGroup := &rmpb.ResourceGroup{
Name: defaultResourceGroupName,
Mode: rmpb.GroupMode_RUMode,
RUSettings: &rmpb.GroupRequestUnitSettings{RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 2000}}},
}

mockProvider.On("GetResourceGroup", mock.Anything, defaultResourceGroupName, mock.Anything).Return(defaultGroup, nil)
mockProvider.On("GetResourceGroup", mock.Anything, "stale-group", mock.Anything).Return(staleGroup, nil)

defaultGC, err := controller.tryGetResourceGroupController(ctx, defaultResourceGroupName, false)
re.NoError(err)
re.NotNil(defaultGC)

staleGC, err := controller.tryGetResourceGroupController(ctx, "stale-group", false)
re.NoError(err)
re.NotNil(staleGC)
re.False(staleGC.tombstone.Load())

watchCh := make(chan []*meta_storagepb.Event)
mockProvider.On("LoadResourceGroups", mock.Anything).
Return([]*rmpb.ResourceGroup{updatedDefaultGroup, newGroup}, int64(88), nil).
Once()
mockProvider.On("Watch", mock.Anything, pd.GroupSettingsPathPrefixBytes(constants.NullKeyspaceID), mock.Anything).
Run(func(args mock.Arguments) {
opts := args.Get(2).([]opt.MetaStorageOption)
metaOp := &opt.MetaStorageOp{}
for _, apply := range opts {
apply(metaOp)
}
re.Equal(int64(89), metaOp.Revision)
re.True(metaOp.IsOptsWithPrefix)
re.True(metaOp.PrevKv)
}).
Return(watchCh, nil).
Once()

reloadedWatchCh, err := controller.reloadResourceGroupMetaWatch(ctx)
re.NoError(err)
re.Equal(watchCh, reloadedWatchCh)
re.Equal(updatedDefaultGroup, defaultGC.getMeta())
staleGC, err = controller.tryGetResourceGroupController(ctx, "stale-group", true)
re.NoError(err)
re.True(staleGC.tombstone.Load())
newGC, ok := controller.loadGroupController("new-group")
re.True(ok)
re.Equal(newGroup, newGC.getMeta())
}

func TestTokenBucketsRequestWithKeyspaceID(t *testing.T) {
re := require.New(t)
ctx, cancel := context.WithCancel(context.Background())
Expand Down
Loading