diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go index e46c68e1124..3bea135b67e 100644 --- a/pkg/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -38,7 +38,6 @@ import ( "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/keypath" - "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/versioninfo/kerneltype" ) @@ -74,6 +73,7 @@ const ( type Config interface { GetPreAlloc() []string ToWaitRegionSplit() bool + GetDisableRawKVRegionSplit() bool GetWaitRegionSplitTimeout() time.Duration GetCheckRegionSplitInterval() time.Duration } @@ -506,7 +506,8 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er }) start := time.Now() - keyspaceRule := MakeLabelRule(id) + skipRaw := manager.config.GetDisableRawKVRegionSplit() + keyspaceRule := buildLabelRule(id, skipRaw) cl, ok := manager.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }) if !ok { return errors.New("cluster does not support region label") @@ -527,11 +528,17 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er zap.Error(err), ) } + return } + log.Info("[keyspace] added region label for keyspace", + zap.Uint32("keyspace-id", id), + zap.Any("label-rule", keyspaceRule), + zap.Duration("takes", time.Since(start)), + ) }() if waitRegionSplit { - err = manager.waitKeyspaceRegionSplit(id) + err = manager.waitKeyspaceRegionSplit(id, skipRaw) if err != nil { log.Warn("[keyspace] wait region split meets error", zap.Uint32("keyspace-id", id), @@ -540,16 +547,10 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er } return err } - - log.Info("[keyspace] added region label for keyspace", - zap.Uint32("keyspace-id", id), - logutil.ZapRedactString("label-rule", keyspaceRule.String()), - zap.Duration("takes", time.Since(start)), - ) - return + return nil } -func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error { +func (manager *Manager) waitKeyspaceRegionSplit(id uint32, skipRaw bool) error { ticker := time.NewTicker(manager.config.GetCheckRegionSplitInterval()) timer := time.NewTimer(manager.config.GetWaitRegionSplitTimeout()) defer func() { @@ -561,7 +562,7 @@ func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error { case <-manager.ctx.Done(): return errors.New("[keyspace] wait region split canceled") case <-ticker.C: - if manager.CheckKeyspaceRegionBound(id) { + if manager.hasKeyspaceRegionBound(id, skipRaw) { log.Info("[keyspace] wait region split successfully", zap.Uint32("keyspace-id", id)) return nil } @@ -576,7 +577,15 @@ func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error { // CheckKeyspaceRegionBound checks whether the keyspace region has been split. func (manager *Manager) CheckKeyspaceRegionBound(id uint32) bool { + return manager.hasKeyspaceRegionBound(id, manager.config.GetDisableRawKVRegionSplit()) +} + +func (manager *Manager) hasKeyspaceRegionBound(id uint32, skipRaw bool) bool { regionBound := MakeRegionBound(id) + if skipRaw { + return manager.checkBound(regionBound.TxnLeftBound) && + manager.checkBound(regionBound.TxnRightBound) + } return manager.checkBound(regionBound.RawLeftBound) && manager.checkBound(regionBound.RawRightBound) && manager.checkBound(regionBound.TxnLeftBound) && diff --git a/pkg/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go index adab5f3b784..639ef4f64a4 100644 --- a/pkg/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -66,6 +66,7 @@ func TestKeyspaceTestSuite(t *testing.T) { type mockConfig struct { PreAlloc []string WaitRegionSplit bool + DisableRawKVRegionSplit bool WaitRegionSplitTimeout typeutil.Duration CheckRegionSplitInterval typeutil.Duration } @@ -86,6 +87,10 @@ func (m *mockConfig) GetCheckRegionSplitInterval() time.Duration { return m.CheckRegionSplitInterval.Duration } +func (m *mockConfig) GetDisableRawKVRegionSplit() bool { + return m.DisableRawKVRegionSplit +} + func (suite *keyspaceTestSuite) SetupTest() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) diff --git a/pkg/keyspace/util.go b/pkg/keyspace/util.go index 7df93eefb19..a7262afd30a 100644 --- a/pkg/keyspace/util.go +++ b/pkg/keyspace/util.go @@ -126,9 +126,21 @@ func MakeRegionBound(id uint32) *RegionBound { } } -// MakeKeyRanges encodes keyspace ID to correct LabelRule data. +// MakeKeyRanges encodes keyspace ID to the correct LabelRule data. func MakeKeyRanges(id uint32) []any { + return buildKeyRanges(id, false) +} + +func buildKeyRanges(id uint32, skipRaw bool) []any { regionBound := MakeRegionBound(id) + if skipRaw { + return []any{ + map[string]any{ + "start_key": hex.EncodeToString(regionBound.TxnLeftBound), + "end_key": hex.EncodeToString(regionBound.TxnRightBound), + }, + } + } return []any{ map[string]any{ "start_key": hex.EncodeToString(regionBound.RawLeftBound), @@ -148,6 +160,10 @@ func getRegionLabelID(id uint32) string { // MakeLabelRule makes the label rule for the given keyspace id. func MakeLabelRule(id uint32) *labeler.LabelRule { + return buildLabelRule(id, false) +} + +func buildLabelRule(id uint32, skipRaw bool) *labeler.LabelRule { return &labeler.LabelRule{ ID: getRegionLabelID(id), Index: 0, @@ -158,7 +174,7 @@ func MakeLabelRule(id uint32) *labeler.LabelRule { }, }, RuleType: labeler.KeyRange, - Data: MakeKeyRanges(id), + Data: buildKeyRanges(id, skipRaw), } } diff --git a/pkg/keyspace/util_test.go b/pkg/keyspace/util_test.go index a4b44e2bdce..7c9e269da05 100644 --- a/pkg/keyspace/util_test.go +++ b/pkg/keyspace/util_test.go @@ -144,10 +144,12 @@ func TestMakeLabelRule(t *testing.T) { re := require.New(t) testCases := []struct { id uint32 + skipRaw bool expectedLabelRule *labeler.LabelRule }{ { - id: 0, + id: 0, + skipRaw: false, expectedLabelRule: &labeler.LabelRule{ ID: getRegionLabelID(0), Index: 0, @@ -171,7 +173,8 @@ func TestMakeLabelRule(t *testing.T) { }, }, { - id: 4242, + id: 4242, + skipRaw: false, expectedLabelRule: &labeler.LabelRule{ ID: getRegionLabelID(4242), Index: 0, @@ -194,9 +197,51 @@ func TestMakeLabelRule(t *testing.T) { }, }, }, + { + id: 0, + skipRaw: true, + expectedLabelRule: &labeler.LabelRule{ + ID: "keyspaces/0", + Index: 0, + Labels: []labeler.RegionLabel{ + { + Key: "id", + Value: "0", + }, + }, + RuleType: "key-range", + Data: []any{ + map[string]any{ + "start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0, 0})), + "end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0, 1})), + }, + }, + }, + }, + { + id: 4242, + skipRaw: true, + expectedLabelRule: &labeler.LabelRule{ + ID: "keyspaces/4242", + Index: 0, + Labels: []labeler.RegionLabel{ + { + Key: "id", + Value: "4242", + }, + }, + RuleType: "key-range", + Data: []any{ + map[string]any{ + "start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0x10, 0x92})), + "end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0x10, 0x93})), + }, + }, + }, + }, } for _, testCase := range testCases { - re.Equal(testCase.expectedLabelRule, MakeLabelRule(testCase.id)) + re.Equal(testCase.expectedLabelRule, buildLabelRule(testCase.id, testCase.skipRaw)) } } diff --git a/pkg/mcs/scheduling/server/rule/watcher_test.go b/pkg/mcs/scheduling/server/rule/watcher_test.go index 8daf8ec8e58..53928cc5799 100644 --- a/pkg/mcs/scheduling/server/rule/watcher_test.go +++ b/pkg/mcs/scheduling/server/rule/watcher_test.go @@ -16,6 +16,7 @@ package rule import ( "context" + "encoding/hex" "encoding/json" "os" "strconv" @@ -99,11 +100,17 @@ func prepare(t require.TestingT) (context.Context, *clientv3.Client, func()) { <-etcd.Server.ReadyNotify() for i := 1; i < rulesNum+1; i++ { + regionBound := keyspace.MakeRegionBound(uint32(i)) rule := &labeler.LabelRule{ ID: "test_" + strconv.Itoa(i), Labels: []labeler.RegionLabel{{Key: "test", Value: "test"}}, RuleType: labeler.KeyRange, - Data: keyspace.MakeKeyRanges(uint32(i)), + Data: labeler.MakeKeyRanges( + hex.EncodeToString(regionBound.TxnLeftBound), + hex.EncodeToString(regionBound.TxnRightBound), + hex.EncodeToString(regionBound.RawLeftBound), + hex.EncodeToString(regionBound.RawRightBound), + ), } value, err := json.Marshal(rule) re.NoError(err) diff --git a/server/config/config.go b/server/config/config.go index b222bf45b6c..70df96c9788 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -245,6 +245,7 @@ const ( defaultGCTunerThreshold = 0.6 minGCTunerThreshold = 0 maxGCTunerThreshold = 0.9 + defaultDisableRawKVRegionSplit = false defaultWaitRegionSplitTimeout = 30 * time.Second defaultCheckRegionSplitInterval = 50 * time.Millisecond @@ -877,6 +878,8 @@ type KeyspaceConfig struct { WaitRegionSplit bool `toml:"wait-region-split" json:"wait-region-split"` // WaitRegionSplitTimeout indicates the max duration to wait region split. WaitRegionSplitTimeout typeutil.Duration `toml:"wait-region-split-timeout" json:"wait-region-split-timeout"` + // DisableRawKVRegionSplit indicates whether to skip raw kv region split. + DisableRawKVRegionSplit bool `toml:"disable-raw-kv-region-split" json:"disable-raw-kv-region-split"` // CheckRegionSplitInterval indicates the interval to check whether the region split is complete CheckRegionSplitInterval typeutil.Duration `toml:"check-region-split-interval" json:"check-region-split-interval"` } @@ -903,6 +906,9 @@ func (c *KeyspaceConfig) adjust(meta *configutil.ConfigMetaData) { if !meta.IsDefined("check-region-split-interval") { c.CheckRegionSplitInterval = typeutil.NewDuration(defaultCheckRegionSplitInterval) } + if !meta.IsDefined("disable-raw-kv-region-split") { + c.DisableRawKVRegionSplit = defaultDisableRawKVRegionSplit + } } // Clone makes a deep copy of the keyspace config. @@ -923,6 +929,11 @@ func (c *KeyspaceConfig) ToWaitRegionSplit() bool { return c.WaitRegionSplit } +// GetDisableRawKVRegionSplit returns whether to skip raw kv region split. +func (c *KeyspaceConfig) GetDisableRawKVRegionSplit() bool { + return c.DisableRawKVRegionSplit +} + // GetWaitRegionSplitTimeout returns the max duration to wait region split. func (c *KeyspaceConfig) GetWaitRegionSplitTimeout() time.Duration { return c.WaitRegionSplitTimeout.Duration diff --git a/server/config/config_test.go b/server/config/config_test.go index 9f8643da521..c592ace2957 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -327,6 +327,25 @@ disable-make-up-replica = false re.Error(err) } +func TestKeyspaceConfigJSONBoolEncoding(t *testing.T) { + re := require.New(t) + cfg := NewConfig() + cfg.Keyspace.DisableRawKVRegionSplit = true + cfg.Keyspace.WaitRegionSplit = true + + data, err := json.Marshal(cfg) + re.NoError(err) + + var payload map[string]any + re.NoError(json.Unmarshal(data, &payload)) + + keyspaceCfg, ok := payload["keyspace"].(map[string]any) + re.True(ok) + re.IsType(true, keyspaceCfg["disable-raw-kv-region-split"]) + re.Equal(true, keyspaceCfg["disable-raw-kv-region-split"]) + re.IsType(true, keyspaceCfg["wait-region-split"]) +} + func TestPDServerConfig(t *testing.T) { re := require.New(t) tests := []struct {