Skip to content

Keeper priority #622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
arssher wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Keeper priority #622

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fce98e
Add --priority keeper option.
arssher Mar 11, 2019
988568e
Regenerate docs for keeper --priority option.
ololobus Jul 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions cmd/keeper/cmd/keeper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ type config struct {
uid string
dataDir string
debug bool
priority int
prioritySpecified bool // true iff explicitly set by user
pgListenAddress string
pgAdvertiseAddress string
pgPort string
Expand Down Expand Up @@ -139,6 +141,7 @@ func init() {
CmdKeeper.PersistentFlags().StringVar(&cfg.pgSUPassword, "pg-su-password", "", "postgres superuser password. Only one of --pg-su-password or --pg-su-passwordfile must be provided. Must be the same for all keepers.")
CmdKeeper.PersistentFlags().StringVar(&cfg.pgSUPasswordFile, "pg-su-passwordfile", "", "postgres superuser password file. Only one of --pg-su-password or --pg-su-passwordfile must be provided. Must be the same for all keepers)")
CmdKeeper.PersistentFlags().BoolVar(&cfg.debug, "debug", false, "enable debug logging")
CmdKeeper.PersistentFlags().IntVar(&cfg.priority, "priority", 0, "keeper priority, integer. Stolon will promote available keeper with higher priority than current master, if this is possible. Healthy keeper with higher priority will be elected even if current master is online. If not specified, priority is set to "+strconv.Itoa(cluster.DefaultPriority)+" on first keeper invocation; on subsequent invocations, last value (which could be also set with 'stolonctl setkeeperpriority') is reused.")

CmdKeeper.PersistentFlags().BoolVar(&cfg.canBeMaster, "can-be-master", true, "prevent keeper from being elected as master")
CmdKeeper.PersistentFlags().BoolVar(&cfg.canBeSynchronousReplica, "can-be-synchronous-replica", true, "prevent keeper from being chosen as synchronous replica")
Expand Down Expand Up @@ -451,6 +454,8 @@ type PostgresKeeper struct {
pgSUUsername string
pgSUPassword string

priority *int // nil means not specified

sleepInterval time.Duration
requestTimeout time.Duration

Expand Down Expand Up @@ -484,6 +489,10 @@ func NewPostgresKeeper(cfg *config, end chan error) (*PostgresKeeper, error) {
return nil, fmt.Errorf("cannot get absolute datadir path for %q: %v", cfg.dataDir, err)
}

var priority *int = nil
if cfg.prioritySpecified {
priority = &cfg.priority
}
p := &PostgresKeeper{
cfg: cfg,

Expand All @@ -503,6 +512,8 @@ func NewPostgresKeeper(cfg *config, end chan error) (*PostgresKeeper, error) {
pgSUUsername: cfg.pgSUUsername,
pgSUPassword: cfg.pgSUPassword,

priority: priority,

sleepInterval: cluster.DefaultSleepInterval,
requestTimeout: cluster.DefaultRequestTimeout,

Expand Down Expand Up @@ -578,6 +589,7 @@ func (p *PostgresKeeper) updateKeeperInfo() error {
Maj: maj,
Min: min,
},
Priority: p.priority,
PostgresState: p.getLastPGState(),

CanBeMaster: p.canBeMaster,
Expand Down Expand Up @@ -2029,6 +2041,10 @@ func keeper(c *cobra.Command, args []string) {
}
}

// if --priority wasn't specified explictily, last value is reused, so
// remember it
cfg.prioritySpecified = c.Flags().Changed("priority")

// Open (and create if needed) the lock file.
// There is no need to clean up this file since we don't use the file as an actual lock. We get a lock
// on the file. So the lock get released when our process stops (or log.Fatalfs).
Expand Down
110 changes: 73 additions & 37 deletions cmd/sentinel/cmd/sentinel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,11 @@ func (s *Sentinel) updateKeepersStatus(cd *cluster.ClusterData, keepersInfo clus
} else {
s.CleanKeeperError(keeperUID)
// Update keeper status infos
// If keeper restarted with specified priority, update it
if ki.Priority != nil &&
k.Status.BootUUID != ki.BootUUID {
k.Spec.Priority = *ki.Priority
}
k.Status.BootUUID = ki.BootUUID
k.Status.PostgresBinaryVersion.Maj = ki.PostgresBinaryVersion.Maj
k.Status.PostgresBinaryVersion.Min = ki.PostgresBinaryVersion.Min
Expand Down Expand Up @@ -699,12 +704,17 @@ func (s *Sentinel) validStandbysByStatus(cd *cluster.ClusterData) (map[string]*c
return goodStandbys, failedStandbys, convergingStandbys
}

// dbSlice implements sort interface to sort by XLogPos
type dbSlice []*cluster.DB

func (p dbSlice) Len() int { return len(p) }
func (p dbSlice) Less(i, j int) bool { return p[i].Status.XLogPos < p[j].Status.XLogPos }
func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// sort dbs by XLogPos and keeper's priority
func sortDBs(cd *cluster.ClusterData, dbs []*cluster.DB) {
sort.Slice(dbs, func(i, j int) bool {
if dbs[i].Status.XLogPos != dbs[j].Status.XLogPos {
return dbs[i].Status.XLogPos < dbs[j].Status.XLogPos
}
pi := cd.Keepers[dbs[i].Spec.KeeperUID].Spec.Priority
pj := cd.Keepers[dbs[j].Spec.KeeperUID].Spec.Priority
return pi < pj

@dineshba dineshba May 25, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Line 700,

pj := cd.Keepers[dbs[i].Spec.KeeperUID].Status.Priority

replace dbs[i] to dbs[j]

@arssher arssher Jun 30, 2019

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ups, good catch, thanks.

})
}

func (s *Sentinel) findBestStandbys(cd *cluster.ClusterData, masterDB *cluster.DB) []*cluster.DB {
goodStandbys, _, _ := s.validStandbysByStatus(cd)
Expand All @@ -726,7 +736,7 @@ func (s *Sentinel) findBestStandbys(cd *cluster.ClusterData, masterDB *cluster.D
bestDBs = append(bestDBs, db)
}
// Sort by XLogPos
sort.Sort(dbSlice(bestDBs))
sortDBs(cd, bestDBs)
return bestDBs
}

Expand Down Expand Up @@ -773,11 +783,54 @@ func (s *Sentinel) findBestNewMasters(cd *cluster.ClusterData, masterDB *cluster
}

// Sort by XLogPos
sort.Sort(dbSlice(bestNewMasters))
sortDBs(cd, bestNewMasters)
log.Debugf("bestNewMasters: %s", spew.Sdump(bestNewMasters))
return bestNewMasters
}

// findBestNewMaster returns the DB who can be a new master. This function mostly takes care of
// sync mode; in async case new master is just a first element of findBestNewMasters.
func (s *Sentinel) findBestNewMaster(cd *cluster.ClusterData, curMasterDB *cluster.DB, logErrors bool) *cluster.DB {

@sgotti sgotti May 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

findBestNewMaster returns the db who can be the new master. This function ....

@arssher arssher Jun 30, 2019

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm?

bestNewMasters := s.findBestNewMasters(cd, curMasterDB)
if len(bestNewMasters) == 0 {
if logErrors {
log.Errorw("no eligible masters")
}
return nil
}

// if synchronous replication is enabled, only choose new master in the synchronous replication standbys.
var bestNewMasterDB *cluster.DB = nil
if curMasterDB.Spec.SynchronousReplication {
commonSyncStandbys := util.CommonElements(curMasterDB.Status.SynchronousStandbys, curMasterDB.Spec.SynchronousStandbys)
if len(commonSyncStandbys) == 0 {
if logErrors {
log.Warnw("cannot choose synchronous standby since there are no common elements between the latest master reported synchronous standbys and the db spec ones", "reported", curMasterDB.Status.SynchronousStandbys, "spec", curMasterDB.Spec.SynchronousStandbys)
}
return nil
}
// In synchronous mode there is no need to choose DB with
// highest LSN; all found dbs must be in sync, so pick the one
// with highest priority.
var newMasterPriority int
for _, nm := range bestNewMasters {
if util.StringInSlice(commonSyncStandbys, nm.UID) {
nmPriority := cd.Keepers[nm.Spec.KeeperUID].Spec.Priority
if (bestNewMasterDB == nil) || (nmPriority > newMasterPriority) {
bestNewMasterDB = nm
newMasterPriority = nmPriority
}
}
}
if bestNewMasterDB == nil && logErrors {
log.Warnw("cannot choose synchronous standby since there's not match between the possible masters and the usable synchronousStandbys", "reported", curMasterDB.Status.SynchronousStandbys, "spec", curMasterDB.Spec.SynchronousStandbys, "common", commonSyncStandbys, "possibleMasters", bestNewMasters)
}
} else {
bestNewMasterDB = bestNewMasters[0]
}
return bestNewMasterDB
}

func (s *Sentinel) updateCluster(cd *cluster.ClusterData, pis cluster.ProxiesInfo) (*cluster.ClusterData, error) {
// take a cd deepCopy to check that the code isn't changing it (it'll be a bug)
origcd := cd.DeepCopy()
Expand Down Expand Up @@ -1002,37 +1055,20 @@ func (s *Sentinel) updateCluster(cd *cluster.ClusterData, pis cluster.ProxiesInf
masterOK = false
}

if !masterOK {
log.Infow("trying to find a new master to replace failed master")
bestNewMasters := s.findBestNewMasters(newcd, curMasterDB)
if len(bestNewMasters) == 0 {
log.Errorw("no eligible masters")
bestNewMasterDB := s.findBestNewMaster(newcd, curMasterDB, !masterOK)
if bestNewMasterDB != nil {
if !masterOK {
log.Infow("electing db as the new master", "db", bestNewMasterDB.UID, "keeper", bestNewMasterDB.Spec.KeeperUID)
wantedMasterDBUID = bestNewMasterDB.UID
} else {
// if synchronous replication is enabled, only choose new master in the synchronous replication standbys.
var bestNewMasterDB *cluster.DB
if curMasterDB.Spec.SynchronousReplication {
commonSyncStandbys := util.CommonElements(curMasterDB.Status.SynchronousStandbys, curMasterDB.Spec.SynchronousStandbys)
if len(commonSyncStandbys) == 0 {
log.Warnw("cannot choose synchronous standby since there are no common elements between the latest master reported synchronous standbys and the db spec ones", "reported", curMasterDB.Status.SynchronousStandbys, "spec", curMasterDB.Spec.SynchronousStandbys)
} else {
for _, nm := range bestNewMasters {
if util.StringInSlice(commonSyncStandbys, nm.UID) {
bestNewMasterDB = nm
break
}
}
if bestNewMasterDB == nil {
log.Warnw("cannot choose synchronous standby since there's not match between the possible masters and the usable synchronousStandbys", "reported", curMasterDB.Status.SynchronousStandbys, "spec", curMasterDB.Spec.SynchronousStandbys, "common", commonSyncStandbys, "possibleMasters", bestNewMasters)
}
}
} else {
bestNewMasterDB = bestNewMasters[0]
}
if bestNewMasterDB != nil {
log.Infow("electing db as the new master", "db", bestNewMasterDB.UID, "keeper", bestNewMasterDB.Spec.KeeperUID)
// Even if current master is ok, we probably still
// want to change it if there is ready DB with higher
// keeper priority.

@sgotti sgotti May 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So the priority will be used also to changing primary also when there's no failure? This should probably be documented

@arssher arssher Jun 30, 2019

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. I have added some words to --priority help describing this.

curMasterPriority := cd.Keepers[curMasterDB.Spec.KeeperUID].Spec.Priority
newMasterPriority := cd.Keepers[bestNewMasterDB.Spec.KeeperUID].Spec.Priority
if newMasterPriority > curMasterPriority {
log.Infow("electing db as the new master because it has higher priority", "db", bestNewMasterDB.UID, "keeper", bestNewMasterDB.Spec.KeeperUID, "currPriority", curMasterPriority, "newPriority", newMasterPriority)
wantedMasterDBUID = bestNewMasterDB.UID
} else {
log.Errorw("no eligible masters")
}
}
}
Expand Down
Loading