8000 Kill etcd process exactly before taking a final snapshot by plkokanov · Pull Request #478 · gardener/etcd-backup-restore · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Kill etcd process exactly before taking a final snapshot #478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 12, 2022
Merged

Kill etcd process exactly before taking a final snapshot #478

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 17 additions & 14 deletions pkg/server/backuprestoreserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,8 +252,9 @@ func (b *BackupRestoreServer) runServer(ctx context.Context, restor 8000 eOpts *brtype
// for the case when backup-restore becomes leading sidecar.
func (b *BackupRestoreServer) runEtcdProbeLoopWithSnapshotter(ctx context.Context, handler *HTTPHandler, ssr *snapshotter.Snapshotter, ss brtypes.SnapStore, ssrStopCh chan struct{}, ackCh chan struct{}) {
var (
err error
initialDeltaSnapshotTaken bool
err error
initialDeltaSnapshotTaken bool
killEtcdBeforeTakingFinalSnapshot bool
)

for {
Expand Down Expand Up @@ -292,7 +293,16 @@ func (b *BackupRestoreServer) runEtcdProbeLoopWithSnapshotter(ctx context.Contex
b.logger.Errorf("ownerChecker check fails: %v", err)
} else if !result {
handler.SetStatus(http.StatusServiceUnavailable)

// If owner check was previously successful, but is now failed, kill the etcd process before attempting
// to take a final snapshot to ensure that any open connections from kube-apiserver are terminated
if killEtcdBeforeTakingFinalSnapshot {
if _, err := b.etcdProcessKiller.Kill(ctx); err != nil {
b.logger.Errorf("Could not kill etcd process: %v", err)
// only attempt to take final snapshot if process was successfully killed.
continue
}
killEtcdBeforeTakingFinalSnapshot = false
}
// If the previous full snapshot doesn't exist or is not marked as final, take a final full snapshot
if ssr.PrevFullSnapshot == nil || !ssr.PrevFullSnapshot.IsFinal {
b.logger.Infof("Taking final full snapshot...")
Expand Down Expand Up @@ -321,6 +331,10 @@ func (b *BackupRestoreServer) runEtcdProbeLoopWithSnapshotter(ctx context.Contex

continue
}

// At this point owner check was successful, so set killEtcdBeforeTakingFinalSnapshot to true.
// The etcd process will be killed before taking a final snapshot if the owner check fails.
killEtcdBeforeTakingFinalSnapshot = true
}

// set server's healthz endpoint status to OK so that
Expand Down Expand Up @@ -442,17 +456,6 @@ func (b *BackupRestoreServer) runEtcdProbeLoopWithSnapshotter(ctx context.Contex
if b.ownerChecker != nil {
// Stop owner check watchdog
ownerCheckWatchdog.Stop()

// If the owner check fails or returns false, kill the etcd process
// to ensure that any open connections from kube-apiserver are terminated
result, err := b.ownerChecker.Check(ctx)
if err != nil {
b.logger.Errorf("ownerChecker check fails: %v", err)
} else if !result {
if _, err := b.etcdProcessKiller.Kill(ctx); err != nil {
b.logger.Errorf("Could not kill etcd process: %v", err)
}
}
}

} else {
Expand Down
0