From 2313a8450e94c8b4de31e63da4ce844d2f3d2233 Mon Sep 17 00:00:00 2001 From: Dmytro Zghoba Date: Wed, 7 Aug 2024 10:39:07 +0200 Subject: [PATCH] [PBM-1341] retry shutdown on ConflictingOperationInProgress --- pbm/restore/physical.go | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/pbm/restore/physical.go b/pbm/restore/physical.go index ffb46f008..778af441c 100644 --- a/pbm/restore/physical.go +++ b/pbm/restore/physical.go @@ -53,6 +53,8 @@ const ( tryConnCount = 5 tryConnTimeout = 5 * time.Minute + + maxShutdownTriesOnStandaloneRecovery = 10 ) type files struct { @@ -1259,7 +1261,17 @@ func (r *PhysRestore) prepareData() error { } func shutdown(c *mongo.Client, dbpath string) error { - err := c.Database("admin").RunCommand(context.TODO(), bson.D{{"shutdown", 1}}).Err() + return shutdownImpl(c, dbpath, false) +} + +func forceShutdown(c *mongo.Client, dbpath string) error { + return shutdownImpl(c, dbpath, true) +} + +func shutdownImpl(c *mongo.Client, dbpath string, force bool) error { + res := c.Database("admin").RunCommand(context.TODO(), + bson.D{{"shutdown", 1}, {"force", force}}) + err := res.Err() if err != nil && !strings.Contains(err.Error(), "socket was unexpectedly closed") { return err } @@ -1285,12 +1297,24 @@ func (r *PhysRestore) recoverStandalone() error { return errors.Wrap(err, "connect to mongo") } - err = shutdown(c, r.dbpath) - if err != nil { - return errors.Wrap(err, "shutdown mongo") + for i := 0; i != maxShutdownTriesOnStandaloneRecovery; i++ { + err = shutdown(c, r.dbpath) + if err == nil { + return nil // OK + } + + if strings.Contains(err.Error(), "ConflictingOperationInProgress") { + r.log.Warning("retry shutdown in 5 seconds. reason: %v", err) + time.Sleep(5 * time.Second) + continue + } + + return errors.Wrap(err, "shutdown mongo") // unexpected } - return nil + r.log.Debug("force shutdown") + err = forceShutdown(c, r.dbpath) + return errors.Wrap(err, "force shutdown mongo") } func (r *PhysRestore) replayOplog(