diff --git a/.gitignore b/.gitignore index 84008f1a65..b079049506 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /bin cover /.dapper +/.dapper.tmp /dist *.swp rancher-machine diff --git a/drivers/vmwarevsphere/vsphere.go b/drivers/vmwarevsphere/vsphere.go index af363c71de..8668d0049d 100644 --- a/drivers/vmwarevsphere/vsphere.go +++ b/drivers/vmwarevsphere/vsphere.go @@ -481,28 +481,73 @@ func (d *Driver) Remove() error { } if machineState == state.Running { if d.GracefulShutdownTimeout > 0 { - // graceful shutdown the machine log.Infof("Gracefully shutting down VM %s (ID: %s)", d.MachineName, d.MachineId) - if err := d.Stop(); err != nil { - return err - } - // wait for machine to stop - elapsed := 0 - for elapsed <= d.GracefulShutdownTimeout { + + start := time.Now() + deadline := start.Add(time.Duration(d.GracefulShutdownTimeout) * time.Second) + sleep := time.Duration(gracefulShutdownSleep) * time.Second + + gracefulShutdownInitiated := false + + for time.Now().Before(deadline) { + // If we haven't initiated shutdown yet, keep trying. + if !gracefulShutdownInitiated { + if err := d.Stop(); err != nil { + log.Debugf( + "Failed to initiate graceful shutdown for VM %s (ID: %s): %v. Elapsed time: %d second(s)", + d.MachineName, d.MachineId, err, int(time.Since(start).Seconds()), + ) + } else { + gracefulShutdownInitiated = true + log.Infof( + "Graceful shutdown initiated for VM %s (ID: %s). Elapsed time: %d second(s)", + d.MachineName, d.MachineId, int(time.Since(start).Seconds()), + ) + } + } + + // Always check state so we can exit early if it stopped (even if Stop() never succeeded). if machineState, err = d.GetState(); err != nil { return err } if machineState == state.Stopped { + if gracefulShutdownInitiated { + log.Infof( + "VM %s (ID: %s) stopped gracefully after %d second(s)", + d.MachineName, d.MachineId, int(time.Since(start).Seconds()), + ) + } break } - elapsed += gracefulShutdownSleep - time.Sleep(gracefulShutdownSleep * time.Second) - log.Debugf("Waiting for VM %s (ID: %s) to stop. Elapsed time: %d second(s)", d.MachineName, d.MachineId, elapsed) + + log.Debugf( + "Waiting for VM %s (ID: %s) to stop. Elapsed time: %d second(s)", + d.MachineName, d.MachineId, int(time.Since(start).Seconds()), + ) + + // Sleep, but don't go past the deadline. + if rem := time.Until(deadline); rem < sleep { + time.Sleep(rem) + } else { + time.Sleep(sleep) + } } - if elapsed > d.GracefulShutdownTimeout { - log.Infof("Timeout for graceful shutdown of VM %s (ID: %s)", d.MachineName, d.MachineId) + + if machineState != state.Stopped { + if !gracefulShutdownInitiated { + log.Infof( + "Failed to initiate graceful shutdown for VM %s (ID: %s) within timeout of %d second(s)", + d.MachineName, d.MachineId, d.GracefulShutdownTimeout, + ) + } else { + log.Infof( + "Timeout for graceful shutdown of VM %s (ID: %s) after %d second(s)", + d.MachineName, d.MachineId, int(time.Since(start).Seconds()), + ) + } } } + if machineState, err = d.GetState(); err != nil { return err }