portainer/pkg/retry/retry.go

package retry

import (
	"math/rand/v2"
	"time"

	"github.com/rs/zerolog/log"
)

type Settings struct {
	MaxRetries    int
	RetryTimeFunc func(failures int) time.Duration
}

var Default = Settings{
	MaxRetries:    3,
	RetryTimeFunc: RetryTime,
}

// RetryWithWarnings executes a function with retries and backoff, logging warnings on failures
func RetryWithWarnings[T any](operation string, settings Settings, fn func() (T, error)) (T, error) {
	timeStart := time.Now()

	var result T
	var err error

	for attempt := 1; attempt <= settings.MaxRetries; attempt++ {
		result, err = fn()
		if err == nil {
			if attempt > 1 {
				log.Info().
					Str("operation", operation).
					Int("attempt", attempt).
					Str("duration", time.Since(timeStart).String()).
					Msg("operation succeeded after retry")
			}
			return result, nil
		}

		if attempt < settings.MaxRetries {
			retryDelay := settings.RetryTimeFunc(attempt)
			log.Warn().
				Err(err).
				Str("operation", operation).
				Int("attempt", attempt).
				Int("max_retries", settings.MaxRetries).
				Str("retry_delay", retryDelay.String()). // the string version of the duration reads better in logs
				Str("duration", time.Since(timeStart).String()).
				Msg("operation failed, retrying")
			time.Sleep(retryDelay)
		}
	}

	log.Error().
		Err(err).
		Str("operation", operation).
		Int("attempts", settings.MaxRetries).
		Str("duration", time.Since(timeStart).String()).
		Msg("operation failed after all retries")

	return result, err
}

func RetryTime(failures int) time.Duration {
	var baseTime time.Duration

	switch failures {
	case 0, 1:
		baseTime = 5 * time.Second
	case 2:
		baseTime = 30 * time.Second
	default:
		baseTime = 90 * time.Second
	}

	// plus random jitter
	jitter := time.Duration(rand.Int64N(int64(10 * time.Second))) // 10s as nanoseconds overflows int on 32bit systems

	return baseTime + jitter
}