feat(client): implement exponential backoff retry for network errors
This commit is contained in:
@@ -3,6 +3,7 @@ ZARINHUB_API_PASSWORD=
|
||||
ZARINHUB_AUTH_ENDPOINT=https://zarin-hub.com/api/v5/Authentication/GetToken
|
||||
ZARINHUB_ENDPOINT=https://zarin-hub.com/api/v5/KYC/CardToIban
|
||||
ZARINHUB_TIMEOUT=5s
|
||||
ZARINHUB_RETRIES=2
|
||||
|
||||
DATABASE_PATH=./card-to-iban.db
|
||||
SERVER_ADDR=127.0.0.1:8080
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -15,6 +16,7 @@ type config struct {
|
||||
ZarinHubUsername string
|
||||
ZarinHubPassword string
|
||||
ZarinHubTimeout time.Duration
|
||||
ZarinHubRetries int
|
||||
DatabasePath string
|
||||
}
|
||||
|
||||
@@ -44,6 +46,18 @@ func readConfig(getenv func(string) string) (config, error) {
|
||||
)
|
||||
}
|
||||
|
||||
retriesText := valueOrDefault(
|
||||
getenv("ZARINHUB_RETRIES"),
|
||||
"2",
|
||||
)
|
||||
|
||||
retries, err := strconv.Atoi(retriesText)
|
||||
if err != nil || retries < 0 {
|
||||
return config{}, fmt.Errorf(
|
||||
"ZARINHUB_RETRIES must be a non-negative integer",
|
||||
)
|
||||
}
|
||||
|
||||
return config{
|
||||
ServerAddr: valueOrDefault(
|
||||
getenv("SERVER_ADDR"),
|
||||
@@ -64,6 +78,7 @@ func readConfig(getenv func(string) string) (config, error) {
|
||||
ZarinHubUsername: username,
|
||||
ZarinHubPassword: password,
|
||||
ZarinHubTimeout: timeout,
|
||||
ZarinHubRetries: retries,
|
||||
DatabasePath: valueOrDefault(
|
||||
getenv("DATABASE_PATH"),
|
||||
"./card-to-iban.db",
|
||||
|
||||
@@ -31,6 +31,9 @@ func TestReadConfigUsesDefaults(t *testing.T) {
|
||||
if config.ZarinHubTimeout != 5*time.Second {
|
||||
t.Errorf("ZarinHubTimeout = %v", config.ZarinHubTimeout)
|
||||
}
|
||||
if config.ZarinHubRetries != 2 {
|
||||
t.Errorf("ZarinHubRetries = %d, want 2", config.ZarinHubRetries)
|
||||
}
|
||||
if config.DatabasePath != "./card-to-iban.db" {
|
||||
t.Errorf("DatabasePath = %q", config.DatabasePath)
|
||||
}
|
||||
@@ -70,3 +73,22 @@ func TestReadConfigRejectsInvalidTimeout(t *testing.T) {
|
||||
t.Fatalf("readConfig() error = %v, want timeout error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadConfigRejectsInvalidRetries(t *testing.T) {
|
||||
testCases := []string{"-1", "invalid", "2.5"}
|
||||
|
||||
for _, tc := range testCases {
|
||||
values := map[string]string{
|
||||
"ZARINHUB_API_USERNAME": "test-username",
|
||||
"ZARINHUB_API_PASSWORD": "test-password",
|
||||
"ZARINHUB_RETRIES": tc,
|
||||
}
|
||||
|
||||
_, err := readConfig(func(key string) string {
|
||||
return values[key]
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "ZARINHUB_RETRIES") {
|
||||
t.Errorf("retries=%q: want error mentioning ZARINHUB_RETRIES, got %v", tc, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,6 +79,7 @@ func run(logger *slog.Logger) error {
|
||||
httpClient,
|
||||
config.ZarinHubEndpoint,
|
||||
tokenManager,
|
||||
config.ZarinHubRetries,
|
||||
)
|
||||
cardService := service.NewCardService(zarinHubClient)
|
||||
auditRepository := repository.NewAuditRepository(db)
|
||||
|
||||
@@ -9,16 +9,24 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"card-to-iban/internal/service"
|
||||
)
|
||||
|
||||
const maxResponseBodySize = 1 << 20
|
||||
const (
|
||||
maxResponseBodySize = 1 << 20
|
||||
|
||||
retryBaseDelay = 200 * time.Millisecond
|
||||
retryMaxDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
endpointURL string
|
||||
tokenProvider TokenProvider
|
||||
maxRetries int
|
||||
retryWait func(ctx context.Context, delay time.Duration) bool
|
||||
}
|
||||
|
||||
var _ service.CardToIBANClient = (*Client)(nil)
|
||||
@@ -32,17 +40,114 @@ func NewClient(
|
||||
httpClient *http.Client,
|
||||
endpointURL string,
|
||||
tokenProvider TokenProvider,
|
||||
maxRetries int,
|
||||
) *Client {
|
||||
if maxRetries < 0 {
|
||||
maxRetries = 0
|
||||
}
|
||||
|
||||
return &Client{
|
||||
httpClient: httpClient,
|
||||
endpointURL: endpointURL,
|
||||
tokenProvider: tokenProvider,
|
||||
maxRetries: maxRetries,
|
||||
retryWait: waitForRetry,
|
||||
}
|
||||
}
|
||||
|
||||
func isRetryable(err error) bool {
|
||||
return errors.Is(err, ErrTimeout) ||
|
||||
errors.Is(err, ErrUnavailable)
|
||||
}
|
||||
|
||||
func retryDelay(attempt int) time.Duration {
|
||||
delay := retryBaseDelay
|
||||
|
||||
for i := 0; i < attempt; i++ {
|
||||
delay *= 2
|
||||
|
||||
if delay >= retryMaxDelay {
|
||||
return retryMaxDelay
|
||||
}
|
||||
}
|
||||
|
||||
return delay
|
||||
}
|
||||
|
||||
func waitForRetry(
|
||||
ctx context.Context,
|
||||
delay time.Duration,
|
||||
) bool {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) Convert(
|
||||
ctx context.Context,
|
||||
cardNumber string,
|
||||
) (service.ConversionResult, error) {
|
||||
var (
|
||||
result service.ConversionResult
|
||||
err error
|
||||
)
|
||||
|
||||
waitFn := c.retryWait
|
||||
if waitFn == nil {
|
||||
waitFn = waitForRetry
|
||||
}
|
||||
|
||||
for attempt := 0; attempt <= c.maxRetries; attempt++ {
|
||||
result, err = c.convertOnce(
|
||||
ctx,
|
||||
cardNumber,
|
||||
)
|
||||
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Don't retry permanent/non-transient errors.
|
||||
if !isRetryable(err) {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// We already used all configured retries.
|
||||
if attempt == c.maxRetries {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// If the caller canceled the request, stop immediately.
|
||||
if ctx.Err() != nil {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return result, fmt.Errorf("zarinhub request canceled: %w", ctx.Err())
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
delay := retryDelay(attempt)
|
||||
|
||||
if !waitFn(ctx, delay) {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return result, fmt.Errorf("zarinhub request canceled: %w", ctx.Err())
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (c *Client) convertOnce(
|
||||
ctx context.Context,
|
||||
cardNumber string,
|
||||
) (service.ConversionResult, error) {
|
||||
accessToken, err := c.tokenProvider.AccessToken(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -98,6 +98,7 @@ func TestClientConvertSuccess(t *testing.T) {
|
||||
},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{token}},
|
||||
0,
|
||||
)
|
||||
|
||||
result, err := client.Convert(
|
||||
@@ -172,6 +173,7 @@ func TestClientReturnsRejectedForApplicationFailure(
|
||||
},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
0,
|
||||
)
|
||||
|
||||
result, err := client.Convert(
|
||||
@@ -264,6 +266,7 @@ func TestClientMapsHTTPStatusErrors(t *testing.T) {
|
||||
},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
0,
|
||||
)
|
||||
|
||||
result, err := client.Convert(
|
||||
@@ -322,6 +325,7 @@ func TestClientReturnsInvalidResponseForMalformedJSON(
|
||||
},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
0,
|
||||
)
|
||||
|
||||
result, err := client.Convert(
|
||||
@@ -367,6 +371,7 @@ func TestClientReturnsTimeout(t *testing.T) {
|
||||
},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
0,
|
||||
)
|
||||
|
||||
_, err := client.Convert(
|
||||
@@ -403,6 +408,7 @@ func TestClientRejectsOversizedResponse(t *testing.T) {
|
||||
&http.Client{Timeout: time.Second},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
0,
|
||||
)
|
||||
|
||||
_, err := client.Convert(
|
||||
@@ -442,7 +448,7 @@ func TestClientRefreshesAndRetriesUnauthorizedRequest(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
tokens := &fakeTokenProvider{tokens: []string{"expired-token", "fresh-token"}}
|
||||
client := NewClient(server.Client(), server.URL, tokens)
|
||||
client := NewClient(server.Client(), server.URL, tokens, 0)
|
||||
|
||||
result, err := client.Convert(context.Background(), "6037997512345678")
|
||||
if err != nil {
|
||||
@@ -478,3 +484,186 @@ func (p *fakeTokenProvider) AccessToken(context.Context) (string, error) {
|
||||
func (p *fakeTokenProvider) Invalidate(token string) {
|
||||
p.invalidated = append(p.invalidated, token)
|
||||
}
|
||||
|
||||
func TestRetryDelay(t *testing.T) {
|
||||
tests := []struct {
|
||||
attempt int
|
||||
want time.Duration
|
||||
}{
|
||||
{attempt: -1, want: 200 * time.Millisecond},
|
||||
{attempt: 0, want: 200 * time.Millisecond},
|
||||
{attempt: 1, want: 400 * time.Millisecond},
|
||||
{attempt: 2, want: 800 * time.Millisecond},
|
||||
{attempt: 3, want: 1600 * time.Millisecond},
|
||||
{attempt: 4, want: 2 * time.Second},
|
||||
{attempt: 10, want: 2 * time.Second},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := retryDelay(tt.attempt)
|
||||
if got != tt.want {
|
||||
t.Errorf("retryDelay(%d) = %v, want %v", tt.attempt, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientSanitizesNegativeRetries(t *testing.T) {
|
||||
client := NewClient(nil, "", nil, -1)
|
||||
if client.maxRetries != 0 {
|
||||
t.Errorf("maxRetries = %d, want 0", client.maxRetries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientRetriesOnTransientErrorsUntilSuccess(t *testing.T) {
|
||||
const expectedIBAN = "IR820540102680020817909002"
|
||||
requestCount := 0
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
if requestCount == 1 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
fmt.Fprint(w, `{"error":"temporary outage"}`)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{
|
||||
"data":{"iban":%q},
|
||||
"meta":{"isSuccess":true,"errors":[]}
|
||||
}`, expectedIBAN)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
var recordedDelays []time.Duration
|
||||
client := NewClient(
|
||||
&http.Client{Timeout: time.Second},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
2,
|
||||
)
|
||||
client.retryWait = func(ctx context.Context, delay time.Duration) bool {
|
||||
recordedDelays = append(recordedDelays, delay)
|
||||
return true
|
||||
}
|
||||
|
||||
result, err := client.Convert(context.Background(), "6037997512345678")
|
||||
if err != nil {
|
||||
t.Fatalf("Convert() unexpected error = %v", err)
|
||||
}
|
||||
if result.IBAN != expectedIBAN {
|
||||
t.Errorf("IBAN = %q, want %q", result.IBAN, expectedIBAN)
|
||||
}
|
||||
if requestCount != 2 {
|
||||
t.Errorf("request count = %d, want 2", requestCount)
|
||||
}
|
||||
if len(recordedDelays) != 1 {
|
||||
t.Fatalf("len(recordedDelays) = %d, want 1", len(recordedDelays))
|
||||
}
|
||||
if recordedDelays[0] != 200*time.Millisecond {
|
||||
t.Errorf("delay[0] = %v, want 200ms", recordedDelays[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientRetriesExhausted(t *testing.T) {
|
||||
requestCount := 0
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
w.WriteHeader(http.StatusGatewayTimeout)
|
||||
fmt.Fprint(w, `{"error":"gateway timeout"}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
var recordedDelays []time.Duration
|
||||
client := NewClient(
|
||||
&http.Client{Timeout: time.Second},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
2,
|
||||
)
|
||||
client.retryWait = func(ctx context.Context, delay time.Duration) bool {
|
||||
recordedDelays = append(recordedDelays, delay)
|
||||
return true
|
||||
}
|
||||
|
||||
_, err := client.Convert(context.Background(), "6037997512345678")
|
||||
if !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("error = %v, want ErrUnavailable", err)
|
||||
}
|
||||
if requestCount != 3 {
|
||||
t.Errorf("request count = %d, want 3", requestCount)
|
||||
}
|
||||
if len(recordedDelays) != 2 {
|
||||
t.Fatalf("len(recordedDelays) = %d, want 2", len(recordedDelays))
|
||||
}
|
||||
if recordedDelays[0] != 200*time.Millisecond {
|
||||
t.Errorf("delay[0] = %v, want 200ms", recordedDelays[0])
|
||||
}
|
||||
if recordedDelays[1] != 400*time.Millisecond {
|
||||
t.Errorf("delay[1] = %v, want 400ms", recordedDelays[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientDoesNotRetryNonTransientErrors(t *testing.T) {
|
||||
requestCount := 0
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprint(w, `{"error":"invalid card format"}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(
|
||||
&http.Client{Timeout: time.Second},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
2,
|
||||
)
|
||||
client.retryWait = func(ctx context.Context, delay time.Duration) bool {
|
||||
t.Error("retryWait should not be called for non-transient errors")
|
||||
return true
|
||||
}
|
||||
|
||||
_, err := client.Convert(context.Background(), "6037997512345678")
|
||||
if !errors.Is(err, ErrRejected) {
|
||||
t.Fatalf("error = %v, want ErrRejected", err)
|
||||
}
|
||||
if requestCount != 1 {
|
||||
t.Errorf("request count = %d, want 1", requestCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientStopsRetryOnContextCancellation(t *testing.T) {
|
||||
requestCount := 0
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
client := NewClient(
|
||||
&http.Client{Timeout: time.Second},
|
||||
server.URL,
|
||||
&fakeTokenProvider{tokens: []string{"test-token"}},
|
||||
2,
|
||||
)
|
||||
client.retryWait = func(c context.Context, delay time.Duration) bool {
|
||||
cancel()
|
||||
return waitForRetry(ctx, delay)
|
||||
}
|
||||
|
||||
_, err := client.Convert(ctx, "6037997512345678")
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("error = %v, want context.Canceled", err)
|
||||
}
|
||||
if requestCount != 1 {
|
||||
t.Errorf("request count = %d, want 1", requestCount)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user