재시도 패턴

using System;
using System.Collections.Generic;

// 앞의 두 호출은 일시적 오류를 내고 세 번째 호출은 성공하는 가짜 재고 서비스
var calls = 0;
Outcome GetStock()
{
    calls++;
    return calls <= 2 ? new Outcome(null, ErrorKind.Transient) : new Outcome("stock=7", null);
}

// 어떤 실패를, 몇 번까지, 얼마나 기다렸다가 다시 시도할지를 데이터로 둔다.
var policy = new RetryPolicy(
    MaxAttempts: 4,
    RetryOn: kind => kind == ErrorKind.Transient,
    Backoff: failedAttempt => TimeSpan.FromMilliseconds(Math.Min(1000, 100 * Math.Pow(2, failedAttempt - 1))));

// 실제로 기다리지 않고 요청받은 대기 시간만 기록한다.
var waited = new List<double>();
var result = Retrier.Run(policy, GetStock, delay => waited.Add(delay.TotalMilliseconds));

foreach (var line in result.Log) Console.WriteLine(line);
Console.WriteLine($"value={result.Value}, attempts={result.Attempts}, delays=[{string.Join(", ", waited)}] ms");

enum ErrorKind { Transient, Permanent }
sealed record Outcome(string? Value, ErrorKind? Error);
sealed record RetryPolicy(int MaxAttempts, Func<ErrorKind, bool> RetryOn, Func<int, TimeSpan> Backoff);
sealed record RetryResult(string? Value, ErrorKind? Error, int Attempts, List<string> Log);

static class Retrier
{
    // 시도마다 결과와 판단을 기록하고, 성공·재시도 불가·횟수 소진 중 하나에서 멈춘다.
    public static RetryResult Run(RetryPolicy policy, Func<Outcome> operation, Action<TimeSpan> delay)
    {
        var log = new List<string>();
        for (var attempt = 1; ; attempt++)
        {
            var outcome = operation();
            if (outcome.Error is not { } error)
            {
                log.Add($"attempt {attempt}: ok");
                return new RetryResult(outcome.Value, null, attempt, log);
            }
            if (!policy.RetryOn(error))
            {
                log.Add($"attempt {attempt}: {error}, not retryable");
                return new RetryResult(null, error, attempt, log);
            }
            if (attempt >= policy.MaxAttempts)
            {
                log.Add($"attempt {attempt}: {error}, attempts exhausted");
                return new RetryResult(null, error, attempt, log);
            }
            var wait = policy.Backoff(attempt);
            log.Add($"attempt {attempt}: {error}, retry in {wait.TotalMilliseconds} ms");
            delay(wait);
        }
    }
}

attempt 1: Transient, retry in 100 ms
attempt 2: Transient, retry in 200 ms
attempt 3: ok
value=stock=7, attempts=3, delays=[100, 200] ms

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

// 시드를 고정한 난수로 지터를 만들어 실행할 때마다 같은 대기 시간이 나오게 한다.
const int Seed = 42;
var rng = new Random(Seed);
var policy = new RetryPolicy(
    MaxAttempts: 4,
    RetryOn: f => f.Kind is ErrorKind.Transient or ErrorKind.Throttled,
    Backoff: failedAttempt => FullJitter(rng, failedAttempt, baseMs: 100, capMs: 2000));

// 운영 코드라면 Task.Delay(delay, ct)를 넘긴다. 그때 대기 중 취소는 예외로 나온다. 여기서는 대기 시간만 받고 바로 끝낸다.
Task NoWait(TimeSpan delay, CancellationToken ct) => Task.CompletedTask;

Console.WriteLine($"jitter seed {Seed}");
await Show("transient twice", Scripted("GetStock",
    Outcome.Fail(ErrorKind.Transient, "503"), Outcome.Fail(ErrorKind.Transient, "503"), Outcome.Ok("stock=7")));
await Show("permanent error", Scripted("GetStock", Outcome.Fail(ErrorKind.Permanent, "400")));
await Show("transient every time", Scripted("GetStock", Outcome.Fail(ErrorKind.Transient, "503")));
await Show("throttled with Retry-After", Scripted("GetStock",
    Outcome.Fail(ErrorKind.Throttled, "429", TimeSpan.FromMilliseconds(2000)), Outcome.Ok("stock=7")));

// 첫 대기 중에 호출자가 취소한 것처럼 만든다.
using var cts = new CancellationTokenSource();
Task CancelDuringWait(TimeSpan delay, CancellationToken ct) { cts.Cancel(); return Task.CompletedTask; }
await Show("cancelled while waiting", Scripted("GetStock", Outcome.Fail(ErrorKind.Transient, "503")), CancelDuringWait, cts.Token);

var server = new OrderServer();
await Show("order without key", new Operation("CreateOrder", Idempotent: false, _ => Task.FromResult(server.Create(null))));
await Show("order with key k-1", new Operation("CreateOrder", Idempotent: true, _ => Task.FromResult(server.Create("k-1"))));
Console.WriteLine($"orders stored: {server.Stored}");

async Task Show(string title, Operation operation,
    Func<TimeSpan, CancellationToken, Task>? delay = null, CancellationToken ct = default)
{
    var result = await Retrier.RunAsync(policy, operation, delay ?? NoWait, ct);
    Console.WriteLine($"== {title}");
    foreach (var a in result.Attempts) Console.WriteLine($"  #{a.Number} {a.Summary}");
    Console.WriteLine($"  -> {result.Stop}{(result.Value is null ? "" : " " + result.Value)}");
}

// 정해 둔 결과를 차례로 돌려주고, 끝나면 마지막 결과를 반복하는 가짜 의존성. 조회라서 멱등이다.
static Operation Scripted(string name, params Outcome[] script)
{
    var next = 0;
    return new Operation(name, Idempotent: true, _ => Task.FromResult(script[Math.Min(next++, script.Length - 1)]));
}

// full jitter: 0 이상 min(cap, base * 2^(n-1)) 미만에서 고르게 고른다.
static TimeSpan FullJitter(Random rng, int failedAttempt, int baseMs, int capMs)
{
    var ceiling = Math.Min(capMs, baseMs * Math.Pow(2, failedAttempt - 1));
    return TimeSpan.FromMilliseconds((int)(rng.NextDouble() * ceiling));
}

enum ErrorKind { Transient, Throttled, Permanent }
sealed record Failure(ErrorKind Kind, string Code, TimeSpan? RetryAfter);
sealed record Outcome(string? Value, Failure? Error)
{
    public static Outcome Ok(string value) => new(value, null);
    public static Outcome Fail(ErrorKind kind, string code, TimeSpan? retryAfter = null) => new(null, new Failure(kind, code, retryAfter));
}
sealed record Operation(string Name, bool Idempotent, Func<CancellationToken, Task<Outcome>> Run);
sealed record RetryPolicy(int MaxAttempts, Func<Failure, bool> RetryOn, Func<int, TimeSpan> Backoff);
sealed record AttemptRecord(int Number, string Summary);
sealed record RetryResult(string Stop, string? Value, Failure? Error, IReadOnlyList<AttemptRecord> Attempts);

static class Retrier
{
    public static async Task<RetryResult> RunAsync(
        RetryPolicy policy, Operation operation,
        Func<TimeSpan, CancellationToken, Task> delay, CancellationToken ct)
    {
        var attempts = new List<AttemptRecord>();
        // 멱등이 아닌 작업은 한 번도 실행하지 않고 거절한다. 다시 보내면 효과가 겹칠 수 있다.
        if (!operation.Idempotent && policy.MaxAttempts > 1)
            return new RetryResult($"refused: {operation.Name} is not idempotent", null, null, attempts);

        for (var n = 1; ; n++)
        {
            // 다음 시도 전에 취소를 확인하고, 취소되면 기록을 남기고 멈춘다.
            if (ct.IsCancellationRequested)
                return new RetryResult("stopped: cancelled", null, null, attempts);
            var outcome = await operation.Run(ct);
            if (outcome.Error is not { } error)
            {
                attempts.Add(new AttemptRecord(n, "ok"));
                return new RetryResult("succeeded", outcome.Value, null, attempts);
            }
            var failed = $"{error.Kind} {error.Code}";
            if (!policy.RetryOn(error))
            {
                attempts.Add(new AttemptRecord(n, failed + ", not retryable"));
                return new RetryResult("stopped: not retryable", null, error, attempts);
            }
            if (n >= policy.MaxAttempts)
            {
                attempts.Add(new AttemptRecord(n, failed + ", attempts exhausted"));
                return new RetryResult("stopped: attempts exhausted", null, error, attempts);
            }
            // 서버가 Retry-After로 대기 시간을 알려 주면 그 값을 따른다.
            var (wait, source) = error.RetryAfter is { } hint ? (hint, "Retry-After") : (policy.Backoff(n), "backoff");
            attempts.Add(new AttemptRecord(n, $"{failed}, wait {wait.TotalMilliseconds} ms ({source})"));
            await delay(wait, ct);
        }
    }
}

// 같은 멱등성 키로 다시 온 요청에는 처음 만든 주문을 돌려준다.
sealed class OrderServer
{
    private readonly Dictionary<string, string> byKey = new();
    private bool dropNextResponse = true;
    public int Stored { get; private set; }

    public Outcome Create(string? idempotencyKey)
    {
        if (idempotencyKey is not null && byKey.TryGetValue(idempotencyKey, out var existing))
            return Outcome.Ok(existing);
        Stored++;
        var id = $"order-{Stored}";
        if (idempotencyKey is not null) byKey[idempotencyKey] = id;
        // 첫 요청은 저장까지 마친 뒤 응답이 끊긴 것처럼 실패를 돌려준다.
        if (!dropNextResponse) return Outcome.Ok(id);
        dropNextResponse = false;
        return Outcome.Fail(ErrorKind.Transient, "connection reset");
    }
}

jitter seed 42
== transient twice
  #1 Transient 503, wait 66 ms (backoff)
  #2 Transient 503, wait 28 ms (backoff)
  #3 ok
  -> succeeded stock=7
== permanent error
  #1 Permanent 400, not retryable
  -> stopped: not retryable
== transient every time
  #1 Transient 503, wait 12 ms (backoff)
  #2 Transient 503, wait 104 ms (backoff)
  #3 Transient 503, wait 67 ms (backoff)
  #4 Transient 503, attempts exhausted
  -> stopped: attempts exhausted
== throttled with Retry-After
  #1 Throttled 429, wait 2000 ms (Retry-After)
  #2 ok
  -> succeeded stock=7
== cancelled while waiting
  #1 Transient 503, wait 26 ms (backoff)
  -> stopped: cancelled
== order without key
  -> refused: CreateOrder is not idempotent
== order with key k-1
  #1 Transient connection reset, wait 72 ms (backoff)
  #2 ok
  -> succeeded order-1
orders stored: 1

type ErrorKind = "Transient" | "Throttled" | "Permanent";
type Failure = { kind: ErrorKind; code: string; retryAfterMs?: number };
type Outcome = { ok: true; value: string } | { ok: false; error: Failure };
type Operation = { name: string; idempotent: boolean; run: (signal: AbortSignal) => Promise<Outcome> };
type RetryPolicy = {
  maxAttempts: number;
  retryOn: (failure: Failure) => boolean;
  backoffMs: (failedAttempt: number) => number;
};
type RetryResult = { stop: string; value?: string; attempts: string[] };

const ok = (value: string): Outcome => ({ ok: true, value });
const fail = (kind: ErrorKind, code: string, retryAfterMs?: number): Outcome =>
  ({ ok: false, error: { kind, code, retryAfterMs } });

// 시드를 고정한 작은 난수 생성기(mulberry32). Math.random은 시드를 정할 수 없다.
function seeded(seed: number): () => number {
  let a = seed >>> 0;
  return () => {
    a = (a + 0x6d2b79f5) >>> 0;
    let t = a;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

// full jitter: 0 이상 min(cap, base * 2^(n-1)) 미만에서 고르게 고른다.
function fullJitter(random: () => number, failedAttempt: number, baseMs: number, capMs: number): number {
  return Math.floor(random() * Math.min(capMs, baseMs * 2 ** (failedAttempt - 1)));
}

async function runWithRetry(
  policy: RetryPolicy, op: Operation,
  delay: (ms: number, signal: AbortSignal) => Promise<void>, signal: AbortSignal,
): Promise<RetryResult> {
  const attempts: string[] = [];
  // 멱등이 아닌 작업은 한 번도 실행하지 않고 거절한다.
  if (!op.idempotent && policy.maxAttempts > 1) return { stop: `refused: ${op.name} is not idempotent`, attempts };
  for (let n = 1; ; n++) {
    // 다음 시도 전에 취소를 확인하고, 취소되면 기록을 남기고 멈춘다.
    if (signal.aborted) return { stop: "stopped: cancelled", attempts };
    const outcome = await op.run(signal);
    if (outcome.ok) {
      attempts.push(`#${n} ok`);
      return { stop: "succeeded", value: outcome.value, attempts };
    }
    const failed = `#${n} ${outcome.error.kind} ${outcome.error.code}`;
    if (!policy.retryOn(outcome.error)) {
      attempts.push(`${failed}, not retryable`);
      return { stop: "stopped: not retryable", attempts };
    }
    if (n >= policy.maxAttempts) {
      attempts.push(`${failed}, attempts exhausted`);
      return { stop: "stopped: attempts exhausted", attempts };
    }
    // 서버가 Retry-After로 대기 시간을 알려 주면 그 값을 따른다.
    const hint = outcome.error.retryAfterMs;
    const [waitMs, source] = hint !== undefined ? [hint, "Retry-After"] : [policy.backoffMs(n), "backoff"];
    attempts.push(`${failed}, wait ${waitMs} ms (${source})`);
    await delay(waitMs, signal);
  }
}

// 정해 둔 결과를 차례로 돌려주고, 끝나면 마지막 결과를 반복하는 가짜 의존성. 조회라서 멱등이다.
function scripted(name: string, ...script: Outcome[]): Operation {
  let next = 0;
  return { name, idempotent: true, run: async () => script[Math.min(next++, script.length - 1)] };
}

const seed = 42;
const random = seeded(seed);
const policy: RetryPolicy = {
  maxAttempts: 4,
  retryOn: (f) => f.kind === "Transient" || f.kind === "Throttled",
  backoffMs: (n) => fullJitter(random, n, 100, 2000),
};
// 운영 코드라면 signal을 존중하는 실제 타이머를 넘긴다. 여기서는 기다리지 않는다.
const noWait = async (_ms: number, _signal: AbortSignal): Promise<void> => {};

async function show(
  title: string, op: Operation,
  delay: (ms: number, signal: AbortSignal) => Promise<void> = noWait, signal = new AbortController().signal,
): Promise<void> {
  const result = await runWithRetry(policy, op, delay, signal);
  console.log(`== ${title}`);
  for (const line of result.attempts) console.log(`  ${line}`);
  console.log(`  -> ${result.stop}${result.value === undefined ? "" : " " + result.value}`);
}

async function main(): Promise<void> {
  console.log(`jitter seed ${seed}`);
  await show("transient twice", scripted("GetStock", fail("Transient", "503"), fail("Transient", "503"), ok("stock=7")));
  await show("permanent error", scripted("GetStock", fail("Permanent", "400")));
  await show("transient every time", scripted("GetStock", fail("Transient", "503")));
  await show("throttled with Retry-After", scripted("GetStock", fail("Throttled", "429", 2000), ok("stock=7")));
  // 첫 대기 중에 호출자가 취소한 것처럼 만든다.
  const controller = new AbortController();
  const cancelDuringWait = async (_ms: number, _signal: AbortSignal): Promise<void> => controller.abort();
  await show("cancelled while waiting", scripted("GetStock", fail("Transient", "503")), cancelDuringWait, controller.signal);
  await show("order without key", { name: "CreateOrder", idempotent: false, run: async () => ok("order-1") });
}

void main();

jitter seed 42
== transient twice
  #1 Transient 503, wait 60 ms (backoff)
  #2 Transient 503, wait 89 ms (backoff)
  #3 ok
  -> succeeded stock=7
== permanent error
  #1 Permanent 400, not retryable
  -> stopped: not retryable
== transient every time
  #1 Transient 503, wait 85 ms (backoff)
  #2 Transient 503, wait 133 ms (backoff)
  #3 Transient 503, wait 69 ms (backoff)
  #4 Transient 503, attempts exhausted
  -> stopped: attempts exhausted
== throttled with Retry-After
  #1 Throttled 429, wait 2000 ms (Retry-After)
  #2 ok
  -> succeeded stock=7
== cancelled while waiting
  #1 Transient 503, wait 52 ms (backoff)
  -> stopped: cancelled
== order without key
  -> refused: CreateOrder is not idempotent