pipeline/packages/tts/src/retry.ts

import { setTimeout as sleep } from "node:timers/promises";

const DEFAULT_RETRIES = 3;
const DEFAULT_BASE_DELAY_MS = 800;
const DEFAULT_MAX_DELAY_MS = 8000;

export interface RetryOptions {
  /** Retries after the initial attempt (default 3). 0 disables retry. */
  retries?: number;
  baseDelayMs?: number;
  maxDelayMs?: number;
}

/** Read `TTS_MAX_RETRIES` if set, else the provided default. */
function resolveRetries(fallback: number): number {
  const raw = process.env.TTS_MAX_RETRIES;
  if (raw === undefined || raw === "") return fallback;
  const n = Number(raw);
  return Number.isFinite(n) && n >= 0 ? Math.floor(n) : fallback;
}

/** HTTP statuses worth retrying: rate limiting + server errors. */
export function isRetriableStatus(status: number): boolean {
  return status === 429 || (status >= 500 && status <= 599);
}

function delayFor(attempt: number, base: number, max: number): number {
  const exp = Math.min(max, base * 2 ** attempt);
  return Math.floor(Math.random() * (exp - base + 1)) + base; // full jitter
}

/**
 * Drop-in replacement for `fetch` that retries transient failures — network
 * errors and HTTP 429/5xx — with exponential backoff + jitter. TTS gateways
 * intermittently return 502/503 (upstream DNS, rolling restarts); without
 * retry a single blip fails the whole render.
 *
 * Returns the final `Response` (ok, or a non-retriable error such as 4xx) so
 * the caller can read its body. When retries are exhausted on a retriable
 * status the last response is still returned (body intact) for diagnostics.
 */
export async function fetchWithRetry(
  input: string | URL,
  init: RequestInit,
  opts: RetryOptions = {}
): Promise<Response> {
  const retries = resolveRetries(opts.retries ?? DEFAULT_RETRIES);
  const baseDelay = opts.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
  const maxDelay = opts.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;

  for (let attempt = 0; attempt <= retries; attempt++) {
    let response: Response;
    try {
      response = await fetch(input, init);
    } catch (err) {
      if (attempt >= retries) throw err;
      console.warn(
        `[tts] request failed (${err instanceof Error ? err.message : err}), retrying ${attempt + 1}/${retries}`
      );
      await sleep(delayFor(attempt, baseDelay, maxDelay));
      continue;
    }

    if (response.ok || !isRetriableStatus(response.status)) {
      return response;
    }

    if (attempt >= retries) {
      return response; // exhausted — leave body intact for the caller to read
    }

    // Retriable status with retries left: drain the body to free the socket,
    // then back off.
    await response.arrayBuffer().catch(() => {});
    console.warn(`[tts] HTTP ${response.status}, retrying ${attempt + 1}/${retries}`);
    await sleep(delayFor(attempt, baseDelay, maxDelay));
  }

  throw new Error("fetchWithRetry exhausted retries unexpectedly");
}