From 91b11019db2d48bd9b6be1a733a3ea068517642e Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Sun, 11 Oct 2026 15:58:39 -0700
Subject: [PATCH] Ride out Venice overloads: retry 429, 5xx and dropped
 connections with backoff

One overloaded reply in the middle of a worker run used to throw away the
whole build (the MDecks relaunch hit exactly that). Seats and the worker
now retry up to three times (15s, 60s, 135s) before giving up; a 4xx like
a bad key still fails at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UpT7eWzeLhRwuYmsSQnbRb
---
 .github/time/test_time.py | 20 ++++++++++++++++++++
 .github/time/time.py      | 23 ++++++++++++++++++++---
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/.github/time/test_time.py b/.github/time/test_time.py
index 887c915..061151c 100644
--- a/.github/time/test_time.py
+++ b/.github/time/test_time.py
@@ -583,3 +583,23 @@ assert vt.external_refs('<link rel="stylesheet" href="https://fonts.x/a.css"><li
     ["https://fonts.x/a.css", "https://img/x.png", "//evil/x.css"]
 assert '"phone", "desktop"' in vt.HARNESS.format(frames="", names=json.dumps(list(vt.SIZES)))
 print("ALL OK 13")
+
+# 22 Venice's brief overloads are ridden out: a 429 and a dropped connection, then the reply
+os.environ["TIME_VENICE_BACKOFF"] = "0"
+seq = [RuntimeError("POST https://api.venice.ai/api/v1/chat/completions -> 429: overloaded"), ConnectionResetError("reset"),
+       {"choices": [{"message": {"content": "fine"}}]}]
+def flaky_http(method, url, token, body=None, timeout=0):
+    x = seq.pop(0)
+    if isinstance(x, Exception):
+        raise x
+    return x
+tm.http, os.environ["VENICE_API_KEY"] = flaky_http, "vk"
+real_think = importlib.util.module_from_spec(spec); spec.loader.exec_module(real_think)
+real_think.http = flaky_http
+assert real_think.think("grok", None, None, "s", "u") == "fine" and seq == []
+real_think.http = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("POST x -> 401: bad key"))
+try:
+    real_think.think("grok", None, None, "s", "u"); raise AssertionError("a 401 must not be retried")
+except RuntimeError as e:
+    assert "401" in str(e)
+print("ALL OK 14")
diff --git a/.github/time/time.py b/.github/time/time.py
index 1d7ab6d..4fd5ea1 100644
--- a/.github/time/time.py
+++ b/.github/time/time.py
@@ -14,6 +14,7 @@ import json
 import os
 import re
 import sys
+import time
 import urllib.error
 import urllib.request
 from pathlib import Path
@@ -100,16 +101,32 @@ def mentions(text: str) -> dict[str, tuple[str | None, str | None]]:
     return out
 
 
-def think(seat: str, model: str | None, effort: str | None, system: str, user: str) -> str:
+def venice_chat(body: dict, timeout: float) -> dict:
+    """POST chat/completions, riding out Venice's brief overloads (429, 5xx) and dropped connections with backoff."""
     key = os.environ.get("VENICE_API_KEY")
     if not key:
         raise LookupError("no VENICE_API_KEY secret on this repo yet")
+    for attempt in range(4):
+        try:
+            return http("POST", f"{VENICE}/chat/completions", key, body, timeout=timeout)
+        except (RuntimeError, OSError) as e:
+            transient = isinstance(e, OSError) or re.search(r"-> (429|5\d\d)\b", str(e))
+            if attempt == 3 or not transient:
+                raise
+            print(f"venice busy ({str(e)[:80]}); retrying")
+            time.sleep(int(os.environ.get("TIME_VENICE_BACKOFF") or 15) * (attempt + 1) ** 2)
+    raise AssertionError("unreachable")
+
+
+def think(seat: str, model: str | None, effort: str | None, system: str, user: str) -> str:
+    if not os.environ.get("VENICE_API_KEY"):
+        raise LookupError("no VENICE_API_KEY secret on this repo yet")
     body = {"model": model or SEATS[seat]["default"], "temperature": 0.4, "max_tokens": 4000,
             "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
             "venice_parameters": {"include_venice_system_prompt": False}}
     if effort:
         body["reasoning_effort"] = effort
-    out = http("POST", f"{VENICE}/chat/completions", key, body, timeout=240)
+    out = venice_chat(body, timeout=240)
     text = out["choices"][0]["message"]["content"] or ""
     return re.sub(r"<think>.*?</think>", "", text, flags=re.S).strip() or "(empty reply)"
 
@@ -483,7 +500,7 @@ def venice_work(card: dict, done: str, spec: str) -> dict:
                 "venice_parameters": {"include_venice_system_prompt": False}}
         if effort:
             body["reasoning_effort"] = effort
-        msg = http("POST", f"{VENICE}/chat/completions", key, body, timeout=300)["choices"][0]["message"]
+        msg = venice_chat(body, timeout=300)["choices"][0]["message"]
         msgs.append({k: v for k, v in msg.items() if k in ("role", "content", "tool_calls")})
         calls = msg.get("tool_calls") or []
         if not calls: