Ride out Venice overloads: retry 429, 5xx and dropped connections with backoff

One overloaded reply in the middle of a worker run used to throw away the
whole build (the MDecks relaunch hit exactly that). Seats and the worker
now retry up to three times (15s, 60s, 135s) before giving up; a 4xx like
a bad key still fails at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UpT7eWzeLhRwuYmsSQnbRb

.github/time/test_time.py

@@ -583,3 +583,23 @@ assert vt.external_refs('<link rel="stylesheet" href="https://fonts.x/a.css"><li
     ["https://fonts.x/a.css", "https://img/x.png", "//evil/x.css"]
 assert '"phone", "desktop"' in vt.HARNESS.format(frames="", names=json.dumps(list(vt.SIZES)))
 print("ALL OK 13")
+
+# 22 Venice's brief overloads are ridden out: a 429 and a dropped connection, then the reply
+os.environ["TIME_VENICE_BACKOFF"] = "0"
+seq = [RuntimeError("POST https://api.venice.ai/api/v1/chat/completions -> 429: overloaded"), ConnectionResetError("reset"),
+       {"choices": [{"message": {"content": "fine"}}]}]
+def flaky_http(method, url, token, body=None, timeout=0):
+    x = seq.pop(0)
+    if isinstance(x, Exception):
+        raise x
+    return x
+tm.http, os.environ["VENICE_API_KEY"] = flaky_http, "vk"
+real_think = importlib.util.module_from_spec(spec); spec.loader.exec_module(real_think)
+real_think.http = flaky_http
+assert real_think.think("grok", None, None, "s", "u") == "fine" and seq == []
+real_think.http = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("POST x -> 401: bad key"))
+try:
+    real_think.think("grok", None, None, "s", "u"); raise AssertionError("a 401 must not be retried")
+except RuntimeError as e:
+    assert "401" in str(e)
+print("ALL OK 14")

.github/time/time.py

@@ -14,6 +14,7 @@ import json
 import os
 import re
 import sys
+import time
 import urllib.error
 import urllib.request
 from pathlib import Path
@@ -100,16 +101,32 @@ def mentions(text: str) -> dict[str, tuple[str | None, str | None]]:
     return out
 
 
-def think(seat: str, model: str | None, effort: str | None, system: str, user: str) -> str:
+def venice_chat(body: dict, timeout: float) -> dict:
+    """POST chat/completions, riding out Venice's brief overloads (429, 5xx) and dropped connections with backoff."""
     key = os.environ.get("VENICE_API_KEY")
     if not key:
         raise LookupError("no VENICE_API_KEY secret on this repo yet")
+    for attempt in range(4):
+        try:
+            return http("POST", f"{VENICE}/chat/completions", key, body, timeout=timeout)
+        except (RuntimeError, OSError) as e:
+            transient = isinstance(e, OSError) or re.search(r"-> (429|5\d\d)\b", str(e))
+            if attempt == 3 or not transient:
+                raise
+            print(f"venice busy ({str(e)[:80]}); retrying")
+            time.sleep(int(os.environ.get("TIME_VENICE_BACKOFF") or 15) * (attempt + 1) ** 2)
+    raise AssertionError("unreachable")
+
+
+def think(seat: str, model: str | None, effort: str | None, system: str, user: str) -> str:
+    if not os.environ.get("VENICE_API_KEY"):
+        raise LookupError("no VENICE_API_KEY secret on this repo yet")
     body = {"model": model or SEATS[seat]["default"], "temperature": 0.4, "max_tokens": 4000,
             "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
             "venice_parameters": {"include_venice_system_prompt": False}}
     if effort:
         body["reasoning_effort"] = effort
-    out = http("POST", f"{VENICE}/chat/completions", key, body, timeout=240)
+    out = venice_chat(body, timeout=240)
     text = out["choices"][0]["message"]["content"] or ""
     return re.sub(r"<think>.*?</think>", "", text, flags=re.S).strip() or "(empty reply)"
 
@@ -483,7 +500,7 @@ def venice_work(card: dict, done: str, spec: str) -> dict:
                 "venice_parameters": {"include_venice_system_prompt": False}}
         if effort:
             body["reasoning_effort"] = effort
-        msg = http("POST", f"{VENICE}/chat/completions", key, body, timeout=300)["choices"][0]["message"]
+        msg = venice_chat(body, timeout=300)["choices"][0]["message"]
         msgs.append({k: v for k, v in msg.items() if k in ("role", "content", "tool_calls")})
         calls = msg.get("tool_calls") or []
         if not calls: