labs/aicharts: show decision confidences, per-request floor, bench user agent (#41)

Live: Clef answered both probes right in ~0.5 s but at 0.21-0.25 confidence, so
the 0.5 floor sent them to the LLM. Expose every confidence and the weakest
question; a request may set its own floor (only changes cost); bench.py takes
clef@0 for calibration and sends a user agent Cloudflare accepts.


Claude-Session: https://claude.ai/code/session_01VDkN7ukK1kXkqov8FaZxva

Co-authored-by: Claude <noreply@anthropic.com>

.github/time/labs/aicharts/bench.py

@@ -1,6 +1,7 @@
 """Compare AICharts brains on one fixed question set against the live demo.
 
     python3 bench.py https://aicharts.yolopreneur.lol [brain ...]      (default brains: llm jev clef)
+    python3 bench.py <url> clef@0                                      Clef with the fallback off (calibration)
 
 Each question lists what a right chart must have; a spec scores 1 when it has all of it. Reports, per brain: right
 answers, median and p90 latency (as the browser sees it), and how often a decision brain handed off to the LLM.
@@ -67,8 +68,10 @@ def misses(spec, want):
 
 
 def ask(url, prompt, brain):
-    body = json.dumps({"dataset": "orders", "prompt": prompt, "brain": brain}).encode()
-    req = urllib.request.Request(url.rstrip("/") + "/api/spec", data=body, headers={"content-type": "application/json"})
+    name, _, floor = brain.partition("@")  # "clef@0" = Clef with no fallback: its own answers, for calibration
+    body = json.dumps({"dataset": "orders", "prompt": prompt, "brain": name, **({"floor": float(floor)} if floor else {})}).encode()
+    req = urllib.request.Request(url.rstrip("/") + "/api/spec", data=body,  # Cloudflare turns away Python's default agent
+                                 headers={"content-type": "application/json", "user-agent": "aicharts-bench/1"})
     t0 = time.time()
     try:
         with urllib.request.urlopen(req, timeout=90) as r:
@@ -87,8 +90,9 @@ def main():
             out, ms = ask(url, prompt, brain)
             m = misses(out["spec"], want) if out.get("spec") else [out.get("error", "no spec")]
             rows.append({"q": prompt, "ok": not m, "ms": ms, "via": out.get("brain"), "handoff": (out.get("handoff") or {}).get("reason"),
-                         "confidence": out.get("confidence"), "misses": m})
-            print(f"[{brain}] {'ok ' if not m else 'MISS'} {ms:>6} ms via {out.get('brain')}: {prompt}" + (f"  -- {'; '.join(m)}" if m else "")
+                         "confidence": out.get("confidence"), "confidences": out.get("confidences"), "misses": m})
+            print(f"[{brain}] {'ok ' if not m else 'MISS'} {ms:>6} ms via {out.get('brain')} conf {out.get('confidence')}: {prompt}"
+                  + (f"  -- {'; '.join(m)}" if m else "")
                   + (f"  (handoff: {rows[-1]['handoff']})" if rows[-1]["handoff"] else ""), flush=True)
         lat = sorted(r["ms"] for r in rows)
         report[brain] = {"right": sum(r["ok"] for r in rows), "of": len(rows), "median_ms": int(statistics.median(lat)),

.github/time/labs/aicharts/decide.js

@@ -100,12 +100,13 @@ export function assemble(answers, info) {
   const only = transform.filter((s) => s.op === "filter").map((s) => s.eq);
   spec.title = mark === "scatter" ? `${spec.y} vs ${x}` : `${what} by ${x}${series ? ` and ${series}` : ""}${only.length ? ` (${only.join(", ")})` : ""}`;
 
-  const weakest = Math.min(...gating.map((k) => conf[k] ?? 0));
+  const weakestOn = gating.reduce((lo, k) => ((conf[k] ?? 0) < (conf[lo] ?? 0) ? k : lo), gating[0]);
+  const weakest = conf[weakestOn] ?? 0;
   // other chart types the model rated close behind: cheap to offer, already the same data
   const probs = answers.mark?.probabilities || {};
   const alternatives = Object.entries(probs).filter(([m, p]) => m !== mark && p >= 0.15).sort((a, b) => b[1] - a[1]).slice(0, 2)
     .map(([m, p]) => ({ mark: m, p: Math.round(p * 100) / 100 }));
-  return { spec, confidence: conf, weakest: Math.round(weakest * 100) / 100, alternatives };
+  return { spec, confidence: conf, weakest: Math.round(weakest * 100) / 100, weakestOn, alternatives };
 }
 
 export async function decide(env, model, info, prompt, prior) {

.github/time/labs/aicharts/test_aicharts.mjs

@@ -208,15 +208,21 @@ decided.push({ answers: { ...westTop3, x: ans("product", 0.31) } });
 answers.push({ response: good });
 body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev" })).json();
 assert.equal(body.brain, "llm");
-assert.match(body.handoff.reason, /low confidence \(0.31\)/);
+assert.match(body.handoff.reason, /low confidence \(0.31 on x\)/);
+assert.equal(body.handoff.confidences.x, 0.31);
 assert.deepEqual(body.spec, good);
+// the same answer with the caller's own lower bar is accepted, with every confidence shown
+decided.push({ answers: { ...westTop3, x: ans("product", 0.31) } });
+body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev", floor: 0.2 })).json();
+assert.equal(body.brain, "jev");
+assert.equal(body.confidences.x, 0.31);
 // decision model down -> the door too
 decided.push(new Error("no such model"));
 answers.push({ response: good });
 body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev" })).json();
 assert.match(body.handoff.reason, /decision model error: no such model/);
 // the server default brain, and an unknown brain falls back to it
-env.BRAIN = "jev"; decided.push({ answers: westTop3 });
+resetQuota(); env.BRAIN = "jev"; decided.push({ answers: westTop3 });
 assert.equal((await (await post("/api/spec", { prompt: "q", brain: "gpt" })).json()).brain, "jev");
 delete env.BRAIN; resetQuota();
 

.github/time/labs/aicharts/worker.js

@@ -176,7 +176,7 @@ async function model(env, messages) {
 // brain: "llm" writes the spec; "jev" / "clef" answer choice questions and decide.js assembles it. A decision that
 // fails validation, or whose weakest needed answer is under FLOOR, goes to the LLM: the cage with a door.
 export const BRAINS = ["llm", "jev", "clef"];
-async function specVia(env, dataset, prompt, prior, brain) {
+async function specVia(env, dataset, prompt, prior, brain, floorAsked) {
   const info = describe(await rows(env), DATASETS[dataset].columns);
   const schema = { columns: info.columns };
   const t0 = Date.now();
@@ -185,10 +185,14 @@ async function specVia(env, dataset, prompt, prior, brain) {
     const model = brain === "jev" ? env.JEV_MODEL || "typesafe/jev" : env.CLEF_MODEL || "@cf/cloudflare/clef-flash";
     try {
       const d = await decide(env, model, info, prompt, prior && validateSpec(prior, schema).ok ? prior : undefined);
-      const v = validateSpec(d.spec, schema), floor = Number(env.FLOOR || 0.5);
+      // a caller may lower or raise the bar for itself (benchmarks; a page that prefers speed); it only changes cost
+      const floor = Number.isFinite(floorAsked) && floorAsked >= 0 && floorAsked <= 1 ? floorAsked : Number(env.FLOOR || 0.5);
+      const v = validateSpec(d.spec, schema);
+      const confidences = Object.fromEntries(Object.entries(d.confidence).map(([k, c]) => [k, Math.round(c * 100) / 100]));
       if (v.ok && d.weakest >= floor)
-        return { spec: d.spec, brain, model: d.model, ms: d.ms, confidence: d.weakest, alternatives: d.alternatives };
-      handoff = { reason: v.ok ? `low confidence (${d.weakest})` : "decision failed checks: " + v.errors[0], decided: d.spec, ms: d.ms };
+        return { spec: d.spec, brain, model: d.model, ms: d.ms, confidence: d.weakest, confidences, alternatives: d.alternatives };
+      handoff = { reason: v.ok ? `low confidence (${d.weakest} on ${d.weakestOn})` : "decision failed checks: " + v.errors[0],
+                  decided: d.spec, ms: d.ms, confidences };
     } catch (e) {
       handoff = { reason: "decision model error: " + String(e.message || e).slice(0, 160) };
     }
@@ -240,7 +244,7 @@ async function api(req, env, url) {
       if (!prompt) return json({ error: "ask a question" }, 400);
       if (!(await spend(env, "ai", await who(req), 60, Number(env.AI_DAILY || 300)))) return json({ error: "over today's limit, try tomorrow" }, 429);
       const brain = BRAINS.includes(body.brain) ? body.brain : BRAINS.includes(env.BRAIN) ? env.BRAIN : "llm";
-      const out = await specVia(env, ds, prompt, body.prior, brain);
+      const out = await specVia(env, ds, prompt, body.prior, brain, body.floor == null ? undefined : Number(body.floor));
       return json(out, out.spec ? 200 : 422);
     }
     const v = validateSpec(body.spec, { columns: DATASETS[ds].columns });