diff --git a/.github/time/labs/aicharts/bench.py b/.github/time/labs/aicharts/bench.py index 625203c..2ff0f8d 100644 --- a/.github/time/labs/aicharts/bench.py +++ b/.github/time/labs/aicharts/bench.py @@ -1,6 +1,7 @@ """Compare AICharts brains on one fixed question set against the live demo. python3 bench.py https://aicharts.yolopreneur.lol [brain ...] (default brains: llm jev clef) + python3 bench.py clef@0 Clef with the fallback off (calibration) Each question lists what a right chart must have; a spec scores 1 when it has all of it. Reports, per brain: right answers, median and p90 latency (as the browser sees it), and how often a decision brain handed off to the LLM. @@ -67,8 +68,10 @@ def misses(spec, want): def ask(url, prompt, brain): - body = json.dumps({"dataset": "orders", "prompt": prompt, "brain": brain}).encode() - req = urllib.request.Request(url.rstrip("/") + "/api/spec", data=body, headers={"content-type": "application/json"}) + name, _, floor = brain.partition("@") # "clef@0" = Clef with no fallback: its own answers, for calibration + body = json.dumps({"dataset": "orders", "prompt": prompt, "brain": name, **({"floor": float(floor)} if floor else {})}).encode() + req = urllib.request.Request(url.rstrip("/") + "/api/spec", data=body, # Cloudflare turns away Python's default agent + headers={"content-type": "application/json", "user-agent": "aicharts-bench/1"}) t0 = time.time() try: with urllib.request.urlopen(req, timeout=90) as r: @@ -87,8 +90,9 @@ def main(): out, ms = ask(url, prompt, brain) m = misses(out["spec"], want) if out.get("spec") else [out.get("error", "no spec")] rows.append({"q": prompt, "ok": not m, "ms": ms, "via": out.get("brain"), "handoff": (out.get("handoff") or {}).get("reason"), - "confidence": out.get("confidence"), "misses": m}) - print(f"[{brain}] {'ok ' if not m else 'MISS'} {ms:>6} ms via {out.get('brain')}: {prompt}" + (f" -- {'; '.join(m)}" if m else "") + "confidence": out.get("confidence"), "confidences": out.get("confidences"), "misses": m}) + print(f"[{brain}] {'ok ' if not m else 'MISS'} {ms:>6} ms via {out.get('brain')} conf {out.get('confidence')}: {prompt}" + + (f" -- {'; '.join(m)}" if m else "") + (f" (handoff: {rows[-1]['handoff']})" if rows[-1]["handoff"] else ""), flush=True) lat = sorted(r["ms"] for r in rows) report[brain] = {"right": sum(r["ok"] for r in rows), "of": len(rows), "median_ms": int(statistics.median(lat)), diff --git a/.github/time/labs/aicharts/decide.js b/.github/time/labs/aicharts/decide.js index c618021..86538cc 100644 --- a/.github/time/labs/aicharts/decide.js +++ b/.github/time/labs/aicharts/decide.js @@ -100,12 +100,13 @@ export function assemble(answers, info) { const only = transform.filter((s) => s.op === "filter").map((s) => s.eq); spec.title = mark === "scatter" ? `${spec.y} vs ${x}` : `${what} by ${x}${series ? ` and ${series}` : ""}${only.length ? ` (${only.join(", ")})` : ""}`; - const weakest = Math.min(...gating.map((k) => conf[k] ?? 0)); + const weakestOn = gating.reduce((lo, k) => ((conf[k] ?? 0) < (conf[lo] ?? 0) ? k : lo), gating[0]); + const weakest = conf[weakestOn] ?? 0; // other chart types the model rated close behind: cheap to offer, already the same data const probs = answers.mark?.probabilities || {}; const alternatives = Object.entries(probs).filter(([m, p]) => m !== mark && p >= 0.15).sort((a, b) => b[1] - a[1]).slice(0, 2) .map(([m, p]) => ({ mark: m, p: Math.round(p * 100) / 100 })); - return { spec, confidence: conf, weakest: Math.round(weakest * 100) / 100, alternatives }; + return { spec, confidence: conf, weakest: Math.round(weakest * 100) / 100, weakestOn, alternatives }; } export async function decide(env, model, info, prompt, prior) { diff --git a/.github/time/labs/aicharts/test_aicharts.mjs b/.github/time/labs/aicharts/test_aicharts.mjs index a068a76..133c670 100644 --- a/.github/time/labs/aicharts/test_aicharts.mjs +++ b/.github/time/labs/aicharts/test_aicharts.mjs @@ -208,15 +208,21 @@ decided.push({ answers: { ...westTop3, x: ans("product", 0.31) } }); answers.push({ response: good }); body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev" })).json(); assert.equal(body.brain, "llm"); -assert.match(body.handoff.reason, /low confidence \(0.31\)/); +assert.match(body.handoff.reason, /low confidence \(0.31 on x\)/); +assert.equal(body.handoff.confidences.x, 0.31); assert.deepEqual(body.spec, good); +// the same answer with the caller's own lower bar is accepted, with every confidence shown +decided.push({ answers: { ...westTop3, x: ans("product", 0.31) } }); +body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev", floor: 0.2 })).json(); +assert.equal(body.brain, "jev"); +assert.equal(body.confidences.x, 0.31); // decision model down -> the door too decided.push(new Error("no such model")); answers.push({ response: good }); body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev" })).json(); assert.match(body.handoff.reason, /decision model error: no such model/); // the server default brain, and an unknown brain falls back to it -env.BRAIN = "jev"; decided.push({ answers: westTop3 }); +resetQuota(); env.BRAIN = "jev"; decided.push({ answers: westTop3 }); assert.equal((await (await post("/api/spec", { prompt: "q", brain: "gpt" })).json()).brain, "jev"); delete env.BRAIN; resetQuota(); diff --git a/.github/time/labs/aicharts/worker.js b/.github/time/labs/aicharts/worker.js index b6423cb..158d0a4 100644 --- a/.github/time/labs/aicharts/worker.js +++ b/.github/time/labs/aicharts/worker.js @@ -176,7 +176,7 @@ async function model(env, messages) { // brain: "llm" writes the spec; "jev" / "clef" answer choice questions and decide.js assembles it. A decision that // fails validation, or whose weakest needed answer is under FLOOR, goes to the LLM: the cage with a door. export const BRAINS = ["llm", "jev", "clef"]; -async function specVia(env, dataset, prompt, prior, brain) { +async function specVia(env, dataset, prompt, prior, brain, floorAsked) { const info = describe(await rows(env), DATASETS[dataset].columns); const schema = { columns: info.columns }; const t0 = Date.now(); @@ -185,10 +185,14 @@ async function specVia(env, dataset, prompt, prior, brain) { const model = brain === "jev" ? env.JEV_MODEL || "typesafe/jev" : env.CLEF_MODEL || "@cf/cloudflare/clef-flash"; try { const d = await decide(env, model, info, prompt, prior && validateSpec(prior, schema).ok ? prior : undefined); - const v = validateSpec(d.spec, schema), floor = Number(env.FLOOR || 0.5); + // a caller may lower or raise the bar for itself (benchmarks; a page that prefers speed); it only changes cost + const floor = Number.isFinite(floorAsked) && floorAsked >= 0 && floorAsked <= 1 ? floorAsked : Number(env.FLOOR || 0.5); + const v = validateSpec(d.spec, schema); + const confidences = Object.fromEntries(Object.entries(d.confidence).map(([k, c]) => [k, Math.round(c * 100) / 100])); if (v.ok && d.weakest >= floor) - return { spec: d.spec, brain, model: d.model, ms: d.ms, confidence: d.weakest, alternatives: d.alternatives }; - handoff = { reason: v.ok ? `low confidence (${d.weakest})` : "decision failed checks: " + v.errors[0], decided: d.spec, ms: d.ms }; + return { spec: d.spec, brain, model: d.model, ms: d.ms, confidence: d.weakest, confidences, alternatives: d.alternatives }; + handoff = { reason: v.ok ? `low confidence (${d.weakest} on ${d.weakestOn})` : "decision failed checks: " + v.errors[0], + decided: d.spec, ms: d.ms, confidences }; } catch (e) { handoff = { reason: "decision model error: " + String(e.message || e).slice(0, 160) }; } @@ -240,7 +244,7 @@ async function api(req, env, url) { if (!prompt) return json({ error: "ask a question" }, 400); if (!(await spend(env, "ai", await who(req), 60, Number(env.AI_DAILY || 300)))) return json({ error: "over today's limit, try tomorrow" }, 429); const brain = BRAINS.includes(body.brain) ? body.brain : BRAINS.includes(env.BRAIN) ? env.BRAIN : "llm"; - const out = await specVia(env, ds, prompt, body.prior, brain); + const out = await specVia(env, ds, prompt, body.prior, brain, body.floor == null ? undefined : Number(body.floor)); return json(out, out.spec ? 200 : 422); } const v = validateSpec(body.spec, { columns: DATASETS[ds].columns });