diff --git a/.github/time/labs/aicharts/aicharts.js b/.github/time/labs/aicharts/aicharts.js index 10f7873..c1bc0cc 100644 --- a/.github/time/labs/aicharts/aicharts.js +++ b/.github/time/labs/aicharts/aicharts.js @@ -144,7 +144,7 @@ async function fetchSpec(endpoint, body) { body: JSON.stringify(body) }); const out = await r.json().catch(() => ({})); if (!r.ok) throw new Error(out.error || `the chart service answered ${r.status}`); - return out.spec; + return out; // { spec, brain?, model?, ms?, confidence?, alternatives?, handoff? } } // Drive an existing Chart.js chart. `data` stays in the browser; only `schema` (plus a sample and aggregates the server @@ -152,8 +152,8 @@ async function fetchSpec(endpoint, body) { export function attach(chart, { data, schema, endpoint, dataset }) { let last; return { - async ask(prompt, prior = last) { - const spec = await fetchSpec(endpoint, { dataset, prompt, prior }); + async ask(prompt, prior = last, brain) { + const { spec } = await fetchSpec(endpoint, { dataset, prompt, prior, ...(brain ? { brain } : {}) }); const v = validateSpec(spec, schema); if (!v.ok) throw new Error("the spec failed validation: " + v.errors.join("; ")); const cfg = chartConfig(applySpec(data, spec), spec); @@ -169,7 +169,7 @@ export function attach(chart, { data, schema, endpoint, dataset }) { const Base = globalThis.HTMLElement ?? class {}; -// +// brain: llm | jev | clef (server default if unset) // Needs Chart.js on the page (window.Chart). Fires "spec" after each answer and "save" when Save is pressed; set // element.onSave = async ({ spec, prompt, dataset }) => ... to store charts yourself (the demo stores them in D1). export class AiChart extends Base { @@ -201,9 +201,10 @@ export class AiChart extends Base { this._msg.textContent = "Thinking…"; try { await this.load(); - const spec = await fetchSpec(this.endpoint, { dataset: this.dataset, prompt, prior: this._spec }); - this.render(spec, prompt); - this.dispatchEvent(new CustomEvent("spec", { detail: { spec, prompt } })); + const brain = this.getAttribute("brain"); + const { spec, ...info } = await fetchSpec(this.endpoint, { dataset: this.dataset, prompt, prior: this._spec, ...(brain ? { brain } : {}) }); + await this.render(spec, prompt); + this.dispatchEvent(new CustomEvent("spec", { detail: { spec, prompt, info } })); } catch (e) { this._msg.textContent = String(e.message || e); } diff --git a/.github/time/labs/aicharts/bench.py b/.github/time/labs/aicharts/bench.py new file mode 100644 index 0000000..625203c --- /dev/null +++ b/.github/time/labs/aicharts/bench.py @@ -0,0 +1,101 @@ +"""Compare AICharts brains on one fixed question set against the live demo. + + python3 bench.py https://aicharts.yolopreneur.lol [brain ...] (default brains: llm jev clef) + +Each question lists what a right chart must have; a spec scores 1 when it has all of it. Reports, per brain: right +answers, median and p90 latency (as the browser sees it), and how often a decision brain handed off to the LLM. +Uses the public API like a visitor, so it counts against the demo's daily AI limit (12 calls per brain). +""" +import json +import statistics +import sys +import time +import urllib.error +import urllib.request + +Q = [ + ("total revenue by region", {"x": "region", "agg": ("sum", "amount")}), + ("monthly revenue, web vs store", {"mark": {"line"}, "x": "month", "series": "channel", "agg": ("sum", "amount")}), + ("share of units by product", {"mark": {"pie", "doughnut"}, "x": "product", "agg": ("sum", "units")}), + ("top 3 products in the West by revenue", {"x": "product", "only": {"region": "West"}, "agg": ("sum", "amount"), "limit": 3}), + ("average order amount by channel", {"x": "channel", "agg": ("avg", "amount")}), + ("how did matcha units trend month by month?", {"mark": {"line"}, "x": "month", "only": {"product": "Matcha"}, "agg": ("sum", "units")}), + ("how many orders per region", {"x": "region", "agg": ("count", None)}), + ("biggest single order amount for each product", {"x": "product", "agg": ("max", "amount")}), + ("store revenue by region", {"x": "region", "only": {"channel": "store"}, "agg": ("sum", "amount")}), + ("teapot units by month in the North", {"x": "month", "only": {"product": "Teapot", "region": "North"}, "agg": ("sum", "units")}), + ("which region sells the most hojicha?", {"x": "region", "only": {"product": "Hojicha"}, "sorted_desc": True}), + ("revenue by product, split by channel", {"x": {"product", "channel"}, "series": {"product", "channel"}, "agg": ("sum", "amount")}), +] + + +def facts(spec): + f = {"mark": spec.get("mark"), "x": spec.get("x"), "series": spec.get("series"), "only": {}, "limit": None, "sorted_desc": False} + for s in spec.get("transform") or []: + if s.get("op") == "filter": + if "eq" in s: + f["only"][s["field"]] = s["eq"] + elif isinstance(s.get("in"), list) and len(s["in"]) == 1: + f["only"][s["field"]] = s["in"][0] + elif s.get("op") == "group": + f["agg"] = (s.get("agg"), None if s.get("agg") == "count" else s.get("field")) + elif s.get("op") == "limit": + f["limit"] = s.get("n") + elif s.get("op") == "sort" and s.get("order") == "desc" and s.get("field") not in ("month", "date", f["x"]): + f["sorted_desc"] = True + return f + + +def misses(spec, want): + f, out = facts(spec), [] + for k, v in want.items(): + got = f.get(k) + if k == "only": + if {a: str(b).lower() for a, b in got.items()} != {a: b.lower() for a, b in v.items()}: + out.append(f"only {got} != {v}") + elif isinstance(v, set): + if got not in v: + out.append(f"{k} {got} not in {sorted(v)}") + elif k == "agg": + if got is None or got[0] != v[0] or (v[1] and got[1] != v[1]): + out.append(f"agg {got} != {v}") + elif got != v: + out.append(f"{k} {got} != {v}") + if "x" in want and "series" in want and f["x"] == f["series"]: + out.append("x == series") + return out + + +def ask(url, prompt, brain): + body = json.dumps({"dataset": "orders", "prompt": prompt, "brain": brain}).encode() + req = urllib.request.Request(url.rstrip("/") + "/api/spec", data=body, headers={"content-type": "application/json"}) + t0 = time.time() + try: + with urllib.request.urlopen(req, timeout=90) as r: + out = json.loads(r.read()) + except urllib.error.HTTPError as e: + out = json.loads(e.read() or b"{}") + return out, int((time.time() - t0) * 1000) + + +def main(): + url, brains = sys.argv[1], sys.argv[2:] or ["llm", "jev", "clef"] + report = {} + for brain in brains: + rows = [] + for prompt, want in Q: + out, ms = ask(url, prompt, brain) + m = misses(out["spec"], want) if out.get("spec") else [out.get("error", "no spec")] + rows.append({"q": prompt, "ok": not m, "ms": ms, "via": out.get("brain"), "handoff": (out.get("handoff") or {}).get("reason"), + "confidence": out.get("confidence"), "misses": m}) + print(f"[{brain}] {'ok ' if not m else 'MISS'} {ms:>6} ms via {out.get('brain')}: {prompt}" + (f" -- {'; '.join(m)}" if m else "") + + (f" (handoff: {rows[-1]['handoff']})" if rows[-1]["handoff"] else ""), flush=True) + lat = sorted(r["ms"] for r in rows) + report[brain] = {"right": sum(r["ok"] for r in rows), "of": len(rows), "median_ms": int(statistics.median(lat)), + "p90_ms": lat[int(0.9 * (len(lat) - 1))], "handoffs": sum(1 for r in rows if r["handoff"]), "rows": rows} + print(json.dumps({b: {k: v for k, v in r.items() if k != "rows"} for b, r in report.items()}, indent=1)) + json.dump(report, open("bench.json", "w"), indent=1) + + +if __name__ == "__main__": + main() diff --git a/.github/time/labs/aicharts/decide.js b/.github/time/labs/aicharts/decide.js new file mode 100644 index 0000000..c618021 --- /dev/null +++ b/.github/time/labs/aicharts/decide.js @@ -0,0 +1,120 @@ +// A decision-model brain for AICharts: Jev (TypeSafe) or Clef (Cloudflare) answers a handful of choice questions in +// one pass, and this module assembles the chart spec from the answers. The model never writes JSON, so a spec can't +// be malformed; the validator still checks it, and weak confidence hands the question to the LLM path. +// +// questions(info) -> System One questions for one dataset (info = describe(rows, columns)) +// assemble(answers, info) -> { spec, confidence, weakest, alternatives } +// decide(env, model, info, prompt, prior) -> the above plus { model, ms } (throws if the model call fails) + +const MARK_MENU = { + bar: "Comparing amounts across categories or a handful of periods.", + line: "A value over time (months, dates) or another ordered sequence; the trend matters.", + pie: "Shares of one whole across a few categories (6 or fewer).", + doughnut: "Shares of one whole, shown as a ring; same use as a pie.", + scatter: "Two numeric measures plotted against each other across many points.", +}; +const AGG_MENU = { + sum: "The total (revenue, units sold): add the values up.", + avg: "The typical or average value per order.", + count: "How many orders or rows, regardless of their values.", + min: "The smallest value.", + max: "The largest value.", +}; +const ORDER_MENU = { + natural: "No particular order asked for (time stays in time order).", + value_desc: "Biggest first: top, best, most, highest, ranking.", + value_asc: "Smallest first: bottom, worst, least, lowest.", +}; +const TOP_MENU = { all: "Show every group.", 3: "Only the top or bottom 3.", 5: "Only the top or bottom 5.", 10: "Only the top or bottom 10." }; + +function shuffled(menu) { + const e = Object.entries(menu); + for (let i = e.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [e[i], e[j]] = [e[j], e[i]]; } + return Object.fromEntries(e); +} +const choice = (instructions, menu) => ({ type: "choice", instructions, criteria: shuffled(menu) }); + +const describeCol = (c, agg) => { + const a = agg[c.name] || {}; + if (a.values) return `${c.type} column; values: ${a.values.join(", ")}`; + if ("min" in a) return `${c.type} column from ${a.min} to ${a.max}`; + return `${c.type} column`; +}; + +export function questions(info) { + const cols = info.columns.filter((c) => c.name !== "id"); + const text = cols.filter((c) => c.type === "string"); + const nums = cols.filter((c) => c.type === "number"); + const axis = Object.fromEntries(cols.map((c) => [c.name, describeCol(c, info.aggregates)])); + const measure = { ...Object.fromEntries(nums.map((c) => [c.name, describeCol(c, info.aggregates)])), rows: "The number of orders (no column)." }; + const q = { + mark: choice("Which chart type answers the request best?", MARK_MENU), + x: choice("Which column goes on the x axis (the categories or time the chart is broken down by)?", axis), + measure: choice("Which quantity does the chart measure?", measure), + agg: choice("How should that quantity be combined within each group?", AGG_MENU), + series: choice("Should the chart split into one coloured series per value of another column? Pick none unless the request compares groups against each other (\"web vs store\", \"by region over time\").", + { none: "One series only.", ...Object.fromEntries(text.map((c) => [c.name, `one series per ${c.name}`])) }), + order: choice("How should the groups be ordered?", ORDER_MENU), + top: choice("Does the request limit how many groups to show?", TOP_MENU), + }; + for (const c of text) { + const vals = info.aggregates[c.name]?.values; + if (vals?.length && vals.length <= 30) + q[`only.${c.name}`] = choice(`Does the request restrict the data to one ${c.name}? Pick any unless it names one.`, + { any: `No restriction on ${c.name}.`, ...Object.fromEntries(vals.map((v) => [v, `Only ${c.name} = ${v}.`])) }); + } + return q; +} + +export function assemble(answers, info) { + const pick = (k) => answers[k]?.choice; + const conf = Object.fromEntries(Object.entries(answers).map(([k, v]) => [k, Number(v?.confidence ?? 0)])); + const types = Object.fromEntries(info.columns.map((c) => [c.name, c.type])); + const mark = pick("mark"), x = pick("x"), measure = pick("measure"); + let agg = pick("agg"); + let series = pick("series"); + if (series === "none" || series === x || ["pie", "doughnut", "scatter"].includes(mark)) series = undefined; + const gating = ["mark", "x", "measure", "order", "top"]; + if (measure === "rows") agg = "count"; else gating.push("agg"); + if (series) gating.push("series"); + + const transform = []; + for (const k of Object.keys(answers).filter((k) => k.startsWith("only."))) { + const v = pick(k); + if (v && v !== "any") { transform.push({ op: "filter", field: k.slice(5), eq: v }); gating.push(k); } + } + let spec; + if (mark === "scatter") { + // a scatter plots rows, not groups: x and the measure are both raw number columns + spec = { mark, x, y: measure === "rows" ? "amount" : measure, transform }; + } else { + transform.push({ op: "group", by: series ? [x, series] : [x], agg, ...(agg === "count" ? {} : { field: measure }), as: "value" }); + const order = pick("order"); + if (order === "value_desc" || order === "value_asc") transform.push({ op: "sort", field: "value", order: order === "value_desc" ? "desc" : "asc" }); + else if (["month", "date", "number"].includes(types[x])) transform.push({ op: "sort", field: x, order: "asc" }); + const top = pick("top"); + if (top && top !== "all") transform.push({ op: "limit", n: Number(top) }); + spec = { mark, x, y: "value", ...(series ? { series } : {}), transform }; + } + const what = measure === "rows" ? "Orders" : `${{ sum: "Total", avg: "Average", count: "Count of", min: "Lowest", max: "Highest" }[agg]} ${measure}`; + const only = transform.filter((s) => s.op === "filter").map((s) => s.eq); + spec.title = mark === "scatter" ? `${spec.y} vs ${x}` : `${what} by ${x}${series ? ` and ${series}` : ""}${only.length ? ` (${only.join(", ")})` : ""}`; + + const weakest = Math.min(...gating.map((k) => conf[k] ?? 0)); + // other chart types the model rated close behind: cheap to offer, already the same data + const probs = answers.mark?.probabilities || {}; + const alternatives = Object.entries(probs).filter(([m, p]) => m !== mark && p >= 0.15).sort((a, b) => b[1] - a[1]).slice(0, 2) + .map(([m, p]) => ({ mark: m, p: Math.round(p * 100) / 100 })); + return { spec, confidence: conf, weakest: Math.round(weakest * 100) / 100, alternatives }; +} + +export async function decide(env, model, info, prompt, prior) { + const state = { user_request: prompt, table: { columns: info.columns.map((c) => c.name), rows: info.rows }, + ...(prior ? { current_chart: prior } : {}) }; + const input = { state, questions: questions(info) }; + if (model.startsWith("@cf/cloudflare/clef")) input.model = model.endsWith("flash") ? "clef-flash" : "clef"; + const t0 = Date.now(); + const res = await env.AI.run(model, input); + if (!res?.answers) throw new Error("the decision model returned no answers"); + return { ...assemble(res.answers, info), model: res.model || model, ms: Date.now() - t0 }; +} diff --git a/.github/time/labs/aicharts/deploy.py b/.github/time/labs/aicharts/deploy.py index 15ec08b..2c05eeb 100644 --- a/.github/time/labs/aicharts/deploy.py +++ b/.github/time/labs/aicharts/deploy.py @@ -26,6 +26,7 @@ def parts() -> list[tuple[str, str, bytes]]: return [("main.js", "application/javascript+module", MAIN), ("worker.js", "application/javascript+module", (HERE / "worker.js").read_bytes()), ("aicharts.js", "application/javascript+module", lib), + ("decide.js", "application/javascript+module", (HERE / "decide.js").read_bytes()), ("aicharts.txt", "text/plain", lib)] @@ -35,7 +36,8 @@ def deploy() -> dict: code = parts() rev = hashlib.sha256(b"".join(d for _, _, d in code)).hexdigest()[:12] vars_ = {"REV": rev, "AI_MODEL": cf.env("AICHARTS_AI_MODEL", "@cf/meta/llama-3.3-70b-instruct-fp8-fast"), "AI_DAILY": cf.env("AICHARTS_AI_DAILY", "300"), - "SAVE_DAILY": cf.env("AICHARTS_SAVE_DAILY", "1000")} + "SAVE_DAILY": cf.env("AICHARTS_SAVE_DAILY", "1000"), + "BRAIN": cf.env("AICHARTS_BRAIN", "llm"), "FLOOR": cf.env("AICHARTS_FLOOR", "0.5")} db = cf.database("aicharts") # named time-aicharts, like the script meta = {"main_module": "main.js", "compatibility_date": cf.COMPAT, "bindings": [{"type": "d1", "name": "DB", "id": db}, {"type": "ai", "name": "AI"}] diff --git a/.github/time/labs/aicharts/test_aicharts.mjs b/.github/time/labs/aicharts/test_aicharts.mjs index ca63948..a068a76 100644 --- a/.github/time/labs/aicharts/test_aicharts.mjs +++ b/.github/time/labs/aicharts/test_aicharts.mjs @@ -85,7 +85,10 @@ class D1 { const answers = []; const calls = []; const env = { DB: new D1(), REV: "r1", AI_MODEL: "@cf/test/m", AI_DAILY: "5", SAVE_DAILY: "100", - AI: { async run(model, input) { calls.push(input); const a = answers.shift(); if (a instanceof Error) throw a; return a; } } }; + AI: { async run(model, input) { + if (input.questions) { decisions.push({ model, input }); const d = decided.shift(); if (d instanceof Error) throw d; return d; } + calls.push(input); const a = answers.shift(); if (a instanceof Error) throw a; return a; } } }; +const decisions = [], decided = []; const worker = W.make(src); const call = (path, init = {}, ip = "1.1.1.1") => worker.fetch(new Request("https://aicharts.t.workers.dev" + path, { ...init, headers: { "cf-connecting-ip": ip, ...(init.headers || {}) } }), env); @@ -152,6 +155,71 @@ assert.equal((await post("/api/spec", { prompt: "b" }, "2.2.2.2")).status, 200); assert.equal((await post("/api/spec", { prompt: "c" }, "3.3.3.3")).status, 429); assert.equal((await post("/api/spec", { prompt: "" })).status, 400); +// ---------- decision brains (Jev / Clef): choice answers -> spec, LLM only as the door ---------- +const D = await import(join(here, "decide.js")); +const ans = (choice, confidence = 0.9, probabilities) => ({ type: "choice", choice, confidence, probabilities: probabilities || { [choice]: confidence } }); +const westTop3 = { mark: ans("bar", 0.8, { bar: 0.8, pie: 0.17, line: 0.03 }), x: ans("product"), measure: ans("amount"), agg: ans("sum"), series: ans("none"), + order: ans("value_desc"), top: ans("3"), "only.region": ans("West", 0.95), "only.product": ans("any"), "only.channel": ans("any") }; +const infoNow = W.describe(data, sch.schema.columns); +const qs = D.questions(infoNow); +assert.deepEqual(Object.keys(qs).sort(), ["agg", "mark", "measure", "only.channel", "only.product", "only.region", "order", "series", "top", "x"]); +assert.ok(!("id" in qs.x.criteria), "id is never an axis"); +assert.ok("rows" in qs.measure.criteria && "amount" in qs.measure.criteria && !("region" in qs.measure.criteria)); +assert.deepEqual(Object.keys(qs["only.region"].criteria).sort(), ["East", "North", "South", "West", "any"]); +const asm = D.assemble(westTop3, infoNow); +assert.deepEqual(asm.spec.transform, [{ op: "filter", field: "region", eq: "West" }, + { op: "group", by: ["product"], agg: "sum", field: "amount", as: "value" }, { op: "sort", field: "value", order: "desc" }, { op: "limit", n: 3 }]); +assert.equal(asm.spec.title, "Total amount by product (West)"); +assert.ok(lib.validateSpec(asm.spec, sch.schema).ok); +assert.equal(lib.applySpec(data, asm.spec).length, 3); +assert.deepEqual(asm.alternatives, [{ mark: "pie", p: 0.17 }]); +assert.equal(asm.weakest, 0.8); +// time on x sorts by time; a series only when asked; count needs no field +const monthly = D.assemble({ ...westTop3, mark: ans("line"), x: ans("month"), measure: ans("rows"), agg: ans("avg", 0.1), series: ans("channel"), + order: ans("natural"), top: ans("all"), "only.region": ans("any") }, infoNow); +assert.deepEqual(monthly.spec.transform, [{ op: "group", by: ["month", "channel"], agg: "count", as: "value" }, { op: "sort", field: "month", order: "asc" }]); +assert.equal(monthly.spec.series, "channel"); +assert.equal(monthly.weakest, 0.9, "agg doesn't gate when counting rows"); +assert.ok(lib.validateSpec(monthly.spec, sch.schema).ok); +assert.equal(D.assemble({ ...westTop3, mark: ans("pie"), series: ans("channel") }, infoNow).spec.series, undefined, "no series on a pie"); + +// through the worker: a confident Jev answer is the spec, no LLM call +const resetQuota = () => env.DB.db.exec("DELETE FROM quota"); +resetQuota(); calls.length = 0; +decided.push({ model: "jev-1", answers: westTop3, usage: {} }); +r = await post("/api/spec", { dataset: "orders", prompt: "top 3 products in the West by revenue", brain: "jev" }); +let body = await r.json(); +assert.equal(r.status, 200, JSON.stringify(body)); +assert.equal(body.brain, "jev"); +assert.equal(decisions.at(-1).model, "typesafe/jev"); +assert.equal(decisions.at(-1).input.state.user_request, "top 3 products in the West by revenue"); +const toJev = JSON.stringify(decisions.at(-1).input), mid = data[200]; +assert.ok(!toJev.includes(mid.date) && !toJev.includes(String(mid.amount)) && !("sample" in decisions.at(-1).input.state), + "rows never reach the decision model"); +assert.equal(calls.length, 0); +assert.deepEqual(body.spec, asm.spec); +// Clef names its variant in the input +decided.push({ answers: westTop3 }); +await post("/api/spec", { dataset: "orders", prompt: "x", brain: "clef" }); +assert.equal(decisions.at(-1).model, "@cf/cloudflare/clef-flash"); +assert.equal(decisions.at(-1).input.model, "clef-flash"); +// unsure -> the door: the LLM answers, and the response says why +decided.push({ answers: { ...westTop3, x: ans("product", 0.31) } }); +answers.push({ response: good }); +body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev" })).json(); +assert.equal(body.brain, "llm"); +assert.match(body.handoff.reason, /low confidence \(0.31\)/); +assert.deepEqual(body.spec, good); +// decision model down -> the door too +decided.push(new Error("no such model")); +answers.push({ response: good }); +body = await (await post("/api/spec", { dataset: "orders", prompt: "hmm", brain: "jev" })).json(); +assert.match(body.handoff.reason, /decision model error: no such model/); +// the server default brain, and an unknown brain falls back to it +env.BRAIN = "jev"; decided.push({ answers: westTop3 }); +assert.equal((await (await post("/api/spec", { prompt: "q", brain: "gpt" })).json()).brain, "jev"); +delete env.BRAIN; resetQuota(); + // save, list, load; only valid specs are stored r = await post("/api/charts", { dataset: "orders", prompt: "revenue by region", spec: good }); assert.equal(r.status, 201); diff --git a/.github/time/labs/aicharts/worker.js b/.github/time/labs/aicharts/worker.js index a1e6cca..b6423cb 100644 --- a/.github/time/labs/aicharts/worker.js +++ b/.github/time/labs/aicharts/worker.js @@ -11,6 +11,7 @@ // Read-only except saved_charts and quota. Bindings: DB (D1), AI (Workers AI). Vars: AI_MODEL, AI_DAILY, SAVE_DAILY, REV. // deploy.py uploads main.js, which hands this module the library source: export default make(src). import { validateSpec, MARKS, AGGS } from "./aicharts.js"; +import { decide } from "./decide.js"; const SCHEMA = [ "CREATE TABLE IF NOT EXISTS orders (id INTEGER PRIMARY KEY, date TEXT, month TEXT, region TEXT, product TEXT, channel TEXT, units INTEGER, amount REAL)", @@ -172,9 +173,33 @@ async function model(env, messages) { } // Ask, check, and give the model one chance to fix what the validator found. -async function specFor(env, dataset, prompt, prior) { +// brain: "llm" writes the spec; "jev" / "clef" answer choice questions and decide.js assembles it. A decision that +// fails validation, or whose weakest needed answer is under FLOOR, goes to the LLM: the cage with a door. +export const BRAINS = ["llm", "jev", "clef"]; +async function specVia(env, dataset, prompt, prior, brain) { const info = describe(await rows(env), DATASETS[dataset].columns); const schema = { columns: info.columns }; + const t0 = Date.now(); + let handoff; + if (brain !== "llm") { + const model = brain === "jev" ? env.JEV_MODEL || "typesafe/jev" : env.CLEF_MODEL || "@cf/cloudflare/clef-flash"; + try { + const d = await decide(env, model, info, prompt, prior && validateSpec(prior, schema).ok ? prior : undefined); + const v = validateSpec(d.spec, schema), floor = Number(env.FLOOR || 0.5); + if (v.ok && d.weakest >= floor) + return { spec: d.spec, brain, model: d.model, ms: d.ms, confidence: d.weakest, alternatives: d.alternatives }; + handoff = { reason: v.ok ? `low confidence (${d.weakest})` : "decision failed checks: " + v.errors[0], decided: d.spec, ms: d.ms }; + } catch (e) { + handoff = { reason: "decision model error: " + String(e.message || e).slice(0, 160) }; + } + } + const out = await specFor(env, dataset, prompt, prior, info); + return { ...out, brain: "llm", model: env.AI_MODEL, ms: Date.now() - t0, ...(handoff ? { handoff } : {}) }; +} + +async function specFor(env, dataset, prompt, prior, known) { + const info = known || describe(await rows(env), DATASETS[dataset].columns); + const schema = { columns: info.columns }; const messages = [{ role: "system", content: systemPrompt(dataset, info) }]; if (prior && validateSpec(prior, schema).ok) messages.push({ role: "user", content: "Current chart spec (change it if the question builds on it): " + JSON.stringify(prior) }); @@ -213,8 +238,9 @@ async function api(req, env, url) { if (path === "/api/spec") { if (!env.AI) return json({ error: "no AI here" }, 501); if (!prompt) return json({ error: "ask a question" }, 400); - if (!(await spend(env, "ai", await who(req), 30, Number(env.AI_DAILY || 300)))) return json({ error: "over today's limit, try tomorrow" }, 429); - const out = await specFor(env, ds, prompt, body.prior); + if (!(await spend(env, "ai", await who(req), 60, Number(env.AI_DAILY || 300)))) return json({ error: "over today's limit, try tomorrow" }, 429); + const brain = BRAINS.includes(body.brain) ? body.brain : BRAINS.includes(env.BRAIN) ? env.BRAIN : "llm"; + const out = await specVia(env, ds, prompt, body.prior, brain); return json(out, out.spec ? 200 : 422); } const v = validateSpec(body.spec, { columns: DATASETS[ds].columns }); @@ -251,12 +277,15 @@ button{padding:10px 14px;border:0;border-radius:8px;background:var(--accent);col .aic-msg{min-height:24px;margin:10px 0;color:var(--mute)}.aic-canvas{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px} .aic-save{margin-top:10px}.chips{display:flex;flex-wrap:wrap;gap:6px;margin:12px 0}.chips button{background:var(--card);color:var(--fg);border:1px solid var(--line);padding:6px 10px;font-size:14px} details{margin-top:16px}pre{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:12px;overflow:auto;font-size:13px} -h2{font-size:18px;margin:28px 0 8px}ul{padding-left:18px}a{color:var(--accent)} +h2{font-size:18px;margin:28px 0 8px}.row{display:flex;flex-wrap:wrap;gap:8px;align-items:center;justify-content:space-between} +.brain{font-size:14px;color:var(--mute)}.brain select{font:inherit;padding:4px;border-radius:6px;border:1px solid var(--line);background:var(--card);color:var(--fg)} +.meta{display:flex;flex-wrap:wrap;gap:6px;align-items:center;margin-top:10px;font-size:14px;color:var(--mute)}.meta button{padding:4px 10px;font-size:14px;background:var(--card);color:var(--fg);border:1px solid var(--line)}ul{padding-left:18px}a{color:var(--accent)}

AICharts

Ask for a chart in plain words. The model writes a small spec that is checked before anything draws, and never sees the rows: only column names, five sample rows and value ranges. Data: a sample tea shop, 24 months of orders.

-
+
+
Spec
Ask something first.

Saved charts

Use it on your page

@@ -272,7 +301,20 @@ for (const q of ["Revenue by region", "Monthly revenue, web vs store", "Share of const b = document.createElement("button"); b.textContent = q; b.onclick = () => { el.querySelector("input").value = q; el.ask(q); }; document.getElementById("chips").append(b); } -el.addEventListener("spec", (e) => { document.getElementById("spec").textContent = JSON.stringify(e.detail.spec, null, 2); }); +const brainSel = document.getElementById("brain"), meta = document.getElementById("meta"); +brainSel.onchange = () => brainSel.value ? el.setAttribute("brain", brainSel.value) : el.removeAttribute("brain"); +el.addEventListener("spec", (e) => { + const { spec, info = {} } = e.detail; + document.getElementById("spec").textContent = JSON.stringify({ ...info, spec }, null, 2); + const bits = [info.brain && "via " + info.brain + (info.handoff ? " (handed off: " + info.handoff.reason + ")" : ""), + info.ms != null && info.ms + " ms", info.confidence != null && "confidence " + info.confidence].filter(Boolean); + meta.replaceChildren(document.createTextNode(bits.join(" · "))); + for (const alt of info.alternatives || []) { + const b = document.createElement("button"); b.textContent = "or as " + alt.mark + " (" + Math.round(alt.p * 100) + "%)"; + b.onclick = () => el.render({ ...spec, mark: alt.mark }, e.detail.prompt).catch((err) => { meta.append(" " + err.message); }); + meta.append(b); + } +}); el.onSave = async (d) => { const r = await fetch("/api/charts", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(d) }); const out = await r.json(); if (!r.ok) throw new Error(out.error); history.replaceState(null, "", "?c=" + out.id); await refresh();