From 0189649a2a3f8df37673e566b32671275480168c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 13:46:49 +0000 Subject: [PATCH] labs/aicharts: fix model spelling drift, show the model's answer on failure First live run: Workers AI answered specs the validator refused (agg spelling). normalizeSpec fixes case and common aliases only; on a 422 the response carries model_said (shown, never run). Smoke now fails on a bad spec. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VDkN7ukK1kXkqov8FaZxva --- .github/time/labs/aicharts/test_aicharts.mjs | 8 ++++- .github/time/labs/aicharts/worker.js | 36 +++++++++++++++++--- .github/workflows/labs-aicharts.yml | 1 + 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.github/time/labs/aicharts/test_aicharts.mjs b/.github/time/labs/aicharts/test_aicharts.mjs index b3d85d3..fd4bb4c 100644 --- a/.github/time/labs/aicharts/test_aicharts.mjs +++ b/.github/time/labs/aicharts/test_aicharts.mjs @@ -61,6 +61,10 @@ assert.deepEqual(sent.at(-1).body.prior, bySum, "follow-ups carry the last chart assert.deepEqual(W.firstObject('Sure!\n```json\n{"a": {"b": "}"}}\n```'), { a: { b: "}" } }); assert.deepEqual(W.firstObject('Sure! {"mark": "bar", "x": "a"} trailing {'), { mark: "bar", x: "a" }); assert.equal(W.firstObject("no json"), null); +assert.deepEqual(W.normalizeSpec({ mark: "Bar", x: "region", y: "value", + transform: [{ type: "groupBy", groupBy: "region", aggregate: "SUM", field: "amount", as: "value" }, { op: "sort", field: "value", order: "DESC" }] }), + { mark: "bar", x: "region", y: "value", transform: [{ op: "group", by: ["region"], agg: "sum", field: "amount", as: "value" }, { op: "sort", field: "value", order: "desc" }] }); +assert.equal(W.normalizeSpec({ mark: "bar", transform: [{ op: "group", agg: "mean" }] }).transform[0].agg, "avg"); class D1 { constructor() { this.db = new DatabaseSync(":memory:"); this.writes = []; } @@ -134,7 +138,9 @@ assert.ok(calls.at(-1).messages.at(-1).content.includes("fails these checks"), " answers.push({ response: "{}" }, { response: '{"mark": "pie"}' }); r = await post("/api/spec", { dataset: "orders", prompt: "nonsense" }); assert.equal(r.status, 422); -assert.match((await r.json()).error, /valid chart/); +const failed = await r.json(); +assert.match(failed.error, /valid chart/); +assert.equal(failed.model_said, '{"mark": "pie"}', "the model's answer is shown on failure"); // limits: AI_DAILY=5 overall; we've used 3 answers.push({ response: good }, { response: good }); diff --git a/.github/time/labs/aicharts/worker.js b/.github/time/labs/aicharts/worker.js index cc7eee1..a388e74 100644 --- a/.github/time/labs/aicharts/worker.js +++ b/.github/time/labs/aicharts/worker.js @@ -147,6 +147,30 @@ export function firstObject(text) { return null; } +// Small models drift on spelling, not meaning: "SUM", "mean", "aggregate", "groupBy": "region". Fix the spelling, then +// the validator decides. Nothing here invents a field or an operation. +const AGG_ALIAS = { average: "avg", mean: "avg", total: "sum", minimum: "min", maximum: "max", cnt: "count" }; +export function normalizeSpec(spec) { + if (!spec || typeof spec !== "object" || Array.isArray(spec)) return spec; + const low = (v) => (typeof v === "string" ? v.trim().toLowerCase() : v); + const out = { ...spec, mark: low(spec.mark) }; + if (spec.chart && !spec.mark) out.mark = low(spec.chart.type ?? spec.chart); + if (Array.isArray(spec.transform)) out.transform = spec.transform.map((s0) => { + if (!s0 || typeof s0 !== "object") return s0; + const s = { ...s0, op: low(s0.op ?? s0.type) }; + if (s.op === "groupby" || s.op === "aggregate") s.op = "group"; + if (s.aggregate != null && s.agg == null) s.agg = s.aggregate; + if (s.groupby != null && s.by == null) s.by = s.groupby; + if (s.groupBy != null && s.by == null) s.by = s.groupBy; + if (typeof s.by === "string") s.by = [s.by]; + if (s.agg != null) { s.agg = low(s.agg); s.agg = AGG_ALIAS[s.agg] ?? s.agg; } + if (s.order != null) s.order = low(s.order); + delete s.type; delete s.aggregate; delete s.groupby; delete s.groupBy; + return s; + }); + return out; +} + async function model(env, messages) { const input = { messages, max_tokens: 700, temperature: 0.1 }; let out; @@ -155,7 +179,8 @@ async function model(env, messages) { } catch { out = await env.AI.run(env.AI_MODEL, input); // not every model takes response_format } - return firstObject(out?.response ?? out?.choices?.[0]?.message?.content ?? ""); + const raw = out?.response ?? out?.choices?.[0]?.message?.content ?? ""; + return { spec: normalizeSpec(firstObject(raw)), raw: typeof raw === "string" ? raw : JSON.stringify(raw) }; } // Ask, check, and give the model one chance to fix what the validator found. @@ -166,14 +191,15 @@ async function specFor(env, dataset, prompt, prior) { if (prior && validateSpec(prior, schema).ok) messages.push({ role: "user", content: "Current chart spec (change it if the question builds on it): " + JSON.stringify(prior) }); messages.push({ role: "user", content: prompt }); - let spec = await model(env, messages), v = validateSpec(spec, schema); + let { spec, raw } = await model(env, messages), v = validateSpec(spec, schema); if (!v.ok) { - messages.push({ role: "assistant", content: JSON.stringify(spec) }, + messages.push({ role: "assistant", content: raw.slice(0, 2000) }, { role: "user", content: `That spec fails these checks: ${v.errors.join("; ")}. Answer with the corrected JSON object only.` }); - spec = await model(env, messages); + ({ spec, raw } = await model(env, messages)); v = validateSpec(spec, schema); } - return v.ok ? { spec } : { error: "couldn't turn that into a valid chart: " + v.errors.slice(0, 3).join("; ") }; + // On failure the model's own answer comes back too: it is shown, never run. + return v.ok ? { spec } : { error: "couldn't turn that into a valid chart: " + v.errors.slice(0, 3).join("; "), model_said: raw.slice(0, 800) }; } const newId = () => [...crypto.getRandomValues(new Uint8Array(6))].map((b) => b.toString(36).padStart(2, "0")).join("").slice(0, 10); diff --git a/.github/workflows/labs-aicharts.yml b/.github/workflows/labs-aicharts.yml index 64db756..322d750 100644 --- a/.github/workflows/labs-aicharts.yml +++ b/.github/workflows/labs-aicharts.yml @@ -26,6 +26,7 @@ jobs: TIME_VENTURE_DOMAIN: ${{ vars.TIME_VENTURE_DOMAIN }} - name: smoke run: | + set -o pipefail u="${{ steps.deploy.outputs.url }}" for i in $(seq 1 18); do curl -fsS "${u}api/health" && break; sleep 10; done curl -fsS "${u}api/schema?dataset=orders" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert len(d["sample"])==5, d; print("schema ok")' -- 2.54.0