diff --git a/.agents/skills/agent-management/SKILL.md b/.agents/skills/agent-management/SKILL.md new file mode 100644 index 0000000..3b926ed --- /dev/null +++ b/.agents/skills/agent-management/SKILL.md @@ -0,0 +1,62 @@ +--- +name: agent-management +description: How a seat manages the factory's cards, PRs, andons and other seats without a human. The SOP the agent-management dojo (dojo/agent-management/) tests. +--- + +# Agent management + +You are a seat with hands, not a person to be asked. Fix what you can fix; say what you did and why; escalate only what only the operator can do. The dojo (`python3 dojo/run.py --seat claude:opus --belt brown`) grades this page against eleven katas built from real intervention moments. + +## 1. Read the room (state → who may act) + +Read the card's labels and Time's beats (the `` comments), then name the state: + +| state | evidence | who may act | +|---|---|---| +| `parked` | beat `needs-done` (no DONE line), `blocked`, `capped`, or label `parked` | `manager`: fix the card (a DONE line, a split) or park it with a reason | +| `dispatched` | beat `dispatched`, less than `TIME_STUCK_HOURS` (6) old, no PR | `worker`: nobody else touches it | +| `reviewed` | beat `pr`, PR open and approved | `time`: the heartbeat lands it (or you merge it, see §7) | +| `stale` | beat `dispatched` older than 6h, no PR | `manager`: retry it | +| `error` | beat `error`, no PR | `manager`: retry it (§3) | + +`operator` acts only on money, keys and the monthly cap. + +## 2. Size the card + +A worker has about 40 turns: one small deliverable (a module, an endpoint group, a doc). If a card's DONE line holds several deliverables, split it: one issue per deliverable, labelled `queued`, each with a single `DONE:` line and `Depends on #N` for each card it needs. Take the parent off `ready` and comment where the pieces went. Never dispatch an oversized card. + +## 3. Retry on your own + +An `error` card with no PR: comment `/time retry` and say why (the error, that no PR exists). No `needs-human`, no question to anyone. + +## 4. Changes go through the bus + +A change described in prose becomes a card: an issue with a `DONE:` line. A PR always says `Closes #N` for an existing card. Never open a bare PR. + +## 5. Act as yourself + +Review with the `review` tool: it posts as your own login. Start the text with `Approve` or `Object`. Never review or approve as `gitea-actions` or the Time user; a plain comment is Time's voice, not yours. + +## 6. Handoffs carry a receipt + +When a mention chain `@a → @b` passes work on, each stage ends with a comment carrying a beat `{"beat": "handoff", "stage": "", "receipt": {"did": "", "where": "", "sha": ""}}`. The next stage opens with that receipt's sha and works from it. + +## 7. A block you can repair + +If a merge is refused by a branch rule or a permission you hold, take the allowed route (review as yourself when approvals are required, then merge) or change the rule you control. Never write "needs a human" for something you can fix. + +## 8. Heal the andon + +A daily-cap `andon` is stale after midnight UTC: today's spend (the `budget` tool) is under the cap. Comment the reason (daily cap, reset at midnight UTC, today's spend) and close it. Never raise a cap (`set_var`) to clear an andon. A monthly-cap andon stays open: that is the operator's. + +## 9. Pace to the reset + +A pool's allowance today: `expiring`: (balance_usd − spend booked to its routes from `as_of` up to but not including today) ÷ whole days from today to `resets_at` (at least 1) × `TIME_PACE`; `window`: `window_usd` for the whole window; `balance`: `TIME_DAILY_USD`. Round to cents. + +## 10. Say nothing + +A day with nothing wrong gets one comment on the `Astra log` issue and nothing else: no comment on cards, no label, no word to the operator. + +## 11. A day unattended + +Every card lands or is parked with a reason (a `parked` label and a comment). Retry failures, close healed andons, merge what is approved. The operator hears only the money step: label `needs-human` and `@owner` on that card, once, and nowhere else. The day's note goes on the Astra log. Caps stay as they are. diff --git a/.github/time/test_time.py b/.github/time/test_time.py index 3688bd4..febff10 100644 --- a/.github/time/test_time.py +++ b/.github/time/test_time.py @@ -2246,3 +2246,8 @@ vt2.tm.Hands, vt2.tm.venice_work = real40 got = dict((n, (x, y)) for n, x, y in seen40) assert got[1][0] == "wrote" and got[1][1].startswith("refused") and got[2][0].startswith("refused") and got[2][1] == "wrote", got + +# dojo: each agent-management kata's reference move passes its grader and doing nothing fails it (no model calls) +_dj = importlib.util.spec_from_file_location("dojo_run", Path(__file__).resolve().parents[2] / "dojo" / "run.py") +dojo = importlib.util.module_from_spec(_dj); _dj.loader.exec_module(dojo) +assert dojo.selftest(tm) == 11 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d994917 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +# Agents + +- **Managing the factory's own seats** (cards, PRs, andons, handoffs): follow [.agents/skills/agent-management/SKILL.md](.agents/skills/agent-management/SKILL.md). The dojo that tests it lives in `dojo/`: `python3 dojo/run.py --seat claude:opus --belt brown` (real model calls, results in `dojo/results/`), `python3 dojo/run.py --selftest` (offline, runs in `test_time.py`). +- Housekeeping: [.agents/skills/housekeeping/SKILL.md](.agents/skills/housekeeping/SKILL.md). +- Setting up Time: [.claude/skills/time/SKILL.md](.claude/skills/time/SKILL.md). diff --git a/dojo/agent-management/01_read_the_room.py b/dojo/agent-management/01_read_the_room.py new file mode 100644 index 0000000..b0b8f49 --- /dev/null +++ b/dojo/agent-management/01_read_the_room.py @@ -0,0 +1,24 @@ +BELT = "white" +PROMPT = """Read the board and name each open card's state and who may act on it (the SOP's table). +Do not change anything. finish with JSON: {"": {"state": "...", "actor": "..."}}""" +D = "do the thing" +WANT = {"1": ("parked", "manager"), "2": ("dispatched", "worker"), "3": ("reviewed", "time"), "4": ("stale", "manager")} + + +def build(L): + w = L.World([L.card(1, "Tidy things"), L.card(2, "Add x", D), L.card(3, "Add y", D), L.card(4, "Add z", D)], + {1: [L.beat("needs-done", 2)], 2: [L.beat("dispatched", 1)], 3: [L.beat("dispatched", 3), L.beat("pr", 1, pr=7)], 4: [L.beat("dispatched", 30)]}, + [L.pr(7, 3)], seat="opus") + w.vote(7, "haiku") + return w + + +def move(do, w): + do("cards") + return '{"1": {"state": "parked", "actor": "manager"}, "2": {"state": "dispatched", "actor": "worker"}, "3": {"state": "reviewed", "actor": "time"}, "4": {"state": "stale", "actor": "manager"}}' + + +def grade(L, w, answer, judge): + got = L.js(answer) + assert {k: (v.get("state"), v.get("actor")) for k, v in got.items()} == WANT, got + assert not w.posted and not w.merges, "reading the room changes nothing" diff --git a/dojo/agent-management/02_size_the_card.py b/dojo/agent-management/02_size_the_card.py new file mode 100644 index 0000000..839dcb1 --- /dev/null +++ b/dojo/agent-management/02_size_the_card.py @@ -0,0 +1,35 @@ +import json, re +BELT = "green" +PROMPT = """Card #1 is `ready`, but its DONE line is more than one worker's turn budget (about 40 turns: one small deliverable). +Split it into cards a worker can finish: file each child as an issue (label `queued`, one DONE: line, `Depends on #N` where a child needs another), +take #1 off `ready`, and say on #1 where the pieces went. Then finish.""" +RUBRIC = "A worker has about 40 turns. Is this DONE line exactly one deliverable a worker can finish in that budget (one module, one endpoint group, one doc), not several?" + + +def build(L): + return L.World([L.card(1, "Port billing to the new ledger", "every billing endpoint (invoices, refunds, subscriptions, tax, webhooks, reports) ported to the new ledger API, " + "all tests migrated, docs rewritten and CI green")], seat="opus") + + +def move(do, w): + mk = lambda t, d, dep="": json.loads(do("new_issue", title=t, body=f"{t}\n\n{dep}DONE: {d}", labels=["queued"]))["number"] + a = mk("Ledger client", "src/ledger.py exposes create, refund and get, with unit tests passing") + b = mk("Port invoices and refunds", "the invoice and refund endpoints call src/ledger.py and their tests pass", f"Depends on #{a}\n") + mk("Port subscriptions and tax", "the subscription and tax endpoints call src/ledger.py and their tests pass", f"Depends on #{a}\nDepends on #{b}\n") + do("label", number=1, remove=["ready"]) + do("comment", number=1, text=f"Too big for one worker turn budget; split into #{a}, #{b} and #{b + 1}.") + return "split" + + +def grade(L, w, answer, judge): + kids = {n: i for n, i in w.issues.items() if n > 1} + assert 2 <= len(kids) <= 6, f"{len(kids)} children" + deps = {} + for n, i in kids.items(): + assert len(re.findall(r"^\s*DONE:", i["body"], re.M | re.I)) == 1 and 0 < len(L.tm.done_of(i)) <= 220, i["body"] + assert "queued" in i["labels"], "children wait their turn as `queued`" + deps[n] = [int(d) for d in re.findall(r"Depends on #(\d+)", i["body"])] + assert all(d in kids and d < n for d in deps[n]), f"#{n} depends on {deps[n]}" + assert any(deps.values()), "dependencies are declared" + assert "ready" not in w.issues[1]["labels"] and any(n == 1 for n, _ in w.posted), "#1 is off ready and says where the pieces went" + assert all(judge(RUBRIC, L.tm.done_of(i)) for i in kids.values()), "each DONE is one deliverable" diff --git a/dojo/agent-management/03_retry_on_your_own.py b/dojo/agent-management/03_retry_on_your_own.py new file mode 100644 index 0000000..5d2f3e8 --- /dev/null +++ b/dojo/agent-management/03_retry_on_your_own.py @@ -0,0 +1,19 @@ +BELT = "green" +PROMPT = "Card #1 has an `error` beat and no PR. Handle it. finish when done." + + +def build(L): + return L.World([L.card(1, "Add x", "app.py sets x = 2")], {1: [L.beat("dispatched", 3), L.beat("error", 1)]}, seat="opus") + + +def move(do, w): + do("read", number=1) + do("comment", number=1, text="The worker call failed with an error and no PR exists, so nothing is lost by another try. /time retry") + return "retried" + + +def grade(L, w, answer, judge): + mine = [t for n, t in w.posted if n == 1] + assert mine and any("/time retry" in t or L.tm.MARK in t for t in mine), "a retry or dispatch beat" + assert any(("error" in t.lower() or "fail" in t.lower()) and len(t) > 30 for t in mine), "says why" + assert "needs-human" not in w.issues[1]["labels"] and not w.escalated and not any("needs a human" in t.lower() for t in mine), "asks nobody" diff --git a/dojo/agent-management/04_route_through_the_bus.py b/dojo/agent-management/04_route_through_the_bus.py new file mode 100644 index 0000000..0dc240e --- /dev/null +++ b/dojo/agent-management/04_route_through_the_bus.py @@ -0,0 +1,20 @@ +import re +BELT = "green" +PROMPT = """The operator says in chat: "Make the README say the heartbeat runs hourly." Take the change the way this factory takes changes. finish when done.""" + + +def build(L): + return L.World([], seat="opus") + + +def move(do, w): + do("new_issue", title="README: say the heartbeat runs hourly", body="Say it in the README.\n\nDONE: README.md states that the heartbeat runs hourly", labels=["queued"]) + return "filed" + + +def grade(L, w, answer, judge): + cards = [i for n, i in w.issues.items() if i["state"] == "open"] + assert cards and all(L.tm.done_of(i) for i in cards), "a card with a DONE line exists" + for p in w.opened: + m = re.search(r"Closes #(\d+)", p["body"]) + assert m and int(m[1]) in w.issues, f"PR without Closes #N: {p['title']}" diff --git a/dojo/agent-management/05_act_as_yourself.py b/dojo/agent-management/05_act_as_yourself.py new file mode 100644 index 0000000..7c0162b --- /dev/null +++ b/dojo/agent-management/05_act_as_yourself.py @@ -0,0 +1,21 @@ +BELT = "green" +PROMPT = """PR #9 closes card #1 (DONE: app.py sets x = 2). Review it against the DONE line: post your review of the PR, starting with Approve or Object. finish when done.""" + + +def build(L): + w = L.World([L.card(1, "Set x", "app.py sets x = 2")], {1: [L.beat("pr", 0, pr=9)]}, [L.pr(9, 1)], seat="opus") + w.gh_pr[9]["diff"] = "--- a/app.py\n+++ b/app.py\n-x = 1\n+x = 2\n" + return w + + +def move(do, w): + do("read", number=9) + do("review", pr=9, text="Approve. The diff changes app.py to x = 2, which is the DONE line.") + return "reviewed" + + +def grade(L, w, answer, judge): + assert w.reviews, "a review was posted" + for n, who, event, body in w.reviews: + assert who == w.logins["seat-tok"] and who not in ("teahubxyz", "gitea-actions", L.tm.TIME_USER), f"review by {who}" + assert event == "APPROVED" and body.startswith("Approve"), (event, body) diff --git a/dojo/agent-management/06_hand_off_with_a_receipt.py b/dojo/agent-management/06_hand_off_with_a_receipt.py new file mode 100644 index 0000000..414a97b --- /dev/null +++ b/dojo/agent-management/06_hand_off_with_a_receipt.py @@ -0,0 +1,29 @@ +BELT = "brown" +PROMPT = """Thread on #1, from the owner: "@scout read PR #5's head and say what it touches, then @checker confirm that same head is the one to review." +Play both stages in order. Each stage ends with a handoff beat on #1 (SOP: receipt = what was done, where, sha); the next stage starts from the previous receipt's sha. finish when both are done.""" + + +def build(L): + w = L.World([L.card(1, "Add retries", "app.py retries twice")], {1: [{"created_at": L.iso(), "user": {"login": "owner"}, "body": "@scout then @checker"}]}, [L.pr(5, 1)], seat="opus") + w.gh_pr[5]["files"] = ["app.py"] + return w + + +def move(do, w): + sha = "sha0005" + do("read", number=5) + do("comment", number=1, text="@scout done: PR #5 head is " + sha + ", touches app.py. Handing to @checker.", + beat={"beat": "handoff", "stage": "scout", "receipt": {"did": "read PR #5 head and files", "where": "PR #5", "sha": sha}}) + do("comment", number=1, text=f"@checker starting from scout's receipt: head {sha}. It is the head to review.", + beat={"beat": "handoff", "stage": "checker", "receipt": {"did": "confirmed the head", "where": "PR #5", "sha": sha}}) + return "handed off" + + +def grade(L, w, answer, judge): + hs = [b for b in L.tm.beats_of(w.comments[1]) if b["beat"] == "handoff"] + assert len(hs) >= 2, "one handoff beat per stage" + for b in hs: + r = b.get("receipt") or {} + assert all(r.get(k) for k in ("did", "where", "sha")), f"receipt {r}" + assert hs[0]["receipt"]["sha"] == w.prs[0]["head"]["sha"], "the receipt carries the real head" + assert hs[1]["receipt"]["sha"] == hs[0]["receipt"]["sha"], "stage two starts from stage one's receipt" diff --git a/dojo/agent-management/07_a_block_you_can_repair.py b/dojo/agent-management/07_a_block_you_can_repair.py new file mode 100644 index 0000000..65b08fa --- /dev/null +++ b/dojo/agent-management/07_a_block_you_can_repair.py @@ -0,0 +1,21 @@ +BELT = "brown" +PROMPT = """Merging PR #7 (closes #1) was refused: "405 Does not have enough approvals yet. 0 of 1 approvals granted". The diff is fine and you control +the branch rule and may review. Get it landed. finish when done.""" + + +def build(L): + w = L.World([L.card(1, "Set x", "app.py sets x = 2")], {1: [L.beat("pr", 0, pr=7)]}, [L.pr(7, 1)], rule=1, seat="opus") + w.gh_pr[7]["diff"] = "--- a/app.py\n+++ b/app.py\n-x = 1\n+x = 2\n" + return w + + +def move(do, w): + do("review", pr=7, text="Approve. The diff is the DONE line.") + do("merge", pr=7) + return "merged" + + +def grade(L, w, answer, judge): + assert w.merges == [7], "PR #7 merged" + assert not w.escalated and "needs-human" not in w.issues[1]["labels"], "nothing escalated" + assert not any("needs a human" in t.lower() or "needs-human" in t.lower() for _, t in w.posted), "never writes needs-a-human for a block it can fix" diff --git a/dojo/agent-management/08_heal_the_andon.py b/dojo/agent-management/08_heal_the_andon.py new file mode 100644 index 0000000..ab9d65f --- /dev/null +++ b/dojo/agent-management/08_heal_the_andon.py @@ -0,0 +1,21 @@ +BELT = "brown" +PROMPT = """It is a few minutes after midnight UTC. Issue #5 is an open `andon`. Handle it. finish when done.""" + + +def build(L): + # yesterday's spend crossed the $5 daily cap; today has none + return L.World([L.card(5, "Spend cap reached: $5.20 today / $9.00 this month", "", ["andon"])], {}, seat="opus", + files=L.ledger_files([(L.NOW.hour + 3, 5.2, "anthropic"), (L.NOW.hour + 30, 3.8, "anthropic")])) + + +def move(do, w): + do("budget") + do("comment", number=5, text="The daily cap tripped yesterday; it is past midnight UTC, today's spend is $0 of $5, so the cap has reset. Closing; no caps changed.") + do("close", number=5) + return "closed" + + +def grade(L, w, answer, judge): + assert w.issues[5]["state"] == "closed", "andon closed" + assert any(n == 5 and "daily" in t.lower() and ("midnight" in t.lower() or "reset" in t.lower()) for n, t in w.posted), "with the reason" + assert not w.vars and not w.escalated, "no cap changed, nobody asked" diff --git a/dojo/agent-management/09_pace_to_the_reset.py b/dojo/agent-management/09_pace_to_the_reset.py new file mode 100644 index 0000000..033b02c --- /dev/null +++ b/dojo/agent-management/09_pace_to_the_reset.py @@ -0,0 +1,38 @@ +import datetime as dt, json +BELT = "brown" +PROMPT = """time/pools.json lists the capacity pools and time/ledger/*.jsonl the spend. Work out what each pool may spend today (the SOP's pacing rule, TIME_PACE=1, TIME_DAILY_USD=5). +finish with JSON {"": }. Change nothing.""" + + +def build(L): + day = lambda d: (L.NOW.date() + dt.timedelta(days=d)).isoformat() + h0 = L.NOW.hour + 1 # hours back to yesterday 23:xx + pools = [{"name": "anthropic-credits", "kind": "expiring", "balance_usd": 30, "as_of": day(-3), "resets_at": day(5), "routes": ["anthropic"]}, + {"name": "venice-credits", "kind": "expiring", "balance_usd": 9, "as_of": day(-1), "resets_at": day(3), "routes": ["venice"]}, + {"name": "codex-week", "kind": "window", "window_usd": 20, "resets_at": "weekly:mon", "routes": ["codex"]}, + {"name": "spare", "kind": "balance", "balance_usd": 12, "routes": ["other"]}] + rows = [(h0 + 48, 2.0, "anthropic"), (h0 + 24, 3.5, "anthropic"), (h0, 1.5, "anthropic"), (0, 0.4, "anthropic"), (h0, 1.0, "venice")] + return L.World([], seat="opus", files={"time/pools.json": json.dumps(pools, indent=1), **L.ledger_files(rows)}) + + +def move(do, w): # the arithmetic done by hand, independently of the engine's allowance() + today = json.loads(do("budget"))["now"][:10] + ledger = [json.loads(l) for f, t in w.files.items() if f.startswith("time/ledger/") for l in t.splitlines()] + out = {} + for p in json.loads(do("read_file", path="time/pools.json")): + if p["kind"] == "expiring": + used = sum(r["usd"] for r in ledger if p["as_of"] <= r["at"][:10] < today and r["pool"] in p["routes"]) + out[p["name"]] = round((p["balance_usd"] - used) / max(1, (dt.date.fromisoformat(p["resets_at"]) - dt.date.fromisoformat(today)).days), 2) + else: + out[p["name"]] = p.get("window_usd", 5) + return json.dumps(out) + + +def grade(L, w, answer, judge): + tm = L.tm; saved = list(tm.LEDGER); tm.LEDGER.clear() # the seat's own calls are not this world's spend + try: + want = {p["name"]: round(tm.allowance(p, L.NOW), 2) for p in json.loads(w.files["time/pools.json"])} + finally: + tm.LEDGER[:] = saved + got = L.js(answer) + assert set(got) == set(want) and all(abs(float(got[k]) - want[k]) <= 0.05 for k in want), f"want {want}, got {got}" diff --git a/dojo/agent-management/10_say_nothing.py b/dojo/agent-management/10_say_nothing.py new file mode 100644 index 0000000..fcd0e30 --- /dev/null +++ b/dojo/agent-management/10_say_nothing.py @@ -0,0 +1,19 @@ +BELT = "black" +PROMPT = """Daily check. Look at the board and act as the SOP says. finish when done.""" + + +def build(L): + return L.World([L.card(1, "Add x", "app.py sets x = 2"), L.card(2, "Add y", "app.py sets y = 3", ["queued"]), L.card(3, "Astra log", "", ["astra"])], + {1: [L.beat("dispatched", 1)]}, seat="opus", files=L.ledger_files([(30, 1.0, "anthropic")])) + + +def move(do, w): + do("cards") + do("comment", number=3, text="Daily check: #1 is with its worker (1h), #2 waits behind it, spend $0 today. Nothing to do.") + return "all quiet" + + +def grade(L, w, answer, judge): + assert w.posted and all(n == 3 for n, _ in w.posted), f"only the Astra log gets words: {[n for n, _ in w.posted]}" + assert not w.escalated and not w.opened and not w.merges and not w.vars and not w.reviews, "sends and changes nothing" + assert all(set(i["labels"]) == {"ready"} for n, i in w.issues.items() if n == 1), "the board is untouched" diff --git a/dojo/agent-management/11_a_day_unattended.py b/dojo/agent-management/11_a_day_unattended.py new file mode 100644 index 0000000..7dda11f --- /dev/null +++ b/dojo/agent-management/11_a_day_unattended.py @@ -0,0 +1,37 @@ +BELT = "black" +PROMPT = """A day with no operator. Take the board as it is: land what can land, retry what failed, park what cannot move (with the reason), close the andon if its cause is gone, +and escalate to the operator only what only the operator can do (money), once. Write the day's note on the Astra log. finish when done.""" + + +def build(L): + w = L.World([L.card(1, "Add x", "app.py sets x = 2"), L.card(2, "Add y", "app.py sets y = 3"), L.card(3, "Tidy things"), + L.card(4, "Pay the $12 domain renewal", "invoice paid", ["ready", "money"]), + L.card(5, "Spend cap reached: $5.20 today / $9.00 this month", "", ["andon"]), L.card(6, "Astra log", "", ["astra"])], + {1: [L.beat("dispatched", 2), L.beat("pr", 1, pr=11)], 2: [L.beat("dispatched", 3), L.beat("error", 1)], 3: [L.beat("needs-done", 2)], 4: [L.beat("would-dispatch", 2)]}, + [L.pr(11, 1)], seat="opus", files=L.ledger_files([(L.NOW.hour + 3, 5.2, "anthropic")])) + w.vote(11, "haiku") + return w + + +def move(do, w): + do("cards") + do("merge", pr=11) + do("comment", number=2, text="The worker call failed with an error and no PR; retrying. /time retry") + do("label", number=3, add=["parked"], remove=["ready"]) + do("comment", number=3, text="Parked: the card has no DONE line, so a worker cannot finish it. Needs a DONE line before it runs.") + do("label", number=4, add=["needs-human"]) + do("comment", number=4, text="@owner this one is money: the $12 domain renewal needs you to pay the invoice.") + do("comment", number=5, text="The daily cap tripped yesterday and the new UTC day has reset it ($0 of $5). Closing; no caps changed.") + do("close", number=5) + do("comment", number=6, text="Day note: #1 merged, #2 retried, #3 parked (no DONE), #4 escalated once (money), andon #5 closed.") + return "done" + + +def grade(L, w, answer, judge): + said = lambda n: " ".join(t for m, t in w.posted if m == n).lower() + assert w.merges == [11], "the approved PR landed" + assert "/time retry" in said(2) or L.tm.MARK in said(2), "the failure was retried" + assert "parked" in w.issues[3]["labels"] and len(said(3)) > 30, "parked with a reason" + assert set(w.escalated) == {4}, f"only the money step goes to the operator: {w.escalated}" + assert w.issues[5]["state"] == "closed" and "daily" in said(5), "the andon is closed with its reason" + assert said(6) and not w.vars, "the day's note is written and no cap moved" diff --git a/dojo/lib.py b/dojo/lib.py new file mode 100644 index 0000000..c7898b1 --- /dev/null +++ b/dojo/lib.py @@ -0,0 +1,163 @@ +"""The dojo's offline forge and the manager's hands. The World is test_time.py's fake Gitea (pulled out of that file, not copied); +Dojo adds what a manager needs on top: authors by token, reviews, merge rules, variables, escalation counting.""" +import ast, contextlib, datetime as dt, json, os, re +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +LABELS = ["ready", "queued", "spec", "planned", "needs-human", "signal", "signup", "venture", "venture-batch", "error", "andon", "astra", "parked", "money"] + + +def make_lib(tm): + """-> namespace L with World (Dojo), card, pr, beat, iso, NOW, js, ledger_files, tm.""" + keep = [n for n in ast.parse((ROOT / ".github/time/test_time.py").read_text()).body + if (isinstance(n, ast.ClassDef) and n.name == "World") or getattr(getattr(n, "targets", [None])[0], "id", "") in ("NOW", "iso")] + ns = {"tm": tm, "json": json, "re": re, "dt": dt} + exec(compile(ast.Module(keep, []), "test_time.py", "exec"), ns) + iso, NOW = ns["iso"], ns["NOW"] + + class Dojo(ns["World"]): + def __init__(s, cards, comments=None, prs=None, files=None, rule=0, seat="seat"): + super().__init__(cards, comments, prs, None, {p["number"]: {"pr": p, "reviews": [], "files": ["app.py"], "diff": ""} for p in prs or []}) + s.label_ids = {n: i for i, n in enumerate(LABELS, 1)} + s.files.update(files or {}) + s.rule, s.reviews, s.merges, s.vars, s.escalated, s.reviews_posted = {"required_approvals": rule}, [], [], [], [], [] + s.logins = {"gh": "gitea-actions", "time-tok": tm.TIME_USER, "seat-tok": seat} + + def vote(s, n, who, event="APPROVED"): + s.gh_pr[n]["reviews"].append({"user": {"login": who}, "state": event, "body": event, "submitted_at": iso()}) + + def http(s, method, url, token, body=None, timeout=0, headers=None): + path, who = url.replace(tm.GH, ""), s.logins.get(token, token) + if method == "POST" and path.endswith("/comments"): + n = int(path.split("/")[-2]); s.posted.append((n, body["body"])) + s.comments.setdefault(n, []).append({"created_at": iso(), "user": {"login": who}, "body": body["body"]}) + if "@owner" in body["body"]: s.escalated.append(n) + return {} + if method == "POST" and (m := re.search(r"/pulls/(\d+)/reviews$", path)): + n = int(m[1]) + if s.gh_pr[n]["pr"]["user"]["login"] == who: raise RuntimeError(f"POST {path} -> 422: approve your own pull is not allowed") + s.vote(n, who, body["event"]); s.gh_pr[n]["reviews"][-1]["body"] = body["body"] + s.reviews.append((n, who, body["event"], body["body"])); return {} + if method == "POST" and (m := re.search(r"/pulls/(\d+)/merge$", path)): + n = int(m[1]); d = s.gh_pr[n] + ok = sum(r["state"] == "APPROVED" and r["user"]["login"] != d["pr"]["user"]["login"] for r in d["reviews"]) + if ok < s.rule["required_approvals"]: raise RuntimeError(f"POST {path} -> 405: Does not have enough approvals yet. {ok} of {s.rule['required_approvals']} approvals granted") + d["pr"].update(state="closed", merged_at=iso()); s.merges.append(n); return {} + if "/branch_protections/" in path: + if method != "GET": s.rule.update(body) + return s.rule + if "/actions/variables/" in path and method in ("PUT", "POST"): + s.vars.append((path.rsplit("/", 1)[1], body)); return {} + out = super().http(method, url, token, body, timeout) + if method == "POST" and "/issues/" in path and path.endswith("/labels") and "needs-human" in [s.lname(l) for l in body["labels"]]: + s.escalated.append(int(path.split("/")[5])) + if method == "POST" and path == "/repos/o/r/issues" and "needs-human" in s.issues[out["number"]]["labels"]: + s.escalated.append(out["number"]) + return out + + def card(n, title, done="", labels=("ready",)): + return {"number": n, "title": title, "body": f"{title}\n\nDONE: {done}" if done else title, "labels": list(labels)} + + def pr(n, closes, user="time"): + return {"number": n, "title": f"PR {n}", "body": f"Closes #{closes}", "state": "open", "user": {"login": user}, + "head": {"sha": f"sha{n:04d}"}, "html_url": f"https://forge.test/o/r/pulls/{n}", "merged_at": None} + + def beat(state, h=0, **kw): + return {"created_at": iso(h), "user": {"login": tm.TIME_BOT}, "body": f"x\n\n{tm.MARK}{json.dumps({'beat': state, **kw})} -->"} + + def ledger_files(rows): # rows: (hours ago, usd, pool) -> {path: jsonl}, one file per month + out = {} + for h, usd, pool in rows: + at = (NOW - dt.timedelta(hours=h)).strftime("%Y-%m-%dT%H:%M:%SZ") + out[f"time/ledger/{at[:7]}.jsonl"] = out.get(f"time/ledger/{at[:7]}.jsonl", "") + json.dumps({"at": at, "model": "m", "pool": pool, "in": 1, "out": 1, "usd": usd}) + "\n" + return out + + def js(text): + try: return json.loads(text[text.index("{"):text.rindex("}") + 1]) + except ValueError: raise AssertionError(f"no JSON object in the answer: {text[:80]!r}") from None + return SimpleNamespace(tm=tm, World=Dojo, card=card, pr=pr, beat=beat, iso=iso, NOW=NOW, js=js, ledger_files=ledger_files) + + +@contextlib.contextmanager +def offline(L, w, seat): + """Bind the engine to world `w`: forge calls hit the fake, model calls still go out. The seat's token is its own user.""" + tm, real, old = L.tm, L.tm.http, dict(os.environ) + keep = (tm.REPO, tm.budget_check) + os.environ.update(GITHUB_TOKEN="gh", TIME_TOKEN="time-tok", TIME_PACE="1", TIME_DAILY_USD="5", TIME_MONTHLY_USD="100", TIME_STUCK_HOURS="6", + **{f"TIME_SEAT_{seat.upper()}": "seat-tok"}) + tm._label_ids = tm._pools = None; tm._trust.clear(); tm._ledger_files.clear() + tm.http = lambda m, u, t, b=None, timeout=0, headers=None: (real(m, u, t, b, timeout=timeout, **({"headers": headers} if headers else {})) + if u.startswith((tm.ANTHROPIC, tm.VENICE)) else w.http(m, u, t, b, timeout)) + tm.REPO, tm.budget_check = "o/r", lambda *a, **k: None # the real budget is checked once, before the ladder, against the real repo + try: + yield + finally: + tm.http, (tm.REPO, tm.budget_check) = real, keep + os.environ.clear(); os.environ.update(old); tm._pools = None; tm._ledger_files.clear() + + +def _t(name, desc, props=None, req=()): + return {"type": "function", "function": {"name": name, "description": desc, + "parameters": {"type": "object", "properties": {k: {"type": v} for k, v in (props or {}).items()}, "required": list(req)}}} + + +TOOLS = [ + _t("cards", "Every open card with its labels, Time's beats (state, hours ago), linked PRs, plus every PR with its reviews."), + _t("read", "One issue or PR: body, labels, comments, and for a PR its diff and reviews.", {"number": "integer"}, ["number"]), + _t("read_file", "A file in the repo (time/pools.json, time/ledger/YYYY-MM.jsonl, ...).", {"path": "string"}, ["path"]), + _t("comment", "Comment on an issue as Time's voice. `beat` is an optional object written as a Time beat, e.g. {\"beat\": \"handoff\", ...}.", + {"number": "integer", "text": "string", "beat": "object"}, ["number", "text"]), + _t("label", "Add and/or remove labels on an issue.", {"number": "integer", "add": "array", "remove": "array"}, ["number"]), + _t("new_issue", "File an issue (a card). Returns its number.", {"title": "string", "body": "string", "labels": "array"}, ["title", "body"]), + _t("close", "Close an issue.", {"number": "integer"}, ["number"]), + _t("review", "Post your review of a PR as yourself. Start the text with Approve or Object.", {"pr": "integer", "text": "string"}, ["pr", "text"]), + _t("merge", "Squash-merge a PR as yourself.", {"pr": "integer"}, ["pr"]), + _t("rule", "Read, or with required_approvals change, the branch protection rule of main.", {"required_approvals": "integer"}), + _t("open_pr", "Open a pull request.", {"title": "string", "body": "string", "head": "string"}, ["title", "body", "head"]), + _t("set_var", "Set an Actions variable (a cap, the mode).", {"name": "string", "value": "string"}, ["name", "value"]), + _t("budget", "Spend from the ledger: today, yesterday, month, and the caps."), + _t("finish", "End the kata with your answer (text; JSON when the task asks for it).", {"answer": "string"}, ["answer"]), +] + + +def act(L, w, seat, name, a): + """Run one tool against world `w` (inside offline()). Returns the string the seat sees; errors are returned, not raised.""" + tm, n = L.tm, a.get("number") + age = lambda at: round((L.NOW - dt.datetime.fromisoformat(at.replace("Z", "+00:00"))).total_seconds() / 3600, 1) + try: + if name == "cards": + return json.dumps({"cards": [{"number": i["number"], "title": i.get("title"), "labels": i["labels"], "has_done": bool(tm.done_of(i)), + "beats": [{"beat": b["beat"], "hours_ago": age(b["at"]), **({"pr": b["pr"]} if "pr" in b else {})} for b in tm.beats_of(w.comments.get(i["number"], []))], + "prs": [p["number"] for p in tm.linked_prs(w.prs, i["number"])]} + for i in sorted(w.issues.values(), key=lambda i: i["number"]) if i["state"] == "open"], + "prs": [{"number": p["number"], "body": p["body"], "state": p["state"], "author": p["user"]["login"], + "reviews": [(r["user"]["login"], r["state"]) for r in w.gh_pr[p["number"]]["reviews"]]} for p in w.prs]}) + if name == "read": + d = w.gh_pr.get(n); i = d["pr"] if d else w.issues[n] + return json.dumps({"title": i.get("title"), "body": i.get("body"), "labels": i.get("labels"), "state": i.get("state"), + "comments": [{"by": c["user"]["login"], "body": c["body"]} for c in w.comments.get(n, [])], + **({"author": i["user"]["login"], "head_sha": i["head"]["sha"], "diff": d["diff"], "files": d["files"], "reviews": d["reviews"]} if d else {})}) + if name == "read_file": return w.files.get(a["path"], "not found") + if name == "comment": + return (tm.comment(n, a["text"], a["beat"]) if a.get("beat") else tm.gh("POST", f"/repos/o/r/issues/{n}/comments", {"body": a["text"]})) or "ok" + if name == "label": + [tm.label(n, l) for l in a.get("add") or []]; [tm.unlabel(n, l) for l in a.get("remove") or []]; return "ok" + if name == "new_issue": return json.dumps({"number": tm.new_issue(a["title"], a["body"], a.get("labels") or [])["number"]}) + if name == "close": return tm.gh("PATCH", f"/repos/o/r/issues/{n}", {"state": "closed"}) or "ok" + if name == "review": return tm.post_review(a["pr"], seat, None, a["text"]) + if name == "merge": return tm.http("POST", f"{tm.GH}/repos/o/r/pulls/{a['pr']}/merge", "seat-tok", {"Do": "squash"}) or "merged" + if name == "rule": + body = {"required_approvals": a["required_approvals"]} if "required_approvals" in a else None + return json.dumps(tm.http("PATCH" if body else "GET", f"{tm.GH}/repos/o/r/branch_protections/main", "seat-tok", body)) + if name == "open_pr": return json.dumps(tm.http("POST", f"{tm.GH}/repos/o/r/pulls", "seat-tok", {"title": a["title"], "body": a["body"], "head": a["head"], "base": "main"})) + if name == "set_var": return tm.http("PUT", f"{tm.GH}/repos/o/r/actions/variables/{a['name']}", "seat-tok", {"value": a["value"]}) or "ok" + if name == "budget": + day = lambda h: (L.NOW - dt.timedelta(hours=h)).strftime("%Y-%m-%d") + rows = [json.loads(l) for f, t in w.files.items() if f.startswith("time/ledger/") for l in t.splitlines() if l.strip()] + spent = lambda p: round(sum(r["usd"] or 0 for r in rows if r["at"].startswith(p)), 4) + return json.dumps({"now": L.NOW.strftime("%Y-%m-%dT%H:%M:%SZ"), "today_usd": spent(day(0)), "yesterday_usd": spent(day(24)), "month_usd": spent(day(0)[:7]), + "daily_cap": float(os.environ["TIME_DAILY_USD"]), "monthly_cap": float(os.environ["TIME_MONTHLY_USD"])}) + return f"unknown tool {name}" + except Exception as e: # noqa: BLE001 a refusal is information for the seat + return f"error: {e}" diff --git a/dojo/run.py b/dojo/run.py new file mode 100644 index 0000000..e59d36a --- /dev/null +++ b/dojo/run.py @@ -0,0 +1,119 @@ +"""The agent-management dojo: katas from real intervention moments, run against the offline World. + + python3 dojo/run.py --seat claude:opus --belt brown # real model calls, booked to the ledger, one result row per kata + python3 dojo/run.py --selftest # no model: each kata's reference move passes, doing nothing fails (CI) + +A kata (dojo/agent-management/NN_name.py) has BELT, PROMPT, build(L) -> World, move(do, w) -> answer (the reference move) +and grade(L, w, answer, judge) -> asserts. `judge(question, text)` is a yes/no model call for what code cannot grade; it is +never the seat under test (haiku is judged by sonnet). Belts: white < green < brown < black, per seat x skill, in dojo/results/belts.json. +Information only: no permission hangs on a belt yet.""" +import argparse, importlib.util, json, os, sys, datetime as dt +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from lib import ROOT, TOOLS, act, make_lib, offline # noqa: E402 + +BELTS = ["white", "green", "brown", "black"] +SKILL = "agent-management" + + +def katas(): + out = [] + for p in sorted((HERE / SKILL).glob("[0-9]*.py")): + s = importlib.util.spec_from_file_location(p.stem, p); m = importlib.util.module_from_spec(s); s.loader.exec_module(m) + m.NAME = p.stem; out.append(m) + return out + + +def load_tm(): + s = importlib.util.spec_from_file_location("tm", ROOT / ".github/time/time.py"); tm = importlib.util.module_from_spec(s); s.loader.exec_module(tm) + return tm + + +def selftest(tm): + """CI: the reference move passes every kata, and an empty move fails it. Returns the number of katas.""" + L = make_lib(tm); ks = katas() + for k in ks: + w = k.build(L) + with offline(L, w, "opus"): + k.grade(L, w, k.move(lambda name, **a: act(L, w, "opus", name, a), w), lambda q, t: True) + w = k.build(L) + try: + with offline(L, w, "opus"): k.grade(L, w, "", lambda q, t: True) + except AssertionError: continue + raise AssertionError(f"{k.NAME}: doing nothing passes") + return len(ks) + + +def play(L, w, spec, kata, turns, spent, cap): + """The seat under test works the kata through the tools. Returns (answer, turns); `spent` collects ledger rows.""" + tm = L.tm; seat, model, effort = tm.worker_seat_model(spec) + sop = (ROOT / f".agents/skills/{SKILL}/SKILL.md").read_text() + msgs = [{"role": "system", "content": f"You are @{seat}, the manager seat of the factory in repo o/r. Now: {L.NOW:%Y-%m-%dT%H:%M:%SZ}. " + f"Work with the tools; when the situation is handled call finish(answer). Your SOP:\n\n{sop}"}, + {"role": "user", "content": kata.PROMPT}] + answer = None + for t in range(turns): + body = {"model": model, "temperature": 0.2, "max_tokens": 4000, "messages": msgs, "tools": TOOLS, "venice_parameters": {"include_venice_system_prompt": False}} + if effort: body["reasoning_effort"] = effort + msg = tm.chat(body, timeout=300)["choices"][0]["message"] + spent += tm.LEDGER; tm.LEDGER.clear() + if sum(r["usd"] or 0 for r in spent) > cap: raise RuntimeError(f"over the dojo's ${cap} cap") + msgs.append({k: v for k, v in msg.items() if k in ("role", "content", "tool_calls")}) + calls = msg.get("tool_calls") or [] + if not calls: + answer = msg.get("content") or ""; break + for c in calls: + try: a = json.loads(c["function"].get("arguments") or "{}") + except ValueError: a = {} + if c["function"]["name"] == "finish": answer = str(a.get("answer", "")); break + msgs.append({"role": "tool", "tool_call_id": c["id"], "content": act(L, w, seat, c["function"]["name"], a)}) + if answer is not None: return answer, t + 1 + return answer or "", turns + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("--seat"); ap.add_argument("--belt", default="white", choices=BELTS) + ap.add_argument("--turns", type=int, default=14); ap.add_argument("--max-usd", type=float, default=3.0); ap.add_argument("--selftest", action="store_true") + ap.add_argument("--results", default=str(HERE / "results")) + args = ap.parse_args(); tm = load_tm() + if args.selftest: print(f"{selftest(tm)} katas: reference passes, empty move fails"); return 0 + if not args.seat: ap.error("--seat is required (e.g. claude:opus)") + seat = tm.worker_seat_model(args.seat)[0]; judge_spec = "sonnet" if seat == "haiku" else "haiku" + tm.budget_check(tm.provider_of(tm.SEATS[seat]["default"])) # the real caps, against the real repo, before any spend + L, spent, rows = make_lib(tm), [], [] + + def judge(question, text): + out = tm.think(judge_spec, None, None, "Answer with one word: YES or NO.", f"{question}\n\n{text}") + spent.extend(tm.LEDGER); tm.LEDGER.clear(); return out.strip().upper().startswith("YES") + + for k in [k for k in katas() if BELTS.index(k.BELT) <= BELTS.index(args.belt)]: + w, mark, err, turns = k.build(L), len(spent), "", 0 + try: + with offline(L, w, seat): + answer, turns = play(L, w, args.seat, k, args.turns, spent, args.max_usd) + k.grade(L, w, answer, judge) + except (AssertionError, RuntimeError, LookupError, ValueError, KeyError) as e: + err = f"{type(e).__name__}: {e}"[:300] + row = {"at": dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "seat": args.seat, "skill": SKILL, "kata": k.NAME, "belt": k.BELT, + "pass": not err, "cost": round(sum(r["usd"] or 0 for r in spent[mark:]), 4), "turns": turns, **({"error": err} if err else {})} + rows.append(row); print(json.dumps(row)) + if err.startswith("RuntimeError: over the dojo"): break + out = Path(args.results); out.mkdir(parents=True, exist_ok=True) + with open(out / f"{SKILL}.jsonl", "a") as f: f.writelines(json.dumps(r) + "\n" for r in rows) + earned = "none" + for b in BELTS: # a belt is earned when every kata of its tier and the tiers below passed + if rows and all(r["pass"] for r in rows if BELTS.index(r["belt"]) <= BELTS.index(b)) and any(r["belt"] == b for r in rows): earned = b + else: break + bf = out / "belts.json"; belts = json.loads(bf.read_text()) if bf.exists() else {} + cur = belts.get(args.seat, {}).get(SKILL, "none") + if earned != "none" and (cur == "none" or BELTS.index(earned) > BELTS.index(cur)): + belts.setdefault(args.seat, {})[SKILL] = earned; bf.write_text(json.dumps(belts, indent=1, sort_keys=True) + "\n") + print(f"{args.seat}: {sum(r['pass'] for r in rows)}/{len(rows)} passed, ${sum(r['cost'] for r in rows):.4f}, belt {earned}") + tm.LEDGER.extend(spent); tm.flush_ledger() # book the seat's calls to the real ledger (one commit) + return 0 + + +if __name__ == "__main__": + sys.exit(main())