notplanned/scout.py

#!/usr/bin/env python3
import argparse, datetime as dt, html, json, math, pathlib, urllib.parse, urllib.request, urllib.error

ROOT = pathlib.Path(__file__).resolve().parent

def api(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": "notplanned-radar/0.3",
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
    })
    with urllib.request.urlopen(req, timeout=20) as r:
        return json.load(r)

def esc(v):
    return html.escape(str(v), quote=True)

def iso_age_days(value, now):
    try:
        return max(0, (now - dt.datetime.fromisoformat(value.replace("Z","+00:00"))).days)
    except Exception:
        return 9999

def repo_heat(repo, now):
    age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now)
    activity = 18 if age <= 3 else 15 if age <= 7 else 11 if age <= 30 else 6 if age <= 90 else 2 if age <= 180 else 0
    stars = int(repo.get("stargazers_count") or 0)
    star_score = min(18, round(math.log10(max(stars, 1)) * 3.5))
    issues = int(repo.get("open_issues_count") or 0)
    issue_surface = min(8, round(math.log10(max(issues, 1)) * 2.3))
    return activity + star_score + issue_surface, {
        "repo_activity": activity,
        "repo_scale": star_score,
        "issue_surface": issue_surface,
    }

def repo_discovery_score(repo, now):
    heat, _ = repo_heat(repo, now)
    archived_bonus = 5 if repo.get("archived") else 0
    return heat + archived_bonus

def fetch_repo(full_name):
    return api(f"https://api.github.com/repos/{full_name}")

def discover_repos(cfg, now):
    universe = {}
    errors = []

    for full_name in cfg["seed_repos"]:
        try:
            repo = fetch_repo(full_name)
            repo["_source"] = "seed"
            repo["_source_query"] = None
            universe[repo["full_name"].lower()] = repo
        except Exception as e:
            errors.append({"repo": full_name, "stage": "seed", "error": type(e).__name__})

    discovered = {}
    for query in cfg["discovery_queries"]:
        url = "https://api.github.com/search/repositories?" + urllib.parse.urlencode({
            "q": query,
            "sort": "updated",
            "order": "desc",
            "per_page": 30,
        })
        try:
            result = api(url)
        except Exception as e:
            errors.append({"query": query, "stage": "discover", "error": type(e).__name__})
            continue
        allow_archived = "archived:true" in query
        for repo in result.get("items", []):
            full = repo.get("full_name")
            if not full or not repo.get("has_issues", True) or repo.get("fork"):
                continue
            if int(repo.get("stargazers_count") or 0) < cfg["min_repo_stars"]:
                continue
            push_age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now)
            if not allow_archived and push_age > cfg["max_repo_push_age_days"]:
                continue
            key = full.lower()
            if key in universe:
                continue
            repo["_source"] = "discover"
            repo["_source_query"] = query
            old = discovered.get(key)
            if old is None or repo_discovery_score(repo, now) > repo_discovery_score(old, now):
                discovered[key] = repo

    ranked = sorted(discovered.values(), key=lambda r: (
        repo_discovery_score(r, now),
        r.get("pushed_at") or "",
        int(r.get("stargazers_count") or 0),
    ), reverse=True)[:cfg["max_discovered_repos"]]

    for repo in ranked:
        universe.setdefault(repo["full_name"].lower(), repo)

    repos = list(universe.values())
    repos.sort(key=lambda r: (
        1 if r.get("_source") == "seed" else 0,
        repo_discovery_score(r, now),
        r.get("full_name", "").lower(),
    ), reverse=True)
    repos = repos[:cfg["max_total_repos"]]
    return repos, errors

def issue_score(issue, repo_meta, cfg, now):
    labels = {str(x.get("name","")).strip().lower() for x in issue.get("labels", [])}
    if labels & {x.lower() for x in cfg["exclude_labels"]}:
        return None

    rejection = 0
    if issue.get("state_reason") == "not_planned":
        rejection = 32
    for label, pts in cfg["rejection_labels"].items():
        if label.lower() in labels:
            rejection = max(rejection, int(pts))
    if rejection == 0:
        return None

    closed = issue.get("closed_at") or issue.get("updated_at") or ""
    age = iso_age_days(closed, now)
    recency = 40 if age <= 7 else 30 if age <= 30 else 16 if age <= 90 else 6 if age <= 180 else 0

    comments = min(20, int(issue.get("comments") or 0) * 2)
    reactions = issue.get("reactions") or {}
    reaction_score = min(20, int(reactions.get("total_count") or 0) * 2)

    title = (issue.get("title") or "").lower()
    body = (issue.get("body") or "").lower()
    text = title + "\n" + body[:4000]
    ext = 0
    matched = []
    for word, pts in cfg["externalizable_keywords"].items():
        if word.lower() in text:
            ext += int(pts)
            matched.append(word)
    ext = min(30, ext)

    _, heat_breakdown = repo_heat(repo_meta, now)
    breakdown = {
        "rejection": rejection,
        "recency": recency,
        "engagement": comments + reaction_score,
        "externalizable": ext,
        **heat_breakdown,
    }
    return {
        "score": sum(breakdown.values()),
        "breakdown": breakdown,
        "age_days": age,
        "matched": matched[:8],
    }

def category(title):
    t = title.lower()
    buckets = [
        ("browser", ["browser","upload","preview","localhost","tab","form"]),
        ("context", ["context","compact","session","memory","history","resume"]),
        ("authority", ["permission","approval","sandbox","security","guardrail","oauth","auth"]),
        ("integration", ["mcp","api","webhook","integration","plugin","extension","sync"]),
        ("interface", ["ui","sidebar","dashboard","preset","menu","export","import"]),
        ("workflow", ["task","project","workflow","plan","goal","schedule","cron","hook"]),
    ]
    for name, words in buckets:
        if any(w in t for w in words):
            return name
    return "wrapper"

def move_for(cat):
    return {
        "browser":"external browser sidecar / extension",
        "context":"session/context sidecar",
        "authority":"policy proxy / scoped authority layer",
        "integration":"adapter / MCP / integration bridge",
        "interface":"thin UI / control surface",
        "workflow":"supervisor / workflow wrapper",
        "wrapper":"thin wrapper around existing capability",
    }[cat]

def scan():
    cfg = json.loads((ROOT/"radar.json").read_text())
    now = dt.datetime.now(dt.timezone.utc)
    repos, discovery_errors = discover_repos(cfg, now)
    rows = []
    issue_errors = []

    for repo in repos:
        repo_name = repo["full_name"]
        try:
            issues = api(f"https://api.github.com/repos/{repo_name}/issues?state=closed&sort=updated&direction=desc&per_page=100")
        except Exception as e:
            issue_errors.append({"repo": repo_name, "stage": "issues", "error": type(e).__name__})
            continue
        for issue in issues:
            if "pull_request" in issue:
                continue
            scored = issue_score(issue, repo, cfg, now)
            if not scored:
                continue
            cat = category(issue.get("title",""))
            rows.append({
                "above_threshold": scored["score"] >= cfg["threshold"],
                "repo": repo_name,
                "repo_source": repo.get("_source", "unknown"),
                "repo_source_query": repo.get("_source_query"),
                "repo_stars": int(repo.get("stargazers_count") or 0),
                "repo_archived": bool(repo.get("archived")),
                "number": issue.get("number"),
                "title": issue.get("title",""),
                "html_url": issue.get("html_url",""),
                "closed_at": issue.get("closed_at") or "",
                "category": cat,
                "move": move_for(cat),
                **scored,
            })

    rows.sort(key=lambda x: (
        x["score"], x["closed_at"], x["repo_stars"], x["repo"], x["number"]
    ), reverse=True)

    snapshot = {
        "generated_at": now.isoformat(),
        "rules": {
            "threshold": cfg["threshold"],
            "limit": cfg["limit"],
            "seed_repos": cfg["seed_repos"],
            "discovery_queries": cfg["discovery_queries"],
            "max_total_repos": cfg["max_total_repos"],
            "max_discovered_repos": cfg["max_discovered_repos"],
        },
        "universe": [{
            "repo": r["full_name"],
            "source": r.get("_source"),
            "source_query": r.get("_source_query"),
            "stars": int(r.get("stargazers_count") or 0),
            "archived": bool(r.get("archived")),
            "pushed_at": r.get("pushed_at"),
        } for r in repos],
        "candidates": rows[:cfg["limit"]],
        "errors": discovery_errors + issue_errors,
    }
    return cfg, snapshot

def curated_card(o):
    return f"""<a class="curated-card" href="{esc(o['source'])}" target="_blank" rel="noreferrer">
      <div class="topline"><span class="repo">{esc(o['repo'])} · #{esc(o['issue'])}</span><span class="heat {esc(o['heat'])}">{esc(o['heat'])}</span></div>
      <h3>{esc(o['name'])}</h3>
      <div class="meta">{esc(o['title'])}</div>
      <p class="thesis">{esc(o['thesis'])}</p>
      <div class="why"><strong>move:</strong> {esc(o['move'])}</div>
    </a>"""

def radar_card(o):
    b = o["breakdown"]
    why = f"rej {b['rejection']} · rec {b['recency']} · eng {b['engagement']} · ext {b['externalizable']} · heat {b['repo_activity']+b['repo_scale']+b['issue_surface']}"
    src = "seed" if o["repo_source"] == "seed" else "discovered"
    arch = " · archived" if o["repo_archived"] else ""
    return f"""<a class="card" href="{esc(o['html_url'])}" target="_blank" rel="noreferrer">
      <div class="topline"><span class="repo">{esc(o['repo'])} · #{o['number']}</span><span class="score">{o['score']}</span></div>
      <h3>{esc(o['title'])}</h3>
      <div class="meta">{esc(o['category'])} · {src}{arch} · closed {o['age_days']}d ago</div>
      <p class="move">{esc(o['move'])}</p>
      <div class="why">{esc(why)}</div>
    </a>"""

def render(out_path, json_out=None):
    cfg, snapshot = scan()
    curated_data = json.loads((ROOT/"opportunities.json").read_text())
    curated = curated_data.get("opportunities", [])
    template = (ROOT/"template.html").read_text()
    now = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M")
    page = (template
      .replace("__COUNT__", str(len(snapshot["candidates"])))
      .replace("__UPDATED__", now)
      .replace("__THRESHOLD__", str(cfg["threshold"]))
      .replace("__REPOS__", str(len(snapshot["universe"])))
      .replace("__SEEDS__", str(sum(1 for x in snapshot["universe"] if x["source"] == "seed")))
      .replace("__DISCOVERED__", str(sum(1 for x in snapshot["universe"] if x["source"] == "discover")))
      .replace("__CURATED_COUNT__", str(len(curated)))
      .replace("__CURATED_CARDS__", "\n".join(curated_card(x) for x in curated))
      .replace("__CARDS__", "\n".join(radar_card(x) for x in snapshot["candidates"]) if snapshot["candidates"] else '<div class="empty">No candidate cleared the deterministic threshold on this pass.</div>')
    )
    pathlib.Path(out_path).write_text(page)
    if json_out:
        pathlib.Path(json_out).write_text(json.dumps(snapshot, indent=2) + "\n")
    print(json.dumps({
        "candidates": len(snapshot["candidates"]),
        "repos": len(snapshot["universe"]),
        "seed": sum(1 for x in snapshot["universe"] if x["source"] == "seed"),
        "discovered": sum(1 for x in snapshot["universe"] if x["source"] == "discover"),
        "curated": len(curated),
        "errors": len(snapshot["errors"]),
        "out": str(out_path),
    }))

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("--out", default=str(ROOT/"index.html"))
    p.add_argument("--json-out", default="")
    a = p.parse_args()
    render(a.out, a.json_out or None)