diff --git a/scout.py b/scout.py index 9b2ce66..679fb55 100644 --- a/scout.py +++ b/scout.py @@ -1,49 +1,126 @@ #!/usr/bin/env python3 -import argparse, datetime as dt, html, json, math, pathlib, urllib.request, urllib.error +import argparse, datetime as dt, html, json, math, pathlib, urllib.parse, urllib.request, urllib.error ROOT = pathlib.Path(__file__).resolve().parent def api(url): req = urllib.request.Request(url, headers={ - "User-Agent": "notplanned-radar/0.2", + "User-Agent": "notplanned-radar/0.3", "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", }) with urllib.request.urlopen(req, timeout=20) as r: return json.load(r) -def esc(v): return html.escape(str(v), quote=True) +def esc(v): + return html.escape(str(v), quote=True) + +def iso_age_days(value, now): + try: + return max(0, (now - dt.datetime.fromisoformat(value.replace("Z","+00:00"))).days) + except Exception: + return 9999 def repo_heat(repo, now): - pushed = repo.get("pushed_at") or repo.get("updated_at") - try: - age = max(0, (now - dt.datetime.fromisoformat(pushed.replace("Z","+00:00"))).days) - except Exception: - age = 999 - activity = 15 if age <= 7 else 10 if age <= 30 else 5 if age <= 90 else 0 + age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now) + activity = 18 if age <= 3 else 15 if age <= 7 else 11 if age <= 30 else 6 if age <= 90 else 2 if age <= 180 else 0 stars = int(repo.get("stargazers_count") or 0) - star_score = min(15, round(math.log10(max(stars, 1)) * 3.2)) - return activity + star_score, {"repo_activity": activity, "repo_scale": star_score} + star_score = min(18, round(math.log10(max(stars, 1)) * 3.5)) + issues = int(repo.get("open_issues_count") or 0) + issue_surface = min(8, round(math.log10(max(issues, 1)) * 2.3)) + return activity + star_score + issue_surface, { + "repo_activity": activity, + "repo_scale": star_score, + "issue_surface": issue_surface, + } + +def repo_discovery_score(repo, now): + heat, _ = repo_heat(repo, now) + archived_bonus = 5 if repo.get("archived") else 0 + return heat + archived_bonus + +def fetch_repo(full_name): + return api(f"https://api.github.com/repos/{full_name}") + +def discover_repos(cfg, now): + universe = {} + errors = [] + + for full_name in cfg["seed_repos"]: + try: + repo = fetch_repo(full_name) + repo["_source"] = "seed" + repo["_source_query"] = None + universe[repo["full_name"].lower()] = repo + except Exception as e: + errors.append({"repo": full_name, "stage": "seed", "error": type(e).__name__}) + + discovered = {} + for query in cfg["discovery_queries"]: + url = "https://api.github.com/search/repositories?" + urllib.parse.urlencode({ + "q": query, + "sort": "updated", + "order": "desc", + "per_page": 30, + }) + try: + result = api(url) + except Exception as e: + errors.append({"query": query, "stage": "discover", "error": type(e).__name__}) + continue + allow_archived = "archived:true" in query + for repo in result.get("items", []): + full = repo.get("full_name") + if not full or not repo.get("has_issues", True) or repo.get("fork"): + continue + if int(repo.get("stargazers_count") or 0) < cfg["min_repo_stars"]: + continue + push_age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now) + if not allow_archived and push_age > cfg["max_repo_push_age_days"]: + continue + key = full.lower() + if key in universe: + continue + repo["_source"] = "discover" + repo["_source_query"] = query + old = discovered.get(key) + if old is None or repo_discovery_score(repo, now) > repo_discovery_score(old, now): + discovered[key] = repo + + ranked = sorted(discovered.values(), key=lambda r: ( + repo_discovery_score(r, now), + r.get("pushed_at") or "", + int(r.get("stargazers_count") or 0), + ), reverse=True)[:cfg["max_discovered_repos"]] + + for repo in ranked: + universe.setdefault(repo["full_name"].lower(), repo) + + repos = list(universe.values()) + repos.sort(key=lambda r: ( + 1 if r.get("_source") == "seed" else 0, + repo_discovery_score(r, now), + r.get("full_name", "").lower(), + ), reverse=True) + repos = repos[:cfg["max_total_repos"]] + return repos, errors def issue_score(issue, repo_meta, cfg, now): labels = {str(x.get("name","")).strip().lower() for x in issue.get("labels", [])} if labels & {x.lower() for x in cfg["exclude_labels"]}: return None + rejection = 0 - reason = issue.get("state_reason") - if reason == "not_planned": - rejection = max(rejection, 32) + if issue.get("state_reason") == "not_planned": + rejection = 32 for label, pts in cfg["rejection_labels"].items(): if label.lower() in labels: - rejection = max(rejection, pts) + rejection = max(rejection, int(pts)) if rejection == 0: return None - closed = issue.get("closed_at") or issue.get("updated_at") - try: - age = max(0, (now - dt.datetime.fromisoformat(closed.replace("Z","+00:00"))).days) - except Exception: - age = 999 + closed = issue.get("closed_at") or issue.get("updated_at") or "" + age = iso_age_days(closed, now) recency = 40 if age <= 7 else 30 if age <= 30 else 16 if age <= 90 else 6 if age <= 180 else 0 comments = min(20, int(issue.get("comments") or 0) * 2) @@ -57,11 +134,11 @@ def issue_score(issue, repo_meta, cfg, now): matched = [] for word, pts in cfg["externalizable_keywords"].items(): if word.lower() in text: - ext += pts + ext += int(pts) matched.append(word) - ext = min(28, ext) + ext = min(30, ext) - heat, heat_breakdown = repo_heat(repo_meta, now) + _, heat_breakdown = repo_heat(repo_meta, now) breakdown = { "rejection": rejection, "recency": recency, @@ -69,26 +146,26 @@ def issue_score(issue, repo_meta, cfg, now): "externalizable": ext, **heat_breakdown, } - score = sum(breakdown.values()) return { - "score": score, + "score": sum(breakdown.values()), "breakdown": breakdown, "age_days": age, - "matched": matched[:6], + "matched": matched[:8], } def category(title): t = title.lower() buckets = [ ("browser", ["browser","upload","preview","localhost","tab","form"]), - ("context", ["context","compact","session","memory","history"]), - ("authority", ["permission","approval","sandbox","security","guardrail"]), - ("integration", ["mcp","api","webhook","integration","plugin","extension"]), - ("interface", ["ui","sidebar","dashboard","preset","menu","export"]), - ("workflow", ["task","project","workflow","plan","goal"]), + ("context", ["context","compact","session","memory","history","resume"]), + ("authority", ["permission","approval","sandbox","security","guardrail","oauth","auth"]), + ("integration", ["mcp","api","webhook","integration","plugin","extension","sync"]), + ("interface", ["ui","sidebar","dashboard","preset","menu","export","import"]), + ("workflow", ["task","project","workflow","plan","goal","schedule","cron","hook"]), ] for name, words in buckets: - if any(w in t for w in words): return name + if any(w in t for w in words): + return name return "wrapper" def move_for(cat): @@ -102,62 +179,124 @@ def move_for(cat): "wrapper":"thin wrapper around existing capability", }[cat] -def card(o): - b=o["breakdown"] - why=f"rej {b['rejection']} · rec {b['recency']} · eng {b['engagement']} · ext {b['externalizable']} · heat {b['repo_activity']+b['repo_scale']}" +def scan(): + cfg = json.loads((ROOT/"radar.json").read_text()) + now = dt.datetime.now(dt.timezone.utc) + repos, discovery_errors = discover_repos(cfg, now) + rows = [] + issue_errors = [] + + for repo in repos: + repo_name = repo["full_name"] + try: + issues = api(f"https://api.github.com/repos/{repo_name}/issues?state=closed&sort=updated&direction=desc&per_page=100") + except Exception as e: + issue_errors.append({"repo": repo_name, "stage": "issues", "error": type(e).__name__}) + continue + for issue in issues: + if "pull_request" in issue: + continue + scored = issue_score(issue, repo, cfg, now) + if not scored or scored["score"] < cfg["threshold"]: + continue + cat = category(issue.get("title","")) + rows.append({ + "repo": repo_name, + "repo_source": repo.get("_source", "unknown"), + "repo_source_query": repo.get("_source_query"), + "repo_stars": int(repo.get("stargazers_count") or 0), + "repo_archived": bool(repo.get("archived")), + "number": issue.get("number"), + "title": issue.get("title",""), + "html_url": issue.get("html_url",""), + "closed_at": issue.get("closed_at") or "", + "category": cat, + "move": move_for(cat), + **scored, + }) + + rows.sort(key=lambda x: ( + x["score"], x["closed_at"], x["repo_stars"], x["repo"], x["number"] + ), reverse=True) + + snapshot = { + "generated_at": now.isoformat(), + "rules": { + "threshold": cfg["threshold"], + "limit": cfg["limit"], + "seed_repos": cfg["seed_repos"], + "discovery_queries": cfg["discovery_queries"], + "max_total_repos": cfg["max_total_repos"], + "max_discovered_repos": cfg["max_discovered_repos"], + }, + "universe": [{ + "repo": r["full_name"], + "source": r.get("_source"), + "source_query": r.get("_source_query"), + "stars": int(r.get("stargazers_count") or 0), + "archived": bool(r.get("archived")), + "pushed_at": r.get("pushed_at"), + } for r in repos], + "candidates": rows[:cfg["limit"]], + "errors": discovery_errors + issue_errors, + } + return cfg, snapshot + +def curated_card(o): + return f""" +
{esc(o['repo'])} · #{esc(o['issue'])}{esc(o['heat'])}
+

{esc(o['name'])}

+
{esc(o['title'])}
+

{esc(o['thesis'])}

+
move: {esc(o['move'])}
+
""" + +def radar_card(o): + b = o["breakdown"] + why = f"rej {b['rejection']} · rec {b['recency']} · eng {b['engagement']} · ext {b['externalizable']} · heat {b['repo_activity']+b['repo_scale']+b['issue_surface']}" + src = "seed" if o["repo_source"] == "seed" else "discovered" + arch = " · archived" if o["repo_archived"] else "" return f"""
{esc(o['repo'])} · #{o['number']}{o['score']}

{esc(o['title'])}

-
{esc(o['category'])} · closed {o['age_days']}d ago
+
{esc(o['category'])} · {src}{arch} · closed {o['age_days']}d ago

{esc(o['move'])}

{esc(why)}
""" -def scan(): - cfg=json.loads((ROOT/"radar.json").read_text()) - now=dt.datetime.now(dt.timezone.utc) - rows=[] - for repo_name in cfg["repos"]: - try: - repo=api(f"https://api.github.com/repos/{repo_name}") - issues=api(f"https://api.github.com/repos/{repo_name}/issues?state=closed&sort=updated&direction=desc&per_page=100") - except Exception as e: - print(f"warning {repo_name}: {type(e).__name__}") - continue - for issue in issues: - if "pull_request" in issue: continue - scored=issue_score(issue, repo, cfg, now) - if not scored or scored["score"] < cfg["threshold"]: continue - cat=category(issue.get("title","")) - rows.append({ - "repo":repo_name, - "number":issue.get("number"), - "title":issue.get("title",""), - "html_url":issue.get("html_url",""), - "closed_at":issue.get("closed_at") or "", - "category":cat, - "move":move_for(cat), - **scored, - }) - rows.sort(key=lambda x:(x["score"], x["closed_at"]), reverse=True) - return cfg, rows[:cfg["limit"]] - -def render(out_path): - cfg, rows=scan() - tpl=(ROOT/"template.html").read_text() - now=dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M") - html_out=(tpl - .replace("__COUNT__",str(len(rows))) - .replace("__UPDATED__",now) - .replace("__THRESHOLD__",str(cfg["threshold"])) - .replace("__REPOS__",str(len(cfg["repos"]))) - .replace("__CARDS__","\n".join(card(x) for x in rows) if rows else '
No candidate cleared the deterministic threshold on this pass.
') +def render(out_path, json_out=None): + cfg, snapshot = scan() + curated_data = json.loads((ROOT/"opportunities.json").read_text()) + curated = curated_data.get("opportunities", []) + template = (ROOT/"template.html").read_text() + now = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M") + page = (template + .replace("__COUNT__", str(len(snapshot["candidates"]))) + .replace("__UPDATED__", now) + .replace("__THRESHOLD__", str(cfg["threshold"])) + .replace("__REPOS__", str(len(snapshot["universe"]))) + .replace("__SEEDS__", str(sum(1 for x in snapshot["universe"] if x["source"] == "seed"))) + .replace("__DISCOVERED__", str(sum(1 for x in snapshot["universe"] if x["source"] == "discover"))) + .replace("__CURATED_COUNT__", str(len(curated))) + .replace("__CURATED_CARDS__", "\n".join(curated_card(x) for x in curated)) + .replace("__CARDS__", "\n".join(radar_card(x) for x in snapshot["candidates"]) if snapshot["candidates"] else '
No candidate cleared the deterministic threshold on this pass.
') ) - pathlib.Path(out_path).write_text(html_out) - print(json.dumps({"candidates":len(rows),"threshold":cfg["threshold"],"repos":len(cfg["repos"]),"out":str(out_path)})) + pathlib.Path(out_path).write_text(page) + if json_out: + pathlib.Path(json_out).write_text(json.dumps(snapshot, indent=2) + "\n") + print(json.dumps({ + "candidates": len(snapshot["candidates"]), + "repos": len(snapshot["universe"]), + "seed": sum(1 for x in snapshot["universe"] if x["source"] == "seed"), + "discovered": sum(1 for x in snapshot["universe"] if x["source"] == "discover"), + "curated": len(curated), + "errors": len(snapshot["errors"]), + "out": str(out_path), + })) -if __name__=="__main__": - p=argparse.ArgumentParser() - p.add_argument("--out",default=str(ROOT/"index.html")) - a=p.parse_args() - render(a.out) +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--out", default=str(ROOT/"index.html")) + p.add_argument("--json-out", default="") + a = p.parse_args() + render(a.out, a.json_out or None)