diff --git a/scout.py b/scout.py
index 9b2ce66..679fb55 100644
--- a/scout.py
+++ b/scout.py
@@ -1,49 +1,126 @@
#!/usr/bin/env python3
-import argparse, datetime as dt, html, json, math, pathlib, urllib.request, urllib.error
+import argparse, datetime as dt, html, json, math, pathlib, urllib.parse, urllib.request, urllib.error
ROOT = pathlib.Path(__file__).resolve().parent
def api(url):
req = urllib.request.Request(url, headers={
- "User-Agent": "notplanned-radar/0.2",
+ "User-Agent": "notplanned-radar/0.3",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
})
with urllib.request.urlopen(req, timeout=20) as r:
return json.load(r)
-def esc(v): return html.escape(str(v), quote=True)
+def esc(v):
+ return html.escape(str(v), quote=True)
+
+def iso_age_days(value, now):
+ try:
+ return max(0, (now - dt.datetime.fromisoformat(value.replace("Z","+00:00"))).days)
+ except Exception:
+ return 9999
def repo_heat(repo, now):
- pushed = repo.get("pushed_at") or repo.get("updated_at")
- try:
- age = max(0, (now - dt.datetime.fromisoformat(pushed.replace("Z","+00:00"))).days)
- except Exception:
- age = 999
- activity = 15 if age <= 7 else 10 if age <= 30 else 5 if age <= 90 else 0
+ age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now)
+ activity = 18 if age <= 3 else 15 if age <= 7 else 11 if age <= 30 else 6 if age <= 90 else 2 if age <= 180 else 0
stars = int(repo.get("stargazers_count") or 0)
- star_score = min(15, round(math.log10(max(stars, 1)) * 3.2))
- return activity + star_score, {"repo_activity": activity, "repo_scale": star_score}
+ star_score = min(18, round(math.log10(max(stars, 1)) * 3.5))
+ issues = int(repo.get("open_issues_count") or 0)
+ issue_surface = min(8, round(math.log10(max(issues, 1)) * 2.3))
+ return activity + star_score + issue_surface, {
+ "repo_activity": activity,
+ "repo_scale": star_score,
+ "issue_surface": issue_surface,
+ }
+
+def repo_discovery_score(repo, now):
+ heat, _ = repo_heat(repo, now)
+ archived_bonus = 5 if repo.get("archived") else 0
+ return heat + archived_bonus
+
+def fetch_repo(full_name):
+ return api(f"https://api.github.com/repos/{full_name}")
+
+def discover_repos(cfg, now):
+ universe = {}
+ errors = []
+
+ for full_name in cfg["seed_repos"]:
+ try:
+ repo = fetch_repo(full_name)
+ repo["_source"] = "seed"
+ repo["_source_query"] = None
+ universe[repo["full_name"].lower()] = repo
+ except Exception as e:
+ errors.append({"repo": full_name, "stage": "seed", "error": type(e).__name__})
+
+ discovered = {}
+ for query in cfg["discovery_queries"]:
+ url = "https://api.github.com/search/repositories?" + urllib.parse.urlencode({
+ "q": query,
+ "sort": "updated",
+ "order": "desc",
+ "per_page": 30,
+ })
+ try:
+ result = api(url)
+ except Exception as e:
+ errors.append({"query": query, "stage": "discover", "error": type(e).__name__})
+ continue
+ allow_archived = "archived:true" in query
+ for repo in result.get("items", []):
+ full = repo.get("full_name")
+ if not full or not repo.get("has_issues", True) or repo.get("fork"):
+ continue
+ if int(repo.get("stargazers_count") or 0) < cfg["min_repo_stars"]:
+ continue
+ push_age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now)
+ if not allow_archived and push_age > cfg["max_repo_push_age_days"]:
+ continue
+ key = full.lower()
+ if key in universe:
+ continue
+ repo["_source"] = "discover"
+ repo["_source_query"] = query
+ old = discovered.get(key)
+ if old is None or repo_discovery_score(repo, now) > repo_discovery_score(old, now):
+ discovered[key] = repo
+
+ ranked = sorted(discovered.values(), key=lambda r: (
+ repo_discovery_score(r, now),
+ r.get("pushed_at") or "",
+ int(r.get("stargazers_count") or 0),
+ ), reverse=True)[:cfg["max_discovered_repos"]]
+
+ for repo in ranked:
+ universe.setdefault(repo["full_name"].lower(), repo)
+
+ repos = list(universe.values())
+ repos.sort(key=lambda r: (
+ 1 if r.get("_source") == "seed" else 0,
+ repo_discovery_score(r, now),
+ r.get("full_name", "").lower(),
+ ), reverse=True)
+ repos = repos[:cfg["max_total_repos"]]
+ return repos, errors
def issue_score(issue, repo_meta, cfg, now):
labels = {str(x.get("name","")).strip().lower() for x in issue.get("labels", [])}
if labels & {x.lower() for x in cfg["exclude_labels"]}:
return None
+
rejection = 0
- reason = issue.get("state_reason")
- if reason == "not_planned":
- rejection = max(rejection, 32)
+ if issue.get("state_reason") == "not_planned":
+ rejection = 32
for label, pts in cfg["rejection_labels"].items():
if label.lower() in labels:
- rejection = max(rejection, pts)
+ rejection = max(rejection, int(pts))
if rejection == 0:
return None
- closed = issue.get("closed_at") or issue.get("updated_at")
- try:
- age = max(0, (now - dt.datetime.fromisoformat(closed.replace("Z","+00:00"))).days)
- except Exception:
- age = 999
+ closed = issue.get("closed_at") or issue.get("updated_at") or ""
+ age = iso_age_days(closed, now)
recency = 40 if age <= 7 else 30 if age <= 30 else 16 if age <= 90 else 6 if age <= 180 else 0
comments = min(20, int(issue.get("comments") or 0) * 2)
@@ -57,11 +134,11 @@ def issue_score(issue, repo_meta, cfg, now):
matched = []
for word, pts in cfg["externalizable_keywords"].items():
if word.lower() in text:
- ext += pts
+ ext += int(pts)
matched.append(word)
- ext = min(28, ext)
+ ext = min(30, ext)
- heat, heat_breakdown = repo_heat(repo_meta, now)
+ _, heat_breakdown = repo_heat(repo_meta, now)
breakdown = {
"rejection": rejection,
"recency": recency,
@@ -69,26 +146,26 @@ def issue_score(issue, repo_meta, cfg, now):
"externalizable": ext,
**heat_breakdown,
}
- score = sum(breakdown.values())
return {
- "score": score,
+ "score": sum(breakdown.values()),
"breakdown": breakdown,
"age_days": age,
- "matched": matched[:6],
+ "matched": matched[:8],
}
def category(title):
t = title.lower()
buckets = [
("browser", ["browser","upload","preview","localhost","tab","form"]),
- ("context", ["context","compact","session","memory","history"]),
- ("authority", ["permission","approval","sandbox","security","guardrail"]),
- ("integration", ["mcp","api","webhook","integration","plugin","extension"]),
- ("interface", ["ui","sidebar","dashboard","preset","menu","export"]),
- ("workflow", ["task","project","workflow","plan","goal"]),
+ ("context", ["context","compact","session","memory","history","resume"]),
+ ("authority", ["permission","approval","sandbox","security","guardrail","oauth","auth"]),
+ ("integration", ["mcp","api","webhook","integration","plugin","extension","sync"]),
+ ("interface", ["ui","sidebar","dashboard","preset","menu","export","import"]),
+ ("workflow", ["task","project","workflow","plan","goal","schedule","cron","hook"]),
]
for name, words in buckets:
- if any(w in t for w in words): return name
+ if any(w in t for w in words):
+ return name
return "wrapper"
def move_for(cat):
@@ -102,62 +179,124 @@ def move_for(cat):
"wrapper":"thin wrapper around existing capability",
}[cat]
-def card(o):
- b=o["breakdown"]
- why=f"rej {b['rejection']} · rec {b['recency']} · eng {b['engagement']} · ext {b['externalizable']} · heat {b['repo_activity']+b['repo_scale']}"
+def scan():
+ cfg = json.loads((ROOT/"radar.json").read_text())
+ now = dt.datetime.now(dt.timezone.utc)
+ repos, discovery_errors = discover_repos(cfg, now)
+ rows = []
+ issue_errors = []
+
+ for repo in repos:
+ repo_name = repo["full_name"]
+ try:
+ issues = api(f"https://api.github.com/repos/{repo_name}/issues?state=closed&sort=updated&direction=desc&per_page=100")
+ except Exception as e:
+ issue_errors.append({"repo": repo_name, "stage": "issues", "error": type(e).__name__})
+ continue
+ for issue in issues:
+ if "pull_request" in issue:
+ continue
+ scored = issue_score(issue, repo, cfg, now)
+ if not scored or scored["score"] < cfg["threshold"]:
+ continue
+ cat = category(issue.get("title",""))
+ rows.append({
+ "repo": repo_name,
+ "repo_source": repo.get("_source", "unknown"),
+ "repo_source_query": repo.get("_source_query"),
+ "repo_stars": int(repo.get("stargazers_count") or 0),
+ "repo_archived": bool(repo.get("archived")),
+ "number": issue.get("number"),
+ "title": issue.get("title",""),
+ "html_url": issue.get("html_url",""),
+ "closed_at": issue.get("closed_at") or "",
+ "category": cat,
+ "move": move_for(cat),
+ **scored,
+ })
+
+ rows.sort(key=lambda x: (
+ x["score"], x["closed_at"], x["repo_stars"], x["repo"], x["number"]
+ ), reverse=True)
+
+ snapshot = {
+ "generated_at": now.isoformat(),
+ "rules": {
+ "threshold": cfg["threshold"],
+ "limit": cfg["limit"],
+ "seed_repos": cfg["seed_repos"],
+ "discovery_queries": cfg["discovery_queries"],
+ "max_total_repos": cfg["max_total_repos"],
+ "max_discovered_repos": cfg["max_discovered_repos"],
+ },
+ "universe": [{
+ "repo": r["full_name"],
+ "source": r.get("_source"),
+ "source_query": r.get("_source_query"),
+ "stars": int(r.get("stargazers_count") or 0),
+ "archived": bool(r.get("archived")),
+ "pushed_at": r.get("pushed_at"),
+ } for r in repos],
+ "candidates": rows[:cfg["limit"]],
+ "errors": discovery_errors + issue_errors,
+ }
+ return cfg, snapshot
+
+def curated_card(o):
+ return f"""
+ {esc(o['thesis'])} {esc(o['move'])}{esc(o['name'])}
+
+ {esc(o['title'])}
-
+