teahubxyz/notplanned
1
0 Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Add deterministic repo discovery and dual raw/curated feeds

Browse Source
This commit is contained in:
matcha committed 2026-10-05 20:21:06 +00:00
1 parent 4bae27af37
commit dcdcfc5c45
1 file changed
+221 -82
Download Patch File Download Diff File Expand all files Collapse all files
scout.py
+221 -82
View File
@@ -1,49 +1,126 @@
#!/usr/bin/env python3
import argparse, datetime as dt, html, json, math, pathlib, urllib.request, urllib.error
import argparse, datetime as dt, html, json, math, pathlib, urllib.parse, urllib.request, urllib.error
ROOT = pathlib.Path(__file__).resolve().parent
def api(url):
req = urllib.request.Request(url, headers={
"User-Agent": "notplanned-radar/0.2",
"User-Agent": "notplanned-radar/0.3",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
})
with urllib.request.urlopen(req, timeout=20) as r:
return json.load(r)
def esc(v): return html.escape(str(v), quote=True)
def esc(v):
return html.escape(str(v), quote=True)
def iso_age_days(value, now):
try:
return max(0, (now - dt.datetime.fromisoformat(value.replace("Z","+00:00"))).days)
except Exception:
return 9999
def repo_heat(repo, now):
pushed = repo.get("pushed_at") or repo.get("updated_at")
try:
age = max(0, (now - dt.datetime.fromisoformat(pushed.replace("Z","+00:00"))).days)
except Exception:
age = 999
activity = 15 if age <= 7 else 10 if age <= 30 else 5 if age <= 90 else 0
age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now)
activity = 18 if age <= 3 else 15 if age <= 7 else 11 if age <= 30 else 6 if age <= 90 else 2 if age <= 180 else 0
stars = int(repo.get("stargazers_count") or 0)
star_score = min(15, round(math.log10(max(stars, 1)) * 3.2))
return activity + star_score, {"repo_activity": activity, "repo_scale": star_score}
star_score = min(18, round(math.log10(max(stars, 1)) * 3.5))
issues = int(repo.get("open_issues_count") or 0)
issue_surface = min(8, round(math.log10(max(issues, 1)) * 2.3))
return activity + star_score + issue_surface, {
"repo_activity": activity,
"repo_scale": star_score,
"issue_surface": issue_surface,
}
def repo_discovery_score(repo, now):
heat, _ = repo_heat(repo, now)
archived_bonus = 5 if repo.get("archived") else 0
return heat + archived_bonus
def fetch_repo(full_name):
return api(f"https://api.github.com/repos/{full_name}")
def discover_repos(cfg, now):
universe = {}
errors = []
for full_name in cfg["seed_repos"]:
try:
repo = fetch_repo(full_name)
repo["_source"] = "seed"
repo["_source_query"] = None
universe[repo["full_name"].lower()] = repo
except Exception as e:
errors.append({"repo": full_name, "stage": "seed", "error": type(e).__name__})
discovered = {}
for query in cfg["discovery_queries"]:
url = "https://api.github.com/search/repositories?" + urllib.parse.urlencode({
"q": query,
"sort": "updated",
"order": "desc",
"per_page": 30,
})
try:
result = api(url)
except Exception as e:
errors.append({"query": query, "stage": "discover", "error": type(e).__name__})
continue
allow_archived = "archived:true" in query
for repo in result.get("items", []):
full = repo.get("full_name")
if not full or not repo.get("has_issues", True) or repo.get("fork"):
continue
if int(repo.get("stargazers_count") or 0) < cfg["min_repo_stars"]:
continue
push_age = iso_age_days(repo.get("pushed_at") or repo.get("updated_at") or "", now)
if not allow_archived and push_age > cfg["max_repo_push_age_days"]:
continue
key = full.lower()
if key in universe:
continue
repo["_source"] = "discover"
repo["_source_query"] = query
old = discovered.get(key)
if old is None or repo_discovery_score(repo, now) > repo_discovery_score(old, now):
discovered[key] = repo
ranked = sorted(discovered.values(), key=lambda r: (
repo_discovery_score(r, now),
r.get("pushed_at") or "",
int(r.get("stargazers_count") or 0),
), reverse=True)[:cfg["max_discovered_repos"]]
for repo in ranked:
universe.setdefault(repo["full_name"].lower(), repo)
repos = list(universe.values())
repos.sort(key=lambda r: (
1 if r.get("_source") == "seed" else 0,
repo_discovery_score(r, now),
r.get("full_name", "").lower(),
), reverse=True)
repos = repos[:cfg["max_total_repos"]]
return repos, errors
def issue_score(issue, repo_meta, cfg, now):
labels = {str(x.get("name","")).strip().lower() for x in issue.get("labels", [])}
if labels & {x.lower() for x in cfg["exclude_labels"]}:
return None
rejection = 0
reason = issue.get("state_reason")
if reason == "not_planned":
rejection = max(rejection, 32)
if issue.get("state_reason") == "not_planned":
rejection = 32
for label, pts in cfg["rejection_labels"].items():
if label.lower() in labels:
rejection = max(rejection, pts)
rejection = max(rejection, int(pts))
if rejection == 0:
return None
closed = issue.get("closed_at") or issue.get("updated_at")
try:
age = max(0, (now - dt.datetime.fromisoformat(closed.replace("Z","+00:00"))).days)
except Exception:
age = 999
closed = issue.get("closed_at") or issue.get("updated_at") or ""
age = iso_age_days(closed, now)
recency = 40 if age <= 7 else 30 if age <= 30 else 16 if age <= 90 else 6 if age <= 180 else 0
comments = min(20, int(issue.get("comments") or 0) * 2)
@@ -57,11 +134,11 @@ def issue_score(issue, repo_meta, cfg, now):
matched = []
for word, pts in cfg["externalizable_keywords"].items():
if word.lower() in text:
ext += pts
ext += int(pts)
matched.append(word)
ext = min(28, ext)
ext = min(30, ext)
heat, heat_breakdown = repo_heat(repo_meta, now)
_, heat_breakdown = repo_heat(repo_meta, now)
breakdown = {
"rejection": rejection,
"recency": recency,
@@ -69,26 +146,26 @@ def issue_score(issue, repo_meta, cfg, now):
"externalizable": ext,
**heat_breakdown,
}
score = sum(breakdown.values())
return {
"score": score,
"score": sum(breakdown.values()),
"breakdown": breakdown,
"age_days": age,
"matched": matched[:6],
"matched": matched[:8],
}
def category(title):
t = title.lower()
buckets = [
("browser", ["browser","upload","preview","localhost","tab","form"]),
("context", ["context","compact","session","memory","history"]),
("authority", ["permission","approval","sandbox","security","guardrail"]),
("integration", ["mcp","api","webhook","integration","plugin","extension"]),
("interface", ["ui","sidebar","dashboard","preset","menu","export"]),
("workflow", ["task","project","workflow","plan","goal"]),
("context", ["context","compact","session","memory","history","resume"]),
("authority", ["permission","approval","sandbox","security","guardrail","oauth","auth"]),
("integration", ["mcp","api","webhook","integration","plugin","extension","sync"]),
("interface", ["ui","sidebar","dashboard","preset","menu","export","import"]),
("workflow", ["task","project","workflow","plan","goal","schedule","cron","hook"]),
]
for name, words in buckets:
if any(w in t for w in words): return name
if any(w in t for w in words):
return name
return "wrapper"
def move_for(cat):
@@ -102,62 +179,124 @@ def move_for(cat):
"wrapper":"thin wrapper around existing capability",
}[cat]
def card(o):
b=o["breakdown"]
why=f"rej {b['rejection']} · rec {b['recency']} · eng {b['engagement']} · ext {b['externalizable']} · heat {b['repo_activity']+b['repo_scale']}"
def scan():
cfg = json.loads((ROOT/"radar.json").read_text())
now = dt.datetime.now(dt.timezone.utc)
repos, discovery_errors = discover_repos(cfg, now)
rows = []
issue_errors = []
for repo in repos:
repo_name = repo["full_name"]
try:
issues = api(f"https://api.github.com/repos/{repo_name}/issues?state=closed&sort=updated&direction=desc&per_page=100")
except Exception as e:
issue_errors.append({"repo": repo_name, "stage": "issues", "error": type(e).__name__})
continue
for issue in issues:
if "pull_request" in issue:
continue
scored = issue_score(issue, repo, cfg, now)
if not scored or scored["score"] < cfg["threshold"]:
continue
cat = category(issue.get("title",""))
rows.append({
"repo": repo_name,
"repo_source": repo.get("_source", "unknown"),
"repo_source_query": repo.get("_source_query"),
"repo_stars": int(repo.get("stargazers_count") or 0),
"repo_archived": bool(repo.get("archived")),
"number": issue.get("number"),
"title": issue.get("title",""),
"html_url": issue.get("html_url",""),
"closed_at": issue.get("closed_at") or "",
"category": cat,
"move": move_for(cat),
**scored,
})
rows.sort(key=lambda x: (
x["score"], x["closed_at"], x["repo_stars"], x["repo"], x["number"]
), reverse=True)
snapshot = {
"generated_at": now.isoformat(),
"rules": {
"threshold": cfg["threshold"],
"limit": cfg["limit"],
"seed_repos": cfg["seed_repos"],
"discovery_queries": cfg["discovery_queries"],
"max_total_repos": cfg["max_total_repos"],
"max_discovered_repos": cfg["max_discovered_repos"],
},
"universe": [{
"repo": r["full_name"],
"source": r.get("_source"),
"source_query": r.get("_source_query"),
"stars": int(r.get("stargazers_count") or 0),
"archived": bool(r.get("archived")),
"pushed_at": r.get("pushed_at"),
} for r in repos],
"candidates": rows[:cfg["limit"]],
"errors": discovery_errors + issue_errors,
}
return cfg, snapshot
def curated_card(o):
return f"""<a class="curated-card" href="{esc(o['source'])}" target="_blank" rel="noreferrer">
<div class="topline"><span class="repo">{esc(o['repo'])} · #{esc(o['issue'])}</span><span class="heat {esc(o['heat'])}">{esc(o['heat'])}</span></div>
<h3>{esc(o['name'])}</h3>
<div class="meta">{esc(o['title'])}</div>
<p class="thesis">{esc(o['thesis'])}</p>
<div class="why"><strong>move:</strong> {esc(o['move'])}</div>
</a>"""
def radar_card(o):
b = o["breakdown"]
why = f"rej {b['rejection']} · rec {b['recency']} · eng {b['engagement']} · ext {b['externalizable']} · heat {b['repo_activity']+b['repo_scale']+b['issue_surface']}"
src = "seed" if o["repo_source"] == "seed" else "discovered"
arch = " · archived" if o["repo_archived"] else ""
return f"""<a class="card" href="{esc(o['html_url'])}" target="_blank" rel="noreferrer">
<div class="topline"><span class="repo">{esc(o['repo'])} · #{o['number']}</span><span class="score">{o['score']}</span></div>
<h3>{esc(o['title'])}</h3>
<div class="meta">{esc(o['category'])} · closed {o['age_days']}d ago</div>
<div class="meta">{esc(o['category'])} · {src}{arch} · closed {o['age_days']}d ago</div>
<p class="move">{esc(o['move'])}</p>
<div class="why">{esc(why)}</div>
</a>"""
def scan():
cfg=json.loads((ROOT/"radar.json").read_text())
now=dt.datetime.now(dt.timezone.utc)
rows=[]
for repo_name in cfg["repos"]:
try:
repo=api(f"https://api.github.com/repos/{repo_name}")
issues=api(f"https://api.github.com/repos/{repo_name}/issues?state=closed&sort=updated&direction=desc&per_page=100")
except Exception as e:
print(f"warning {repo_name}: {type(e).__name__}")
continue
for issue in issues:
if "pull_request" in issue: continue
scored=issue_score(issue, repo, cfg, now)
if not scored or scored["score"] < cfg["threshold"]: continue
cat=category(issue.get("title",""))
rows.append({
"repo":repo_name,
"number":issue.get("number"),
"title":issue.get("title",""),
"html_url":issue.get("html_url",""),
"closed_at":issue.get("closed_at") or "",
"category":cat,
"move":move_for(cat),
**scored,
})
rows.sort(key=lambda x:(x["score"], x["closed_at"]), reverse=True)
return cfg, rows[:cfg["limit"]]
def render(out_path):
cfg, rows=scan()
tpl=(ROOT/"template.html").read_text()
now=dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M")
html_out=(tpl
.replace("__COUNT__",str(len(rows)))
.replace("__UPDATED__",now)
.replace("__THRESHOLD__",str(cfg["threshold"]))
.replace("__REPOS__",str(len(cfg["repos"])))
.replace("__CARDS__","\n".join(card(x) for x in rows) if rows else '<div class="empty">No candidate cleared the deterministic threshold on this pass.</div>')
def render(out_path, json_out=None):
cfg, snapshot = scan()
curated_data = json.loads((ROOT/"opportunities.json").read_text())
curated = curated_data.get("opportunities", [])
template = (ROOT/"template.html").read_text()
now = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M")
page = (template
.replace("__COUNT__", str(len(snapshot["candidates"])))
.replace("__UPDATED__", now)
.replace("__THRESHOLD__", str(cfg["threshold"]))
.replace("__REPOS__", str(len(snapshot["universe"])))
.replace("__SEEDS__", str(sum(1 for x in snapshot["universe"] if x["source"] == "seed")))
.replace("__DISCOVERED__", str(sum(1 for x in snapshot["universe"] if x["source"] == "discover")))
.replace("__CURATED_COUNT__", str(len(curated)))
.replace("__CURATED_CARDS__", "\n".join(curated_card(x) for x in curated))
.replace("__CARDS__", "\n".join(radar_card(x) for x in snapshot["candidates"]) if snapshot["candidates"] else '<div class="empty">No candidate cleared the deterministic threshold on this pass.</div>')
)
pathlib.Path(out_path).write_text(html_out)
print(json.dumps({"candidates":len(rows),"threshold":cfg["threshold"],"repos":len(cfg["repos"]),"out":str(out_path)}))
pathlib.Path(out_path).write_text(page)
if json_out:
pathlib.Path(json_out).write_text(json.dumps(snapshot, indent=2) + "\n")
print(json.dumps({
"candidates": len(snapshot["candidates"]),
"repos": len(snapshot["universe"]),
"seed": sum(1 for x in snapshot["universe"] if x["source"] == "seed"),
"discovered": sum(1 for x in snapshot["universe"] if x["source"] == "discover"),
"curated": len(curated),
"errors": len(snapshot["errors"]),
"out": str(out_path),
}))
if __name__=="__main__":
p=argparse.ArgumentParser()
p.add_argument("--out",default=str(ROOT/"index.html"))
a=p.parse_args()
render(a.out)
if __name__ == "__main__":
p = argparse.ArgumentParser()
p.add_argument("--out", default=str(ROOT/"index.html"))
p.add_argument("--json-out", default="")
a = p.parse_args()
render(a.out, a.json_out or None)