notplanned/trending.py

#!/usr/bin/env python3
import argparse, datetime as dt, html, json, pathlib, re, urllib.parse, urllib.request
from html import unescape

ROOT = pathlib.Path(__file__).resolve().parent
PERIODS = [("daily","today"),("weekly","this week"),("monthly","this month")]

def fetch(period):
    url = "https://github.com/trending?" + urllib.parse.urlencode({"since": period})
    req = urllib.request.Request(url, headers={
        "User-Agent": "notplanned-trending/0.1",
        "Accept": "text/html,application/xhtml+xml",
    })
    with urllib.request.urlopen(req, timeout=20) as r:
        return r.read().decode("utf-8", "replace")

def clean(fragment):
    fragment = re.sub(r"<[^>]+>", " ", fragment)
    return " ".join(unescape(fragment).split())

def parse(period, body):
    articles = re.findall(r'<article[^>]+class="[^"]*Box-row[^"]*"[^>]*>(.*?)</article>', body, re.S|re.I)
    rows = []
    period_phrase = dict(PERIODS)[period]
    for rank, article in enumerate(articles, 1):
        m = re.search(r'<h2[^>]*>.*?<a[^>]+href="/([^"#?]+/[^"#?]+)"', article, re.S|re.I)
        if not m:
            continue
        repo = unescape(m.group(1)).strip()
        if repo.count("/") != 1:
            continue
        desc = ""
        pm = re.search(r'<p[^>]+class="[^"]*color-fg-muted[^"]*"[^>]*>(.*?)</p>', article, re.S|re.I)
        if pm:
            desc = clean(pm.group(1))
        lang = ""
        lm = re.search(r'itemprop="programmingLanguage"[^>]*>(.*?)</span>', article, re.S|re.I)
        if lm:
            lang = clean(lm.group(1))
        stars = None
        sm = re.search(r'([0-9][0-9,]*)\s+stars\s+' + re.escape(period_phrase), clean(article), re.I)
        if sm:
            stars = int(sm.group(1).replace(",",""))
        rows.append({
            "repo": repo,
            "url": f"https://github.com/{repo}",
            "rank": rank,
            "description": desc,
            "language": lang,
            "stars_gained": stars,
        })
    return rows

def build():
    now = dt.datetime.now(dt.timezone.utc).isoformat()
    periods = {}
    for period, _ in PERIODS:
        periods[period] = parse(period, fetch(period))

    merged = {}
    for period, rows in periods.items():
        for row in rows:
            x = merged.setdefault(row["repo"], {
                "repo": row["repo"],
                "url": row["url"],
                "description": row["description"],
                "language": row["language"],
                "daily_rank": None, "weekly_rank": None, "monthly_rank": None,
                "daily_stars": None, "weekly_stars": None, "monthly_stars": None,
            })
            x[f"{period}_rank"] = row["rank"]
            x[f"{period}_stars"] = row["stars_gained"]
            if not x["description"] and row["description"]:
                x["description"] = row["description"]
            if not x["language"] and row["language"]:
                x["language"] = row["language"]

    for x in merged.values():
        x["periods"] = sum(x[f"{p}_rank"] is not None for p, _ in PERIODS)
        # Mechanical prominence only: preserve GitHub rank, favor appearing in more windows.
        x["prominence"] = sum(
            (26 - min(x[f"{p}_rank"], 25)) * w
            for p, w in [("daily",3),("weekly",2),("monthly",1)]
            if x[f"{p}_rank"] is not None
        )

    items = sorted(merged.values(), key=lambda x: (
        -x["periods"],
        x["daily_rank"] if x["daily_rank"] is not None else 999,
        x["weekly_rank"] if x["weekly_rank"] is not None else 999,
        x["monthly_rank"] if x["monthly_rank"] is not None else 999,
        x["repo"].lower(),
    ))
    return {"generated_at": now, "source":"https://github.com/trending", "periods": periods, "merged": items}

def card(x):
    def badge(label, rank, stars):
        if rank is None: return ""
        gain = f" · +{stars:,}" if stars is not None else ""
        return f'<span class="badge">{label} #{rank}{gain}</span>'
    badges = "".join([
        badge("day", x["daily_rank"], x["daily_stars"]),
        badge("week", x["weekly_rank"], x["weekly_stars"]),
        badge("month", x["monthly_rank"], x["monthly_stars"]),
    ])
    desc = html.escape(x["description"])
    lang = html.escape(x["language"] or "—")
    repo = html.escape(x["repo"])
    url = html.escape(x["url"], quote=True)
    return f"""<a class="repo-card" href="{url}" target="_blank" rel="noreferrer">
      <div class="repo-top"><h3>{repo}</h3><span class="period-count">{x['periods']}/3</span></div>
      <p>{desc}</p>
      <div class="badges">{badges}</div>
      <div class="lang">{lang}</div>
    </a>"""

def render(out_html, out_json):
    data = build()
    tpl = (ROOT/"trending-template.html").read_text()
    counts = {p: len(data["periods"][p]) for p, _ in PERIODS}
    overlap = sum(1 for x in data["merged"] if x["periods"] >= 2)
    page = (tpl
      .replace("__UPDATED__", data["generated_at"][:16].replace("T"," "))
      .replace("__DAILY__", str(counts["daily"]))
      .replace("__WEEKLY__", str(counts["weekly"]))
      .replace("__MONTHLY__", str(counts["monthly"]))
      .replace("__OVERLAP__", str(overlap))
      .replace("__CARDS__", "\n".join(card(x) for x in data["merged"]))
    )
    pathlib.Path(out_html).write_text(page)
    pathlib.Path(out_json).write_text(json.dumps(data, indent=2)+"\n")
    print(json.dumps({"daily":counts["daily"],"weekly":counts["weekly"],"monthly":counts["monthly"],"merged":len(data["merged"]),"overlap":overlap}))

if __name__ == "__main__":
    p=argparse.ArgumentParser()
    p.add_argument("--html-out", required=True)
    p.add_argument("--json-out", required=True)
    a=p.parse_args()
    render(a.html_out,a.json_out)