Add deterministic GitHub Trending feed
This commit is contained in:
matcha committed
1 parent
5b84b0d427
commit
5a96c258e1
1 file changed
+139
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse, datetime as dt, html, json, pathlib, re, urllib.parse, urllib.request
|
||||
from html import unescape
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent
|
||||
PERIODS = [("daily","today"),("weekly","this week"),("monthly","this month")]
|
||||
|
||||
def fetch(period):
|
||||
url = "https://github.com/trending?" + urllib.parse.urlencode({"since": period})
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "notplanned-trending/0.1",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
return r.read().decode("utf-8", "replace")
|
||||
|
||||
def clean(fragment):
|
||||
fragment = re.sub(r"<[^>]+>", " ", fragment)
|
||||
return " ".join(unescape(fragment).split())
|
||||
|
||||
def parse(period, body):
|
||||
articles = re.findall(r'<article[^>]+class="[^"]*Box-row[^"]*"[^>]*>(.*?)</article>', body, re.S|re.I)
|
||||
rows = []
|
||||
period_phrase = dict(PERIODS)[period]
|
||||
for rank, article in enumerate(articles, 1):
|
||||
m = re.search(r'<h2[^>]*>.*?<a[^>]+href="/([^"#?]+/[^"#?]+)"', article, re.S|re.I)
|
||||
if not m:
|
||||
continue
|
||||
repo = unescape(m.group(1)).strip()
|
||||
if repo.count("/") != 1:
|
||||
continue
|
||||
desc = ""
|
||||
pm = re.search(r'<p[^>]+class="[^"]*color-fg-muted[^"]*"[^>]*>(.*?)</p>', article, re.S|re.I)
|
||||
if pm:
|
||||
desc = clean(pm.group(1))
|
||||
lang = ""
|
||||
lm = re.search(r'itemprop="programmingLanguage"[^>]*>(.*?)</span>', article, re.S|re.I)
|
||||
if lm:
|
||||
lang = clean(lm.group(1))
|
||||
stars = None
|
||||
sm = re.search(r'([0-9][0-9,]*)\s+stars\s+' + re.escape(period_phrase), clean(article), re.I)
|
||||
if sm:
|
||||
stars = int(sm.group(1).replace(",",""))
|
||||
rows.append({
|
||||
"repo": repo,
|
||||
"url": f"https://github.com/{repo}",
|
||||
"rank": rank,
|
||||
"description": desc,
|
||||
"language": lang,
|
||||
"stars_gained": stars,
|
||||
})
|
||||
return rows
|
||||
|
||||
def build():
|
||||
now = dt.datetime.now(dt.timezone.utc).isoformat()
|
||||
periods = {}
|
||||
for period, _ in PERIODS:
|
||||
periods[period] = parse(period, fetch(period))
|
||||
|
||||
merged = {}
|
||||
for period, rows in periods.items():
|
||||
for row in rows:
|
||||
x = merged.setdefault(row["repo"], {
|
||||
"repo": row["repo"],
|
||||
"url": row["url"],
|
||||
"description": row["description"],
|
||||
"language": row["language"],
|
||||
"daily_rank": None, "weekly_rank": None, "monthly_rank": None,
|
||||
"daily_stars": None, "weekly_stars": None, "monthly_stars": None,
|
||||
})
|
||||
x[f"{period}_rank"] = row["rank"]
|
||||
x[f"{period}_stars"] = row["stars_gained"]
|
||||
if not x["description"] and row["description"]:
|
||||
x["description"] = row["description"]
|
||||
if not x["language"] and row["language"]:
|
||||
x["language"] = row["language"]
|
||||
|
||||
for x in merged.values():
|
||||
x["periods"] = sum(x[f"{p}_rank"] is not None for p, _ in PERIODS)
|
||||
# Mechanical prominence only: preserve GitHub rank, favor appearing in more windows.
|
||||
x["prominence"] = sum(
|
||||
(26 - min(x[f"{p}_rank"], 25)) * w
|
||||
for p, w in [("daily",3),("weekly",2),("monthly",1)]
|
||||
if x[f"{p}_rank"] is not None
|
||||
)
|
||||
|
||||
items = sorted(merged.values(), key=lambda x: (
|
||||
-x["periods"],
|
||||
x["daily_rank"] if x["daily_rank"] is not None else 999,
|
||||
x["weekly_rank"] if x["weekly_rank"] is not None else 999,
|
||||
x["monthly_rank"] if x["monthly_rank"] is not None else 999,
|
||||
x["repo"].lower(),
|
||||
))
|
||||
return {"generated_at": now, "source":"https://github.com/trending", "periods": periods, "merged": items}
|
||||
|
||||
def card(x):
|
||||
def badge(label, rank, stars):
|
||||
if rank is None: return ""
|
||||
gain = f" · +{stars:,}" if stars is not None else ""
|
||||
return f'<span class="badge">{label} #{rank}{gain}</span>'
|
||||
badges = "".join([
|
||||
badge("day", x["daily_rank"], x["daily_stars"]),
|
||||
badge("week", x["weekly_rank"], x["weekly_stars"]),
|
||||
badge("month", x["monthly_rank"], x["monthly_stars"]),
|
||||
])
|
||||
desc = html.escape(x["description"])
|
||||
lang = html.escape(x["language"] or "—")
|
||||
repo = html.escape(x["repo"])
|
||||
url = html.escape(x["url"], quote=True)
|
||||
return f"""<a class="repo-card" href="{url}" target="_blank" rel="noreferrer">
|
||||
<div class="repo-top"><h3>{repo}</h3><span class="period-count">{x['periods']}/3</span></div>
|
||||
<p>{desc}</p>
|
||||
<div class="badges">{badges}</div>
|
||||
<div class="lang">{lang}</div>
|
||||
</a>"""
|
||||
|
||||
def render(out_html, out_json):
|
||||
data = build()
|
||||
tpl = (ROOT/"trending-template.html").read_text()
|
||||
counts = {p: len(data["periods"][p]) for p, _ in PERIODS}
|
||||
overlap = sum(1 for x in data["merged"] if x["periods"] >= 2)
|
||||
page = (tpl
|
||||
.replace("__UPDATED__", data["generated_at"][:16].replace("T"," "))
|
||||
.replace("__DAILY__", str(counts["daily"]))
|
||||
.replace("__WEEKLY__", str(counts["weekly"]))
|
||||
.replace("__MONTHLY__", str(counts["monthly"]))
|
||||
.replace("__OVERLAP__", str(overlap))
|
||||
.replace("__CARDS__", "\n".join(card(x) for x in data["merged"]))
|
||||
)
|
||||
pathlib.Path(out_html).write_text(page)
|
||||
pathlib.Path(out_json).write_text(json.dumps(data, indent=2)+"\n")
|
||||
print(json.dumps({"daily":counts["daily"],"weekly":counts["weekly"],"monthly":counts["monthly"],"merged":len(data["merged"]),"overlap":overlap}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
p=argparse.ArgumentParser()
|
||||
p.add_argument("--html-out", required=True)
|
||||
p.add_argument("--json-out", required=True)
|
||||
a=p.parse_args()
|
||||
render(a.html_out,a.json_out)
|
||||
Reference in new issue
Block a user