notplanned/ventures/up-to-date/monitor.py

"""Bounded public pilot monitor. No credentials, code writes or outbound notifications."""
import argparse
import datetime as dt
import fcntl
import json
import os
from pathlib import Path
import re
import sqlite3
import time
import urllib.parse
import urllib.error
import urllib.request


def atomic(path, value):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_suffix('.tmp')
    tmp.write_text(json.dumps(value, indent=2) + '\n')
    os.replace(tmp, path)


class APIError(Exception):
    def __init__(self, message, retry=3600):
        super().__init__(message)
        self.retry = min(max(int(retry), 900), 86400)


class Client:
    def __init__(self):
        self.calls = 0

    def get(self, url, body=None):
        self.calls += 1
        if self.calls > 24:
            raise APIError('Request budget exhausted')
        req = urllib.request.Request(url, data=json.dumps(body).encode() if body else None,
                                     headers={'User-Agent': 'UpToDate-pilot/3', 'Content-Type': 'application/json'})
        try:
            with urllib.request.urlopen(req, timeout=12) as r:
                raw = r.read(2000001)
                if len(raw) > 2000000:
                    raise APIError('Response exceeds 2 MB bound')
                return json.loads(raw)
        except urllib.error.HTTPError as e:
            retry = e.headers.get('Retry-After', '3600')
            reset = e.headers.get('X-RateLimit-Reset')
            if reset and reset.isdigit():
                retry = max(int(reset) - int(time.time()), 900)
            raise APIError('Upstream HTTP %s; coverage unknown' % e.code,
                           retry if str(retry).isdigit() else 3600) from None
        except (urllib.error.URLError, TimeoutError, ValueError) as e:
            raise APIError('Upstream unavailable; coverage unknown: ' + type(e).__name__) from None


def validate(registry):
    entries = registry['repos']
    if len(entries) > 3:
        raise ValueError('Pilot bounded to three repositories')
    seen = set()
    for e in entries:
        if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', e['repo']) or e['repo'] in seen:
            raise ValueError('Invalid or duplicate public GitHub repository')
        seen.add(e['repo'])
        if e['state'] not in ('requested', 'active', 'paused') or e['notification'] not in ('feed', 'none'):
            raise ValueError('Invalid activation or notification preference')
        if e['cadence_hours'] not in (24, 168):
            raise ValueError('Cadence must be daily or weekly')
        if e['state'] == 'active' and (not e.get('public_consent') or not e.get('activation_evidence')):
            raise ValueError('Activation needs explicit public consent and evidence')
    return entries


def scan(repo, get):
    api = 'https://api.github.com/repos/' + repo
    meta = get(api)
    if meta.get('private') is not False:
        raise APIError('Only verified public repositories are supported')
    commit = get(api + '/commits/' + urllib.parse.quote(meta['default_branch'], safe=''))
    sha = commit['sha']
    releases = get(api + '/releases?per_page=3')
    gaps = ['Root package.json only; nested/other ecosystems unsupported.',
            'Ranges do not identify installed versions; runtime reachability unknown.']
    deps = {}
    try:
        package = get('https://raw.githubusercontent.com/' + repo + '/' + sha + '/package.json')
        deps = {**package.get('dependencies', {}), **package.get('devDependencies', {})}
    except APIError as e:
        raise APIError('Manifest unavailable; previous baseline preserved: ' + str(e), e.retry)
    signals = {}
    for name, version in sorted(deps.items())[:6]:
        pkg = get('https://registry.npmjs.org/' + urllib.parse.quote(name, safe=''))
        latest = pkg.get('dist-tags', {}).get('latest')
        signals['npm:' + name] = {'declared': version, 'latest': latest,
                                 'url': 'https://www.npmjs.com/package/' + urllib.parse.quote(name, safe='')}
        if re.fullmatch(r'\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?', version):
            current = pkg.get('versions', {}).get(version, {})
            signals['npm:' + name]['deprecated'] = current.get('deprecated')
            osv = get('https://api.osv.dev/v1/query', {'package': {'name': name, 'ecosystem': 'npm'}, 'version': version})
            for v in osv.get('vulns', []):
                if not v.get('withdrawn'):
                    signals['osv:' + name + ':' + v['id']] = {'summary': v.get('summary', 'Review advisory'),
                                                             'url': 'https://osv.dev/vulnerability/' + v['id']}
        else:
            gaps.append(name + ': advisory/deprecation coverage unknown (range).')
    if len(deps) > 6:
        gaps.append('First six sorted npm dependencies checked out of ' + str(len(deps)))
    for r in releases:
        signals['release:' + str(r['id'])] = {'tag': r['tag_name'], 'url': r['html_url']}
    return {'commit': sha, 'dependencies': deps, 'signals': signals, 'gaps': gaps}


def diff(old, new):
    changes = []
    if old['commit'] != new['commit']:
        changes.append({'kind': 'source', 'before': old['commit'], 'after': new['commit']})
    for section in ('dependencies', 'signals'):
        for key in sorted(set(old[section]) | set(new[section])):
            before, after = old[section].get(key), new[section].get(key)
            if before != after:
                changes.append({'kind': section, 'component': key, 'before': before, 'after': after})
    return changes


def connect(path):
    db = sqlite3.connect(path)
    db.executescript('''PRAGMA journal_mode=WAL;
    CREATE TABLE IF NOT EXISTS runs(repo TEXT PRIMARY KEY, checked REAL, due REAL, status TEXT, error TEXT);
    CREATE TABLE IF NOT EXISTS snapshots(id INTEGER PRIMARY KEY, repo TEXT, checked REAL, body TEXT, changes TEXT);
    CREATE TABLE IF NOT EXISTS alerts(id INTEGER PRIMARY KEY, repo TEXT, checked REAL, changes TEXT);''')
    return db


def tick(registry, db, public, get=None, now=None):
    entries = validate(registry)
    now = time.time() if now is None else now
    # Registry is authoritative on every invocation; paused/none hides old alerts too.
    output = {'generated_at': dt.datetime.fromtimestamp(now, dt.timezone.utc).isoformat(),
              'channel': 'Pull-only public feed. No email or GitHub messages sent.', 'repos': []}
    for e in entries:
        repo = e['repo']
        run = db.execute('SELECT checked,due,status,error FROM runs WHERE repo=?', (repo,)).fetchone()
        if e['state'] == 'active' and (not run or now >= run[1]):
            try:
                snapshot = scan(repo, get or Client().get)
                previous = db.execute('SELECT body FROM snapshots WHERE repo=? ORDER BY id DESC LIMIT 1', (repo,)).fetchone()
                changes = diff(json.loads(previous[0]), snapshot) if previous else []
                with db:
                    db.execute('INSERT INTO snapshots(repo,checked,body,changes) VALUES(?,?,?,?)',
                               (repo, now, json.dumps(snapshot), json.dumps(changes)))
                    if changes and e['notification'] == 'feed':
                        db.execute('INSERT INTO alerts(repo,checked,changes) VALUES(?,?,?)', (repo, now, json.dumps(changes)))
                    db.execute('INSERT OR REPLACE INTO runs VALUES(?,?,?,?,?)',
                               (repo, now, now + e['cadence_hours'] * 3600, 'complete', None))
                    db.execute('DELETE FROM snapshots WHERE repo=? AND id NOT IN (SELECT id FROM snapshots WHERE repo=? ORDER BY id DESC LIMIT 30)', (repo, repo))
                    db.execute('DELETE FROM alerts WHERE repo=? AND id NOT IN (SELECT id FROM alerts WHERE repo=? ORDER BY id DESC LIMIT 100)', (repo, repo))
            except APIError as ex:
                with db:
                    db.execute('INSERT OR REPLACE INTO runs VALUES(?,?,?,?,?)', (repo, now, now + ex.retry, 'unknown', str(ex)))
            run = db.execute('SELECT checked,due,status,error FROM runs WHERE repo=?', (repo,)).fetchone()
        latest = db.execute('SELECT checked,body,changes FROM snapshots WHERE repo=? ORDER BY id DESC LIMIT 1', (repo,)).fetchone()
        row = {**e, 'last_attempt': run[0] if run else None, 'next_due': run[1] if run and e['state'] == 'active' else None,
               'scan_status': run[2] if run else 'pending', 'error': run[3] if run else None,
               'last_success': latest[0] if latest else None, 'snapshot': json.loads(latest[1]) if latest else None,
               'diff': json.loads(latest[2]) if latest else [], 'alerts': []}
        if e['state'] == 'active' and e['notification'] == 'feed':
            row['alerts'] = [{'checked': r[0], 'changes': json.loads(r[1])} for r in db.execute('SELECT checked,changes FROM alerts WHERE repo=? ORDER BY id DESC LIMIT 10', (repo,))]
        output['repos'].append(row)
    atomic(public, output)
    return output


def main():
    p = argparse.ArgumentParser()
    p.add_argument('--registry', required=True)
    p.add_argument('--state-dir', required=True)
    p.add_argument('--public', required=True)
    a = p.parse_args()
    state = Path(a.state_dir)
    state.mkdir(parents=True, exist_ok=True)
    with (state / 'scan.lock').open('w') as lock:
        try:
            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except BlockingIOError:
            return
        db = connect(state / 'monitor.sqlite3')
        try:
            out = tick(json.loads(Path(a.registry).read_text()), db, a.public)
            print(json.dumps({'generated_at': out['generated_at'], 'repos': [(r['repo'], r['state'], r['scan_status']) for r in out['repos']]}))
        finally:
            db.close()


if __name__ == '__main__':
    main()