krz/hutch

 1#!/usr/bin/env python3
 2"""Regenerate the bundled sr.ht man page catalog from man.sr.ht.
 3
 4Fetches the man.sr.ht landing page, extracts the per-service "User Manual"
 5links, and writes them to ``Hutch/Resources/man-pages.json``. The scheduled
 6GitHub workflow that runs this opens a pull request whenever the result differs
 7from the committed copy, so the in-app list stays in sync with upstream without
 8hand edits.
 9
10Run locally with ``python3 scripts/sync_man_pages.py``; exits non-zero (without
11writing) if upstream markup changed enough that too few pages were found, so a
12bad scrape can never wipe the bundled list.
13"""
14import json
15import re
16import sys
17import urllib.request
18from pathlib import Path
19
20INDEX_URL = "https://man.sr.ht/"
21OUTPUT = Path(__file__).resolve().parent.parent / "Hutch" / "Resources" / "man-pages.json"
22# The suite has ~12 service manuals; a scrape returning far fewer means the page
23# structure changed and we should fail loudly rather than commit a gutted list.
24MINIMUM_EXPECTED = 8
25
26
27def fetch(url: str) -> str:
28    request = urllib.request.Request(url, headers={"User-Agent": "hutch-man-page-sync"})
29    with urllib.request.urlopen(request, timeout=30) as response:
30        return response.read().decode("utf-8")
31
32
33def build_catalog(html: str) -> list[dict[str, str]]:
34    """Extract official man-page links, deduplicated and sorted by title."""
35    entries: dict[str, str] = {}
36    for href in re.findall(r'href="([^"]+)"', html):
37        service = re.fullmatch(r"/([a-z0-9][a-z0-9.-]*\.sr\.ht)/?", href)
38        if service:
39            title = service.group(1)
40            entries[title] = f"https://man.sr.ht/{title}/"
41        elif re.fullmatch(r"sr\.ht/?", href):
42            entries["sr.ht"] = "https://man.sr.ht/sr.ht/"
43        elif re.fullmatch(r"https://srht\.site/?", href):
44            entries["srht.site"] = "https://srht.site/"
45    return [{"title": title, "url": entries[title]} for title in sorted(entries)]
46
47
48def main() -> int:
49    catalog = build_catalog(fetch(INDEX_URL))
50    if len(catalog) < MINIMUM_EXPECTED:
51        print(
52            f"Refusing to write catalog with only {len(catalog)} entries; "
53            "man.sr.ht markup may have changed.",
54            file=sys.stderr,
55        )
56        return 1
57    OUTPUT.write_text(json.dumps(catalog, indent=2, ensure_ascii=False) + "\n")
58    print(f"Wrote {len(catalog)} man page(s) to {OUTPUT}")
59    return 0
60
61
62if __name__ == "__main__":
63    sys.exit(main())