scripts/check-library.py

 1#!/usr/bin/env python3
 2"""Check Library pages: JSON shape, and that every source and image URL resolves.
 3
 4Usage: scripts/check-library.py [--offline]
 5"""
 6import json
 7import pathlib
 8import sys
 9import urllib.request
10from concurrent.futures import ThreadPoolExecutor
11
12PAGES = pathlib.Path(__file__).resolve().parent.parent / "Packages/MoonriseKit/Sources/Library/Resources/pages"
13UA = {"User-Agent": "moonrise-library-check (sh.krz.moonrise)"}
14BLOCKS_SCRIPTS = ("https://www.loc.gov/item/", "https://www.jpl.nasa.gov/news/press_kits/", "https://rocketlabcorp.com/", "https://www.blueorigin.com/", "https://blueorigin.com/")
15# Launch Library 2 allows 15 anonymous requests an hour; its pages are not checked here.
16SKIP = ("https://ll.thespacedevs.com/",)
17
18
19def status(url, image=False):
20    for method in ("HEAD", "GET"):
21        try:
22            req = urllib.request.Request(url, method=method, headers=UA)
23            with urllib.request.urlopen(req, timeout=20) as r:
24                kind = r.headers.get("content-type", "")
25                if image and not kind.startswith("image/"):
26                    return f"{r.status} {kind}"
27                return r.status
28        except Exception as e:
29            err = getattr(e, "code", None) or type(e).__name__
30    return err
31
32
33def main():
34    offline = "--offline" in sys.argv
35    pages, problems = [], []
36    for f in sorted(PAGES.glob("*.json")):
37        for p in json.loads(f.read_text()):
38            pages.append((f.name, p))
39    print(f"{len(pages)} pages in {len(list(PAGES.glob('*.json')))} files")
40    if offline:
41        return 0
42    checks = []
43    for name, p in pages:
44        for s in p.get("sources", []):
45            if not s["url"].startswith(SKIP):
46                checks.append((p["id"], "source", s["url"], False))
47        if p.get("image"):
48            checks.append((p["id"], "image", p["image"]["url"], True))
49            checks.append((p["id"], "image page", p["image"]["page"], False))
50    with ThreadPoolExecutor(8) as pool:
51        results = list(pool.map(lambda c: (c, status(c[2], c[3])), checks))
52    for (pid, kind, url, _), st in results:
53        # Sites that refuse scripts (403/429) but load in a browser.
54        if st in (403, 429) and any(url.startswith(p) for p in BLOCKS_SCRIPTS):
55            continue
56        if st != 200:
57            problems.append(f"{pid}: {kind} {st} {url}")
58    print(f"{len(checks)} URLs checked, {len(problems)} problems")
59    for line in problems:
60        print("  " + line)
61    return 1 if problems else 0
62
63
64if __name__ == "__main__":
65    sys.exit(main())