build.py

  1#!/usr/bin/env python3
  2"""
  3This script automates the process of building and deploying the website.
  4It handles tasks such as:
  5
  6- Running orgo to generate site content.
  7- Rewriting the footer's orgo version to match the orgo that built the site.
  8- Rewriting image URLs for the onion service (production only).
  9- Optionally deploying the built site to a remote server.
 10- Starting a local development server for previewing changes.
 11
 12Usage:
 13    Set BUILD=true to build, DEPLOY=true to deploy or serve.
 14    Set ENV=prod for production builds; anything else builds for development.
 15
 16    Builds are incremental. Production writes .build/ and development .build-dev/;
 17    delete either one for a clean build.
 18
 19Dependencies:
 20    - Python 3
 21    - orgo, the static site generator (reads content/orgo.toml)
 22    - rsync for deployment
 23
 24Author:
 25    Christian Cleberg <hello@cleberg.net>
 26"""
 27
 28import hashlib
 29import os
 30import re
 31import subprocess
 32import sys
 33from pathlib import Path
 34
 35
 36def run(cmd, error, echo=False):
 37    """Run cmd quietly, exiting with its stderr if it fails.
 38
 39    echo prints stdout on success. Only the dry run needs it: its output is the
 40    whole reason to run it, and the default of swallowing stdout would hide it.
 41    """
 42    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
 43    if result.returncode != 0:
 44        print(error, file=sys.stderr)
 45        print(result.stderr, file=sys.stderr)
 46        sys.exit(1)
 47    if echo:
 48        print(result.stdout, end="")
 49
 50
 51def run_ruff():
 52    print("Running ruff...")
 53    for cmd in [["ruff", "check", "--fix"], ["ruff", "format"]]:
 54        run(cmd, f"ruff error ({' '.join(cmd)}):")
 55
 56
 57def rewrite_img_urls(build_dir=".build"):
 58    """
 59    Rewrite absolute img.cleberg.net URLs to root-relative /img/ paths so the
 60    onion serves images from its own origin instead of fetching them off-onion.
 61    Production only: dev builds keep the absolute URLs so local previews still
 62    load images from the live image host.
 63
 64    Requires the server to serve /var/www/img/ at /img/ on the cleberg.net vhost
 65    (e.g. `ln -s /var/www/img /var/www/cleberg.net/img`).
 66
 67    The pattern tolerates a stray number of slashes after the scheme
 68    (https:/img, https:///img, ...) so a typo in the org source can't silently
 69    slip through un-rewritten and ship a broken cross-origin URL.
 70    """
 71    pattern = re.compile(r"https:/+img\.cleberg\.net/")
 72    count = 0
 73    for html in Path(build_dir).rglob("*.html"):
 74        text = html.read_text(encoding="utf-8")
 75        new_text, n = pattern.subn("/img/", text)
 76        if n:
 77            count += n
 78            html.write_text(new_text, encoding="utf-8")
 79    print(f"Rewrote {count} img.cleberg.net references to /img/")
 80
 81
 82# The footer's "built with orgo <version>" note. Matching the whole phrase rewrites a
 83# page carrying an old version as readily as one still carrying the template's
 84# placeholder, and leaves prose that ends a paragraph with a link to orgo alone.
 85ORGO_NOTE = re.compile(
 86    r'(built with <a href="https://gitbay\.org/krz/orgo">orgo</a> )[^<]*(</p>)'
 87)
 88
 89
 90def orgo_version():
 91    """The version reported by the orgo on PATH, e.g. "0.22.0"."""
 92    result = subprocess.run(
 93        ["orgo", "--version"], capture_output=True, text=True, check=False
 94    )
 95    if result.returncode != 0:
 96        print("Could not read the orgo version:", file=sys.stderr)
 97        print(result.stderr, file=sys.stderr)
 98        sys.exit(1)
 99    # `orgo --version` prints "orgo X.Y.Z". Anything else means the output format
100    # changed, and guessing at it would ship a wrong version to every page.
101    parts = result.stdout.split()
102    if len(parts) != 2 or parts[0] != "orgo":
103        print(f"Unexpected `orgo --version` output: {result.stdout!r}", file=sys.stderr)
104        sys.exit(1)
105    return parts[1]
106
107
108def rewrite_orgo_version(build_dir):
109    """Point the footer's orgo version at the orgo that just built the site.
110
111    Runs over every page rather than only the re-rendered ones, because the build cache
112    keys on a cache *format* version and not on orgo's release version: after a version
113    bump that leaves the format alone, the pages carried over from the previous build
114    would otherwise keep printing the old one.
115
116    The whole point of this is that the note cannot go stale, so a footer that no longer
117    matches is a failure and not a no-op — matching nothing anywhere means the note was
118    removed or its markup changed, and a silent pass would ship the template's
119    placeholder to every page.
120    """
121    version = orgo_version()
122    count = 0
123    for html in Path(build_dir).rglob("*.html"):
124        text = html.read_text(encoding="utf-8")
125        new_text, n = ORGO_NOTE.subn(rf"\g<1>{version}\g<2>", text)
126        if n and new_text != text:
127            html.write_text(new_text, encoding="utf-8")
128        count += n
129    if count == 0:
130        print(
131            "No 'built with orgo' note found in the build — the footer in "
132            "content/templates/base.html no longer matches ORGO_NOTE",
133            file=sys.stderr,
134        )
135        sys.exit(1)
136    print(f"Set the footer orgo version to {version} on {count} pages")
137
138
139# Stylesheet links, with or without a ?v= from an earlier build. Cloudflare and browsers
140# cache these for a year, so the URL has to change whenever the file does.
141STYLESHEETS = ("styles.css", "syntax.css")
142STYLESHEET_LINK = re.compile(
143    r'(<link rel="stylesheet" href="[^"]*?)(styles\.css|syntax\.css)(?:\?v=[0-9a-f]+)?(")'
144)
145
146
147def rewrite_stylesheet_versions(build_dir):
148    """Append ?v=<content hash> to every page's stylesheet links.
149
150    Runs over every page for the same reason as rewrite_orgo_version: a page carried
151    over from an earlier build still names the old hash. A build that matches no link
152    fails, because shipping unversioned links means the next CSS change sits behind
153    a year-long cache.
154    """
155    versions = {
156        name: hashlib.sha256((Path(build_dir) / name).read_bytes()).hexdigest()[:10]
157        for name in STYLESHEETS
158    }
159    count = 0
160    for html in Path(build_dir).rglob("*.html"):
161        text = html.read_text(encoding="utf-8")
162        new_text, n = STYLESHEET_LINK.subn(
163            lambda m: f"{m[1]}{m[2]}?v={versions[m[2]]}{m[3]}", text
164        )
165        if n and new_text != text:
166            html.write_text(new_text, encoding="utf-8")
167        count += n
168    if count == 0:
169        print(
170            "No stylesheet links found in the build — the <link> tags in "
171            "content/templates/base.html no longer match STYLESHEET_LINK",
172            file=sys.stderr,
173        )
174        sys.exit(1)
175    print(f"Versioned {count} stylesheet links: {versions}")
176
177
178def run_orgo_build(build_dir):
179    """
180    Build the site with orgo.
181
182    orgo reads content/orgo.toml, which holds the routes, templates and collections: the
183    blog index groups itself by year, the tags page is a collection, the recent-posts
184    list comes from the blog collection, and the sitemap is written once base_url is set.
185
186    The output directory is left in place between builds rather than wiped, because the
187    .orgo-cache.json inside it is what makes a build incremental: orgo re-renders only
188    the pages whose content, config or templates changed, re-emits any page whose output
189    is missing, and deletes the outputs of pages that have since been removed. Wiping the
190    directory would throw that away and force a full render every time. For a clean build
191    from scratch, delete the output directory by hand.
192
193    What is left around it is what orgo does not do: rewriting image URLs for the onion.
194    """
195    print("Building with orgo...")
196    result = subprocess.run(
197        ["orgo", "build", "content", "-o", str(build_dir), "--strict"],
198        stdout=subprocess.PIPE,
199        stderr=subprocess.STDOUT,
200        text=True,
201        check=False,
202    )
203    print(result.stdout, end="")
204    if result.returncode != 0:
205        print("orgo build failed", file=sys.stderr)
206        sys.exit(1)
207
208
209def deploy_to_server(build_dir, server, dry_run=False):
210    """Push the built site to the server, or show what pushing it would do.
211
212    The deploy deletes remote files the build no longer produces, so "what would
213    this remove" is a question worth being able to ask before answering it
214    irreversibly. DRY_RUN=true asks it: rsync connects and compares, then reports
215    instead of transferring.
216    """
217    remote_path = f"{server}:/var/www/cleberg.net/"
218    print(f"{'Would deploy' if dry_run else 'Deploying'} {build_dir}/ → {remote_path}")
219    cmd = [
220        "rsync",
221        "-rt",
222        # -t, or rsync gives every transferred file the time it arrived, no two
223        # deploys agree on a timestamp, and the next run re-sends the whole site
224        # because mtimes never match. With it, only what changed goes over.
225        "--delete-before",
226        # The build cache lives in the output directory because it describes it, but it
227        # is not part of the site. Excluding it also stops --delete removing it locally.
228        "--exclude",
229        ".orgo-cache.json",
230    ]
231    if dry_run:
232        # --itemize-changes because --dry-run alone prints almost nothing: the
233        # point is to name every file that would be sent or deleted.
234        cmd += ["--dry-run", "--itemize-changes"]
235    cmd += [f"{build_dir}/", remote_path]
236    run(cmd, "Error during rsync deployment:", echo=dry_run)
237
238
239def start_dev_server(build_dir):
240    print(f"Starting development HTTP server from {build_dir}/ on port 8000")
241    os.chdir(build_dir)
242    # This will run until interrupted (Ctrl+C)
243    try:
244        subprocess.run([sys.executable, "-m", "http.server", "8000"], check=True)
245    except KeyboardInterrupt:
246        print("\nDevelopment server stopped.")
247    except subprocess.CalledProcessError as e:
248        print(f"Error starting development server: {e}", file=sys.stderr)
249        sys.exit(1)
250
251
252def main():
253    prod = os.environ.get("ENV", "").casefold() == "prod"
254    if not prod:
255        run_ruff()
256
257    # One output directory per environment, because the two differ after orgo has run:
258    # production rewrites image URLs in place and development does not. Sharing a
259    # directory would let an incremental build reuse a page rendered for the other one —
260    # a dev preview showing /img/ paths that only resolve on the server. Production keeps
261    # .build/ because that is the directory the deploy and the CI manifest name.
262    build_dir = Path(".build" if prod else ".build-dev")
263
264    print(f"Environment: {'Production' if prod else 'Development'}")
265
266    if os.environ.get("BUILD", "").casefold() == "true":
267        run_orgo_build(build_dir)
268        rewrite_orgo_version(build_dir)
269        rewrite_stylesheet_versions(build_dir)
270        # The onion needs same-origin images; dev previews keep the absolute URLs.
271        # Runs over every page, not just the re-rendered ones, so a page carried over
272        # from an earlier build is rewritten too.
273        if prod:
274            rewrite_img_urls(build_dir)
275
276    if os.environ.get("DEPLOY", "").casefold() == "true":
277        if prod:
278            dry_run = os.environ.get("DRY_RUN", "").casefold() == "true"
279            print(
280                "Dry run — the server will not be modified"
281                if dry_run
282                else "Deploying to production..."
283            )
284            deploy_to_server(build_dir, "homelab", dry_run)
285        else:
286            start_dev_server(build_dir)
287
288
289if __name__ == "__main__":
290    main()