build.py
631 lines · 21387 bytes · executable
1#!/usr/bin/env python3
2"""
3This script automates the process of building, testing, and deploying the website.
4It handles tasks such as:
5
6- Removing and recreating the build directory.
7- Minifying CSS assets.
8- Running the Emacs publishing script to generate site content.
9- Updating the index.html file with the latest blog posts.
10- Optionally deploying the built site to a remote server.
11- Starting a local development server for previewing changes.
12
13Usage:
14 Set the environment variable ENV to 'prod' for production builds.
15 Run the script to perform the build process accordingly.
16
17Dependencies:
18 - Python 3
19 - Emacs with the publish.el script
20 - minify tool for CSS minification
21 - rsync for deployment
22
23Author:
24 Christian Cleberg <hello@cleberg.net>
25"""
26
27import os
28import re
29import shutil
30import subprocess
31import sys
32from datetime import datetime
33from html import escape
34from pathlib import Path
35from urllib.parse import quote
36
37SITE_TEMPLATE_VARS = {
38 "site_name": "cleberg.net",
39 "site_owner": "Christian Cleberg <hello@cleberg.net>",
40 "site_description": "Stillness amidst the chaos.",
41}
42
43
44def run_ruff():
45 print("Running ruff...")
46 for cmd in [["ruff", "check", "--fix"], ["ruff", "format"]]:
47 result = subprocess.run(cmd, capture_output=True, text=True, check=False)
48 if result.returncode != 0:
49 print(f"ruff error ({' '.join(cmd)}):")
50 print(result.stderr, file=sys.stderr)
51 sys.exit(1)
52
53
54def update_marked_section(
55 html_snippet,
56 template_path="./.build/index.html",
57 begin_marker="<!-- BEGIN_POSTS -->",
58 end_marker="<!-- END_POSTS -->",
59):
60 """
61 Read the file at `template_path`, replace everything between `begin_marker`
62 and `end_marker` with the provided html_snippet, and write the updated
63 content back to the same file.
64 """
65 with open(template_path, "r", encoding="utf-8") as f:
66 content = f.read()
67
68 # Find the indices of the markers
69 begin_index = content.find(begin_marker)
70 end_index = content.find(end_marker)
71
72 if begin_index == -1 or end_index == -1:
73 raise ValueError(f"Markers not found in {template_path}")
74
75 # Compute insertion points: after the end of begin_marker line, before end_marker
76 # Include the newline after BEGIN_POSTS
77 insert_start = begin_index + len(begin_marker)
78 # Ensure we capture the newline character if present
79 if content[insert_start : insert_start + 1] == "\n":
80 insert_start += 1
81
82 # If there is a newline before END_POSTS, trim trailing whitespace from snippet block
83 # We will preserve indentation of BEGIN_POSTS line
84 indent = ""
85 # Determine the indentation by looking at characters after the newline that follows begin_marker
86 lines_after_begin = content[begin_index:].splitlines(True)
87 if len(lines_after_begin) > 1:
88 # The second line starts with the indentation to preserve
89 second_line = lines_after_begin[1]
90 indent = ""
91 for ch in second_line:
92 if ch.isspace():
93 indent += ch
94 else:
95 break
96
97 # Prepare the replacement block: indent each line of html_snippet
98 snippet_lines = html_snippet.splitlines()
99 indented_snippet = "\n".join(indent + line for line in snippet_lines) + "\n"
100
101 # Compute the position just before end_marker (excluding any preceding whitespace/newline)
102 end_line_start = content.rfind("\n", 0, end_index)
103 if end_line_start == -1:
104 end_line_start = end_index
105
106 # Construct the new content
107 new_content = content[:insert_start] + indented_snippet + content[end_line_start:]
108
109 # Write back to index.html
110 with open(template_path, "w", encoding="utf-8") as f:
111 f.write(new_content)
112
113
114def render_base_template(main_html, subtitle="", title=None):
115 """
116 Render a small subset of the site's shared templates for Python-generated
117 pages that still need to follow the common chrome.
118 """
119 base_template = Path("theme/templates/base.html").read_text(encoding="utf-8")
120
121 if title is None:
122 title = SITE_TEMPLATE_VARS["site_name"]
123
124 rendered = base_template
125 rendered = rendered.replace(
126 "{% block subtitle %}{% endblock %}",
127 escape(subtitle),
128 )
129 rendered = rendered.replace(
130 '{% block title %}{{ site_name | default("cleberg.net") }}{% endblock %}',
131 escape(title),
132 )
133 rendered = rendered.replace(
134 '{% if site_owner is defined %}<meta name="author" content="{{ site_owner }}">{% endif %}',
135 f'<meta name="author" content="{escape(SITE_TEMPLATE_VARS["site_owner"])}">',
136 )
137 rendered = rendered.replace(
138 '{% if site_description is defined %}<meta name="description" content="{{ site_description }}">{% endif %}',
139 f'<meta name="description" content="{escape(SITE_TEMPLATE_VARS["site_description"])}">',
140 )
141 rendered = rendered.replace(
142 '{% if site_keywords is defined %}<meta name="keywords" content="{{ site_keywords }}">{% endif %}',
143 "",
144 )
145 rendered = rendered.replace("{% block meta %}{% endblock %}", "")
146 rendered = rendered.replace("{% block head %}", "")
147 rendered = rendered.replace("{% endblock %}", "", 1)
148 rendered = rendered.replace("{% block main %}{% endblock %}", main_html)
149
150 return rendered
151
152
153def render_tags_page_html(tags_html):
154 """
155 Render the tags page by using the shared tags/base templates instead of a
156 hand-authored standalone HTML document.
157 """
158 tags_template = Path("theme/templates/tags.html").read_text(encoding="utf-8")
159
160 main_html = tags_template
161 main_html = main_html.replace('{% extends "base.html" %}', "")
162 main_html = main_html.replace(
163 "{% block subtitle %}tags - {% endblock %}",
164 "",
165 )
166 main_html = main_html.replace("{% block main %}", "")
167 main_html = main_html.replace("{% endblock %}", "")
168 main_html = main_html.replace("<!-- BEGIN_TAGS -->\n<!-- END_TAGS -->", tags_html)
169
170 return render_base_template(main_html.strip(), subtitle="tags - ")
171
172
173def get_blog_posts(content_dir="./content/blog"):
174 """
175 Scan blog posts and return normalized metadata for non-draft entries.
176 """
177 posts = []
178
179 header_patterns = {
180 "title": re.compile(r"^#\+title:\s*(.+)$", re.IGNORECASE),
181 "date": re.compile(r"^#\+date:\s*[\[<](\d{4}-\d{2}-\d{2})"),
182 "slug": re.compile(r"^#\+slug:\s*(.+)$", re.IGNORECASE),
183 "tags": re.compile(r"^#\+filetags:\s*(.+)$", re.IGNORECASE),
184 "draft": re.compile(r"^#\+draft:\s*(.+)$", re.IGNORECASE),
185 }
186
187 for org_path in Path(content_dir).glob("*.org"):
188 title = None
189 date_str = None
190 slug = None
191 tags = []
192 is_draft = False
193
194 with org_path.open("r", encoding="utf-8") as f:
195 for line in f:
196 if title is None:
197 m = header_patterns["title"].match(line)
198 if m:
199 title = m.group(1).strip()
200 continue
201
202 if date_str is None:
203 m = header_patterns["date"].match(line)
204 if m:
205 # date_str is just YYYY-MM-DD
206 date_str = m.group(1)
207 continue
208
209 if slug is None:
210 m = header_patterns["slug"].match(line)
211 if m:
212 slug = m.group(1).strip()
213 continue
214
215 if not tags:
216 m = header_patterns["tags"].match(line)
217 if m:
218 raw = m.group(1).strip().strip(":")
219 tags = [t.strip() for t in raw.split(":") if t.strip()]
220 continue
221
222 m = header_patterns["draft"].match(line)
223 if m:
224 draft_value = m.group(1).strip().lower()
225 if draft_value != "nil":
226 is_draft = True
227 break
228 continue
229
230 # Stop scanning once we have all required fields
231 if title and date_str and slug:
232 break
233
234 if is_draft:
235 continue
236
237 if title and date_str and slug:
238 try:
239 date_obj = datetime.strptime(date_str, "%Y-%m-%d")
240 date_full = date_obj.strftime("%Y-%m-%d")
241 except ValueError:
242 # Skip files with invalid date format
243 continue
244
245 posts.append(
246 {
247 "title": title,
248 "date_str": date_str,
249 "date_obj": date_obj,
250 "date_full": date_full,
251 "slug": slug,
252 "tags": tags,
253 }
254 )
255
256 posts.sort(key=lambda x: x["date_obj"], reverse=True)
257 return posts
258
259
260def get_recent_posts_html(content_dir="./content/blog", num_posts=3):
261 """
262 Return an HTML snippet for the `num_posts` most recent blog posts.
263 """
264 recent = get_blog_posts(content_dir)[:num_posts]
265
266 lines = []
267 for post in recent:
268 lines.append('\t<li class="post-list-item">')
269 lines.append(
270 f'\t\t<time datetime="{post["date_str"]}">{post["date_full"]}</time>'
271 )
272 lines.append(f'\t\t<a href="/blog/{post["slug"]}.html">{post["title"]}</a>')
273 lines.append("\t</li>")
274
275 return "\n".join(lines)
276
277
278def prompt(prompt_text):
279 try:
280 return input(prompt_text).strip()
281 except EOFError:
282 return ""
283
284
285def remove_build_directory(build_dir):
286 if build_dir.exists():
287 print(f"Removing previous build directory: {build_dir}/")
288 shutil.rmtree(build_dir)
289 build_dir.mkdir(parents=True, exist_ok=True)
290
291
292def minify_css(src_css, dest_css):
293 print(f"Minifying CSS: {src_css} → {dest_css}")
294 result = subprocess.run(
295 ["minify", "-o", str(dest_css), str(src_css)],
296 capture_output=True,
297 text=True,
298 check=False,
299 )
300 if result.returncode != 0:
301 print("Error during CSS minification:")
302 print(result.stderr, file=sys.stderr)
303 sys.exit(1)
304
305
306def minify_html(src_html, dest_html):
307 print(f"Minifying HTML: {src_html} → {dest_html}")
308 result = subprocess.run(
309 ["minify", "-o", str(dest_html), str(src_html)],
310 capture_output=True,
311 text=True,
312 check=False,
313 )
314 if result.returncode != 0:
315 print("Error during HTML minification:")
316 print(result.stderr, file=sys.stderr)
317 sys.exit(1)
318
319
320def rewrite_img_urls(build_dir=".build"):
321 """
322 Rewrite absolute img.cleberg.net URLs to root-relative /img/ paths so the
323 onion serves images from its own origin instead of fetching them off-onion.
324 Production only: dev builds keep the absolute URLs so local previews still
325 load images from the live image host.
326
327 Requires the server to serve /var/www/img/ at /img/ on the cleberg.net vhost
328 (e.g. `ln -s /var/www/img /var/www/cleberg.net/img`).
329
330 The pattern tolerates a stray number of slashes after the scheme
331 (https:/img, https:///img, ...) so a typo in the org source can't silently
332 slip through un-rewritten and ship a broken cross-origin URL.
333 """
334 pattern = re.compile(r"https:/+img\.cleberg\.net/")
335 count = 0
336 for html in Path(build_dir).rglob("*.html"):
337 text = html.read_text(encoding="utf-8")
338 new_text, n = pattern.subn("/img/", text)
339 if n:
340 count += n
341 html.write_text(new_text, encoding="utf-8")
342 print(f"Rewrote {count} img.cleberg.net references to /img/")
343
344
345def run_emacs_publish(dev_mode=True):
346 mode = "development" if dev_mode else "production"
347 print(f"Running Emacs publish script ({mode})...")
348
349 result = subprocess.run(
350 ["emacs", "--script", "publish.el"],
351 stdout=subprocess.PIPE,
352 stderr=subprocess.STDOUT,
353 text=True,
354 check=False,
355 )
356
357 if result.returncode != 0:
358 print("Error running publish.el output:")
359 print(result.stdout)
360 sys.exit(1)
361
362 annoying_file = Path(".build/cleberg-net.html")
363 if annoying_file.exists():
364 os.remove(annoying_file)
365 else:
366 print(
367 "Warning: .build/cleberg-net.html not found, but Emacs exited successfully."
368 )
369
370
371def copy_org_sources(content_dir="./content", build_dir="./.build/org"):
372 print(f"Copying org sources: {content_dir} → {build_dir}")
373 if os.path.exists(build_dir):
374 shutil.rmtree(build_dir)
375
376 slug_pattern = re.compile(r"^#\+slug:\s*(.+)$", re.IGNORECASE)
377
378 for src_path in Path(content_dir).rglob("*.org"):
379 rel_dir = src_path.parent.relative_to(content_dir)
380 dest_dir = Path(build_dir) / rel_dir
381 dest_dir.mkdir(parents=True, exist_ok=True)
382
383 # Try to extract slug from file headers
384 slug = None
385 with src_path.open("r", encoding="utf-8") as f:
386 for line in f:
387 m = slug_pattern.match(line)
388 if m:
389 slug = m.group(1).strip()
390 break
391
392 dest_name = f"{slug}.org" if slug else src_path.name
393 shutil.copy2(src_path, dest_dir / dest_name)
394 if slug:
395 print(f" {src_path.name} → {dest_name}")
396
397
398def generate_sitemap(build_dir=".build", base_url="https://cleberg.net"):
399 """
400 Generates a sitemap.xml based on contents of the .build directory.
401 Only includes .html files (except 404.html).
402 """
403 sitemap_entries = []
404 for root, dirs, files in os.walk(build_dir):
405 for filename in files:
406 if filename.endswith(".html") and filename != "404.html":
407 full_path = os.path.join(root, filename)
408 rel_path = os.path.relpath(full_path, build_dir)
409 url_path = "/" + quote(rel_path.replace(os.sep, "/"))
410 # Remove index.html for cleaner URLs
411 if url_path.endswith("/index.html"):
412 url_path = url_path[:-10] or "/"
413 elif url_path == "/index.html":
414 url_path = "/"
415 loc = f"{base_url}{url_path}"
416
417 # Last modified time
418 lastmod = datetime.fromtimestamp(os.path.getmtime(full_path)).strftime(
419 "%Y-%m-%d"
420 )
421
422 sitemap_entries.append(f""" <url>
423 <loc>{loc}</loc>
424 <lastmod>{lastmod}</lastmod>
425 </url>""")
426
427 sitemap_xml = f"""<?xml version="1.0" encoding="UTF-8"?>
428<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
429{os.linesep.join(sitemap_entries)}
430</urlset>
431"""
432 # Write to .build/sitemap.xml
433 sitemap_path = os.path.join(build_dir, "sitemap.xml")
434 with open(sitemap_path, "w", encoding="utf-8") as f:
435 f.write(sitemap_xml)
436 print(f"Sitemap generated at {sitemap_path} with {len(sitemap_entries)} entries.")
437
438
439def inject_blog_year_separators(blog_index_path="./.build/blog/index.html"):
440 """
441 Post-processes the rendered blog index to inject year separator <li> elements
442 between groups of posts. Weblorg/templatel doesn't support mutable loop state,
443 so this runs after the HTML is generated.
444
445 Finds each <li class="post-list-item"> that contains a <time datetime="YYYY-MM-DD">,
446 and inserts <li class="post-list-year">YYYY</li> before the first post of each year.
447 """
448 path = Path(blog_index_path)
449 if not path.exists():
450 print(f"Warning: {blog_index_path} not found, skipping year separators.")
451 return
452
453 content = path.read_text(encoding="utf-8")
454
455 # Match each post list item, capturing the date and the full element
456 item_pattern = re.compile(
457 r'(<li class="post-list-item">.*?</li>)',
458 re.DOTALL,
459 )
460 date_pattern = re.compile(r"datetime=['\"]?(\d{4})-\d{2}-\d{2}['\"]?")
461
462 current_year = None
463
464 def replace_item(m):
465 nonlocal current_year
466 item_html = m.group(1)
467 date_match = date_pattern.search(item_html)
468 if not date_match:
469 return item_html
470 year = date_match.group(1)
471 if year != current_year:
472 current_year = year
473 separator = f'<li class="post-list-year">{year}</li>'
474 return f"{separator}\n{item_html}"
475 return item_html
476
477 new_content = item_pattern.sub(replace_item, content)
478 path.write_text(new_content, encoding="utf-8")
479 print(f"Blog year separators injected into {blog_index_path}")
480
481
482def get_tags_html(content_dir="./content/blog"):
483 """
484 Build the tag index HTML snippet for the rendered tags template.
485 """
486 preferred_tag_order = [
487 "audit",
488 "emacs",
489 "development",
490 "ios",
491 "linux",
492 "personal",
493 "privacy",
494 "security",
495 "self-hosting",
496 "web",
497 ]
498
499 tag_map = {}
500
501 for post in get_blog_posts(content_dir):
502 for tag in post["tags"]:
503 tag_map.setdefault(tag, []).append(
504 {
505 "title": post["title"],
506 "slug": post["slug"],
507 "date_obj": post["date_obj"],
508 "date_str": post["date_str"],
509 }
510 )
511
512 ordered_tags = [tag for tag in preferred_tag_order if tag in tag_map]
513 ordered_tags.extend(
514 sorted(tag for tag in tag_map if tag not in preferred_tag_order)
515 )
516
517 for tag in ordered_tags:
518 tag_map[tag].sort(key=lambda x: x["date_obj"], reverse=True)
519
520 toc_items = "".join(
521 f'<li><a href="#{tag}">{tag}</a> <span class="tag-count">({len(tag_map[tag])})</span></li>'
522 for tag in ordered_tags
523 )
524
525 sections = []
526 for tag in ordered_tags:
527 posts = tag_map[tag]
528 items = "\n".join(
529 f'<li class="post-list-item">'
530 f'<time datetime="{p["date_str"]}">{p["date_str"]}</time>'
531 f'<a href="/blog/{p["slug"]}.html">{p["title"]}</a>'
532 f"</li>"
533 for p in posts
534 )
535 sections.append(
536 f'<h2 id="{tag}">{tag}</h2>\n<ul class="post-list">\n{items}\n</ul>'
537 )
538
539 return f'<ul class="tag-toc">{toc_items}</ul>\n' + "".join(
540 f"<section>{section}</section>" for section in sections
541 )
542
543
544def generate_tags_page(content_dir="./content/blog", build_dir="./.build"):
545 """
546 Render the tags page using the shared template structure.
547 """
548 tags_html = get_tags_html(content_dir)
549 out_path = Path(build_dir) / "tags" / "index.html"
550 out_path.parent.mkdir(parents=True, exist_ok=True)
551 out_path.write_text(render_tags_page_html(tags_html), encoding="utf-8")
552 print(f"Tags page written to {out_path}")
553
554
555def deploy_to_server(build_dir, server):
556 remote_path = f"{server}:/var/www/cleberg.net/"
557 print(f"Deploying .build/ → {remote_path}")
558 result = subprocess.run(
559 ["rsync", "-r", "--delete-before", f"{build_dir}/", remote_path],
560 capture_output=True,
561 text=True,
562 check=False,
563 )
564 if result.returncode != 0:
565 print("Error during rsync deployment:")
566 print(result.stderr, file=sys.stderr)
567 sys.exit(1)
568
569
570def start_dev_server(build_dir):
571 print(f"Starting development HTTP server from {build_dir}/ on port 8000")
572 os.chdir(build_dir)
573 # This will run until interrupted (Ctrl+C)
574 try:
575 subprocess.run([sys.executable, "-m", "http.server", "8000"], check=True)
576 except KeyboardInterrupt:
577 print("\nDevelopment server stopped.")
578 except subprocess.CalledProcessError as e:
579 print(f"Error starting development server: {e}", file=sys.stderr)
580 sys.exit(1)
581
582
583def main():
584 env = os.environ.get("ENV", "").casefold()
585 if env != "prod":
586 run_ruff()
587 html_snippet = get_recent_posts_html("./content/blog", num_posts=3)
588
589 build_dir = Path(".build")
590 theme_dir = Path("theme/static")
591 css_src = theme_dir / "styles.css"
592 css_min = theme_dir / "styles.min.css"
593
594 build = os.environ.get("BUILD", "").casefold() == "true"
595 deploy = os.environ.get("DEPLOY", "").casefold() == "true"
596
597 if env == "prod":
598 print("Environment: Production")
599 if build:
600 remove_build_directory(build_dir)
601 minify_css(css_src, css_min)
602 run_emacs_publish(dev_mode=False)
603 copy_org_sources()
604 update_marked_section(html_snippet)
605 inject_blog_year_separators()
606 generate_tags_page()
607 rewrite_img_urls(build_dir)
608 # minify_html("./.build/index.html", "./.build/index.html")
609 generate_sitemap()
610 if deploy:
611 print("Deploying to production...")
612 deploy_to_server(build_dir, "homelab")
613 return
614 else:
615 print("Environment: Development")
616 if build:
617 remove_build_directory(build_dir)
618 minify_css(css_src, css_min)
619 run_emacs_publish(dev_mode=True)
620 copy_org_sources()
621 update_marked_section(html_snippet)
622 inject_blog_year_separators()
623 generate_tags_page()
624 minify_html("./.build/index.html", "./.build/index.html")
625 generate_sitemap()
626 if deploy:
627 start_dev_server(build_dir)
628
629
630if __name__ == "__main__":
631 main()