krz/orgo
Lightning fast org-mode static site generator.
clone: git clone https://gitbay.org/krz/orgo.git
1//! Site build: walk a source directory, PARSE every `.org` file, INDEX their targets,
2//! then RESOLVE + RENDER + TEMPLATE each page into a linked static site, copying
3//! non-`.org` assets through unchanged (spec §2.1 DISCOVER…EMIT).
4//!
5//! v0.3 wires in the incremental layer (spec §4, [`crate::incremental`]): a persisted
6//! cache manifest lets a rebuild re-render only the pages whose composed `render_key`
7//! changed, plus the pages that *link into* a changed file's targets (the dependency
8//! graph, spec §4.3). Unchanged pages keep their existing on-disk output untouched.
9//! `--no-cache` forces a full rebuild; the cache is never a correctness dependency, so a
10//! full rebuild and an incremental rebuild produce byte-identical output.
11
12use std::collections::HashSet;
13use std::fs;
14
15use anyhow::{Context, Result};
16use camino::{Utf8Path, Utf8PathBuf};
17use walkdir::WalkDir;
18
19use crate::incremental::{
20 self, combine, config_hash, render_key, resolved_links_hash, site_structure_hash,
21 template_hash, BuildConfig, DepGraph, Hash, Manifest, PageRecord, CACHE_FORMAT_VERSION,
22};
23use crate::index::{document_targets, SymbolTable, TargetId};
24use crate::model::{ContentHash, Document};
25use crate::parser::parse;
26use crate::render::{render, Html, SyntectHighlighter};
27use crate::resolve::resolve;
28use crate::template::{template_sources, NavItem, Templater};
29use crate::util::output_url;
30
31/// A fully built page: source and output paths (relative to their roots) and its
32/// final templated HTML.
33#[derive(Debug, Clone)]
34pub struct BuiltPage {
35 pub source: Utf8PathBuf,
36 pub output: Utf8PathBuf,
37 pub title: String,
38 pub html: String,
39}
40
41/// Unresolved internal links found during a build: `(page, target)` (spec §4.3.4).
42pub type BrokenLinks = Vec<(Utf8PathBuf, TargetId)>;
43
44/// Options controlling a site build.
45#[derive(Debug, Clone, Default)]
46pub struct BuildOptions {
47 /// Bypass the incremental cache and re-render every page (spec §4.5).
48 pub no_cache: bool,
49 /// Treat broken internal links as a build error rather than a warning (spec §4.3.4).
50 pub strict: bool,
51}
52
53/// Summary of a site build.
54#[derive(Debug, Default)]
55pub struct SiteReport {
56 /// Every output page (rendered this build or reused from cache).
57 pub pages: Vec<Utf8PathBuf>,
58 /// Pages actually re-rendered and written this build (the invalidation set).
59 pub rendered: Vec<Utf8PathBuf>,
60 /// Pages whose existing on-disk output was reused unchanged (spec §4.1 skip rule).
61 pub skipped: Vec<Utf8PathBuf>,
62 pub assets: Vec<Utf8PathBuf>,
63 /// Unresolved internal links: `(page, target)`. Warnings, not failures (spec §4.3.4).
64 pub broken: Vec<(Utf8PathBuf, TargetId)>,
65}
66
67/// Everything a build needs about one page *before* the decision to render it: its
68/// hashes, its resolved element tree, and the dependency edges it participates in.
69struct PagePrep {
70 source: Utf8PathBuf,
71 output: Utf8PathBuf,
72 title: String,
73 content_hash: ContentHash,
74 resolved: crate::resolve::ResolvedDoc,
75 used: HashSet<TargetId>,
76 defines: HashSet<TargetId>,
77 broken: Vec<TargetId>,
78 nav: Vec<NavItem>,
79}
80
81/// DISCOVER + PARSE + INDEX + RESOLVE the whole site, returning per-page prep and the
82/// global symbol table. RENDER/TEMPLATE is deferred to the caller so the incremental
83/// build can render only the pages it must. PARSE/INDEX/RESOLVE are cheap and pure, so
84/// they run for every file each build; the incremental win is on RENDER + EMIT (spec §4.4).
85fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> {
86 let (org_rel, _assets) = discover(src)?;
87
88 // PARSE every file (relative paths keep snapshots and links machine-independent).
89 let mut docs: Vec<Document> = Vec::new();
90 for rel in &org_rel {
91 let abs = src.join(rel);
92 let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?;
93 let doc = parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))?;
94 docs.push(doc);
95 }
96
97 // INDEX: collect every link target across the corpus.
98 let mut symbols = SymbolTable::new();
99 for doc in &docs {
100 symbols.index_document(doc);
101 }
102
103 // Nav is global; titles come from #+TITLE (falling back to the file stem).
104 let entries: Vec<(Utf8PathBuf, String)> = docs
105 .iter()
106 .map(|d| (d.source_path.clone(), page_title(d)))
107 .collect();
108
109 let mut pages = Vec::new();
110 for doc in &docs {
111 let out = resolve(doc, &symbols);
112 let used: HashSet<TargetId> = out.used_targets.iter().cloned().collect();
113 let broken: Vec<TargetId> = out.broken.iter().map(|b| b.target.clone()).collect();
114 let defines: HashSet<TargetId> = document_targets(doc).into_iter().collect();
115
116 // Nav links are relative to *this* page (spec URL scheme, §8 Q3).
117 let nav: Vec<NavItem> = entries
118 .iter()
119 .map(|(path, title)| NavItem {
120 title: title.clone(),
121 url: output_url(&doc.source_path, path, None),
122 })
123 .collect();
124
125 pages.push(PagePrep {
126 source: doc.source_path.clone(),
127 output: doc.source_path.with_extension("html"),
128 title: page_title(doc),
129 content_hash: doc.content_hash,
130 resolved: out.resolved,
131 used,
132 defines,
133 broken,
134 nav,
135 });
136 }
137
138 Ok((pages, symbols))
139}
140
141/// Parse + index + resolve + render + template a whole site *in memory*, without
142/// touching the output directory. Shared by the tests (full render, every page).
143pub fn render_site(src: &Utf8Path) -> Result<(Vec<BuiltPage>, BrokenLinks)> {
144 let (preps, _symbols) = prepare_pages(src)?;
145 let highlighter = SyntectHighlighter;
146 let templater = Templater::new();
147
148 let mut pages = Vec::new();
149 let mut broken = Vec::new();
150 for p in &preps {
151 for t in &p.broken {
152 broken.push((p.source.clone(), t.clone()));
153 }
154 let html = render_page(&templater, &highlighter, p)?;
155 pages.push(BuiltPage {
156 source: p.source.clone(),
157 output: p.output.clone(),
158 title: p.title.clone(),
159 html,
160 });
161 }
162 Ok((pages, broken))
163}
164
165/// RENDER + TEMPLATE one prepared page into its final HTML string.
166fn render_page(
167 templater: &Templater,
168 highlighter: &SyntectHighlighter,
169 p: &PagePrep,
170) -> Result<String> {
171 let Html(fragment) = render(&p.resolved, highlighter);
172 templater
173 .render_page(&p.title, &fragment, &p.nav)
174 .with_context(|| format!("templating {}", p.source))
175}
176
177/// Full site build with the incremental layer (spec §4). Renders only the pages whose
178/// `render_key` changed or that link into a changed file's targets; reuses the on-disk
179/// output of everything else; persists an updated cache manifest.
180pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result<SiteReport> {
181 let (_org_rel, assets) = discover(src)?;
182 let (preps, symbols) = prepare_pages(src)?;
183
184 // The global hash classes (spec §4.1): a change in any invalidates the site. The
185 // config hash is combined with a site-structure hash because the nav bar — global
186 // chrome on every page — is built from every page's (path, title), so a title/path
187 // change or a page add/remove must re-render every page (else stale nav on disk).
188 let cfg = BuildConfig::default();
189 let nav_entries: Vec<(String, String)> = preps
190 .iter()
191 .map(|p| (p.source.to_string(), p.title.clone()))
192 .collect();
193 let cfg_hash = combine(config_hash(&cfg), site_structure_hash(&nav_entries));
194 let tmpl_hash = template_hash(template_sources());
195
196 // Compose each page's render key and record its dependency edges.
197 let mut new_graph = DepGraph::default();
198 let mut new_records: Vec<(Utf8PathBuf, PageRecord, Hash)> = Vec::new();
199 for p in &preps {
200 let rlh = resolved_links_hash(&p.source, &p.used, &symbols);
201 let key = render_key(p.content_hash, rlh, cfg_hash, tmpl_hash);
202 new_graph.defines.insert(p.source.clone(), p.defines.clone());
203 new_graph.uses.insert(p.source.clone(), p.used.clone());
204 new_records.push((
205 p.source.clone(),
206 PageRecord {
207 content_hash: p.content_hash,
208 render_key: key,
209 output_path: p.output.clone(),
210 },
211 key,
212 ));
213 }
214
215 // Load the prior manifest (unless bypassed). Absent/corrupt/version-mismatch ⇒ None
216 // ⇒ full rebuild (spec §4.5).
217 let prior = if opts.no_cache {
218 None
219 } else {
220 incremental::load_manifest(out)
221 };
222
223 let rebuild: HashSet<Utf8PathBuf> = compute_rebuild_set(
224 &preps,
225 &new_records,
226 &new_graph,
227 cfg_hash,
228 tmpl_hash,
229 out,
230 prior.as_ref(),
231 );
232
233 // Delete outputs for pages that existed last build but are gone now (spec §4.3 step 1:
234 // removed files). Their targets are already in the merged graph, so their linkers were
235 // invalidated above.
236 if let Some(prior) = &prior {
237 let current: HashSet<&Utf8PathBuf> = preps.iter().map(|p| &p.source).collect();
238 for (src_path, rec) in &prior.pages {
239 if !current.contains(src_path) {
240 let dest = out.join(&rec.output_path);
241 let _ = fs::remove_file(&dest);
242 }
243 }
244 }
245
246 let highlighter = SyntectHighlighter;
247 let templater = Templater::new();
248 let mut report = SiteReport::default();
249
250 for p in &preps {
251 for t in &p.broken {
252 report.broken.push((p.source.clone(), t.clone()));
253 }
254 report.pages.push(p.output.clone());
255
256 let dest = out.join(&p.output);
257 if rebuild.contains(&p.source) {
258 if let Some(parent) = dest.parent() {
259 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
260 }
261 let html = render_page(&templater, &highlighter, p)?;
262 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
263 report.rendered.push(p.output.clone());
264 } else {
265 // Skip: the on-disk output is already correct (spec §4.1). Leave it untouched.
266 report.skipped.push(p.output.clone());
267 }
268 }
269
270 // Assets are a dumb copy in v0.3 (spec §8 Q11): copy every run. Cheap, and keeps the
271 // full-vs-incremental byte equivalence trivially true for non-`.org` files.
272 for rel in &assets {
273 let from = src.join(rel);
274 let dest = out.join(rel);
275 if let Some(parent) = dest.parent() {
276 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
277 }
278 fs::copy(&from, &dest).with_context(|| format!("copying {from} -> {dest}"))?;
279 report.assets.push(rel.clone());
280 }
281
282 // Persist the manifest for the next build.
283 let manifest = Manifest {
284 format_version: CACHE_FORMAT_VERSION,
285 config_hash: Some(cfg_hash),
286 template_hash: Some(tmpl_hash),
287 pages: new_records
288 .into_iter()
289 .map(|(src_path, rec, _)| (src_path, rec))
290 .collect(),
291 graph: new_graph,
292 };
293 incremental::save_manifest(out, &manifest)
294 .with_context(|| format!("writing cache manifest under {out}"))?;
295
296 if opts.strict && !report.broken.is_empty() {
297 for (page, target) in &report.broken {
298 eprintln!("error: unresolved link in {page}: {target:?}");
299 }
300 anyhow::bail!(
301 "{} unresolved internal link(s) under --strict",
302 report.broken.len()
303 );
304 }
305 for (page, target) in &report.broken {
306 eprintln!("warning: unresolved link in {page}: {target:?}");
307 }
308
309 Ok(report)
310}
311
312/// The set of source files to (re)render this build (spec §4.3 invalidation algorithm),
313/// as the union of:
314/// - **no prior cache** (absent/corrupt/version-mismatch/`--no-cache`) ⇒ every page;
315/// - a **global** config- or template-hash change ⇒ every page (spec §4.1);
316/// - **content-changed** files ∪ pages that link into a changed file's targets, via the
317/// dependency graph merged with the prior build's `defines` (spec §4.3, so a removed
318/// target still invalidates its linkers);
319/// - any page whose composed **render_key** differs from the cached one (catches URL
320/// changes on linked targets precisely);
321/// - any page whose **output file is missing** on disk.
322fn compute_rebuild_set(
323 preps: &[PagePrep],
324 new_records: &[(Utf8PathBuf, PageRecord, Hash)],
325 new_graph: &DepGraph,
326 cfg_hash: Hash,
327 tmpl_hash: Hash,
328 out: &Utf8Path,
329 prior: Option<&Manifest>,
330) -> HashSet<Utf8PathBuf> {
331 let all: HashSet<Utf8PathBuf> = preps.iter().map(|p| p.source.clone()).collect();
332
333 let Some(prior) = prior else {
334 return all; // No usable cache ⇒ full rebuild.
335 };
336
337 // A global config/template change invalidates every page (spec §4.1).
338 if prior.config_hash != Some(cfg_hash) || prior.template_hash != Some(tmpl_hash) {
339 return all;
340 }
341
342 // Content-changed = hash differs from the cached record, or the file is new.
343 let mut changed: HashSet<Utf8PathBuf> = HashSet::new();
344 for p in preps {
345 match prior.pages.get(&p.source) {
346 Some(rec) if rec.content_hash == p.content_hash => {}
347 _ => {
348 changed.insert(p.source.clone());
349 }
350 }
351 }
352
353 // Graph expansion: changed files ∪ pages that link into a changed file's targets.
354 // Merge prior `defines` so a target a changed file removed still pulls its linkers.
355 let merged = prior.graph.merged_defines_with(new_graph);
356 let mut rebuild = incremental::invalidation_set(&changed, &merged);
357
358 // Precise render_key delta (catches a linked target's URL change; also a belt for the
359 // graph). A page whose render_key matches the cache and whose output exists is correct.
360 for (src_path, _rec, key) in new_records {
361 let unchanged = prior
362 .pages
363 .get(src_path)
364 .map(|old| old.render_key == *key)
365 .unwrap_or(false);
366 if !unchanged {
367 rebuild.insert(src_path.clone());
368 }
369 }
370
371 // Any page whose output file is missing must be re-emitted regardless.
372 for p in preps {
373 if !out.join(&p.output).exists() {
374 rebuild.insert(p.source.clone());
375 }
376 }
377
378 rebuild
379}
380
381/// Walk `src`, returning `.org` source paths and non-`.org` asset paths, both relative
382/// to `src` and sorted for deterministic output. The cache manifest is not an asset.
383fn discover(src: &Utf8Path) -> Result<(Vec<Utf8PathBuf>, Vec<Utf8PathBuf>)> {
384 let mut org = Vec::new();
385 let mut assets = Vec::new();
386 for entry in WalkDir::new(src).sort_by_file_name() {
387 let entry = entry.with_context(|| format!("walking {src}"))?;
388 if !entry.file_type().is_file() {
389 continue;
390 }
391 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
392 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
393 let rel = abs
394 .strip_prefix(src)
395 .map(|p| p.to_owned())
396 .unwrap_or_else(|_| abs.clone());
397 if rel.extension() == Some("org") {
398 org.push(rel);
399 } else {
400 assets.push(rel);
401 }
402 }
403 org.sort();
404 assets.sort();
405 Ok((org, assets))
406}
407
408fn page_title(doc: &Document) -> String {
409 doc.keywords
410 .entries
411 .iter()
412 .find(|(k, _)| k.eq_ignore_ascii_case("TITLE"))
413 .map(|(_, v)| v.clone())
414 .unwrap_or_else(|| {
415 doc.source_path
416 .file_stem()
417 .unwrap_or("untitled")
418 .to_string()
419 })
420}