krz/orgo
Lightning fast org-mode static site generator.
clone: git clone https://gitbay.org/krz/orgo.git
1//! Site build: walk a source directory, PARSE every `.org` file, INDEX their targets,
2//! then RESOLVE + RENDER + TEMPLATE each page into a linked static site, copying
3//! non-`.org` assets through unchanged (spec §2.1 DISCOVER…EMIT).
4//!
5//! v0.3 wires in the incremental layer (spec §4, [`crate::incremental`]): a persisted
6//! cache manifest lets a rebuild re-render only the pages whose composed `render_key`
7//! changed, plus the pages that *link into* a changed file's targets (the dependency
8//! graph, spec §4.3). Unchanged pages keep their existing on-disk output untouched.
9//! `--no-cache` forces a full rebuild; the cache is never a correctness dependency, so a
10//! full rebuild and an incremental rebuild produce byte-identical output.
11
12use std::collections::{HashMap, HashSet};
13use std::fs;
14
15use anyhow::{Context, Result};
16use camino::{Utf8Path, Utf8PathBuf};
17use rayon::prelude::*;
18use walkdir::WalkDir;
19
20use crate::incremental::{
21 self, combine, config_hash, render_key, resolved_links_hash, site_structure_hash,
22 site_structure_hash_ordered, template_hash, DepGraph, Hash, Manifest, PageRecord,
23 CACHE_FORMAT_VERSION,
24};
25use crate::index::{document_targets, SymbolTable, TargetId};
26use crate::model::{ContentHash, Diagnostic, Document};
27use crate::parser::parse;
28use crate::render::{self, render_with, Html, RenderOptions, SyntectHighlighter};
29use crate::resolve::resolve;
30use crate::config::{self, Config, NavMode, SortKey, SortOrder};
31use crate::template::{
32 GroupContext, NavItem, PageContext, Paginator, PaginatorPage, RenderContext, SiteContext,
33 Templater,
34};
35use crate::theme;
36use crate::util::{
37 document_text, first_paragraph, is_draft, iso_date, iso_time, option_enabled,
38 output_path, output_url,
39 relative_root, slugify, table_of_contents,
40};
41
42/// Reading speed for [`PageContext::reading_time`]. 200 wpm is the conventional figure
43/// for prose on screen.
44const WORDS_PER_MINUTE: usize = 200;
45
46/// A fully built page: source and output paths (relative to their roots) and its
47/// final templated HTML.
48#[derive(Debug, Clone)]
49pub struct BuiltPage {
50 pub source: Utf8PathBuf,
51 pub output: Utf8PathBuf,
52 pub title: String,
53 pub html: String,
54}
55
56/// Unresolved internal links found during a build: `(page, target)` (spec §4.3.4).
57pub type BrokenLinks = Vec<(Utf8PathBuf, TargetId)>;
58
59/// Options controlling a site build.
60#[derive(Debug, Clone, Default)]
61pub struct BuildOptions {
62 /// Bypass the incremental cache and re-render every page (spec §4.5).
63 pub no_cache: bool,
64 /// Treat broken internal links as a build error rather than a warning (spec §4.3.4).
65 pub strict: bool,
66 /// Explicit config file, overriding `orgo.toml` in the source directory.
67 pub config_path: Option<Utf8PathBuf>,
68 /// Include pages marked `#+DRAFT:`, overriding `build.drafts` when set.
69 pub drafts: bool,
70}
71
72/// Summary of a site build.
73#[derive(Debug, Default)]
74pub struct SiteReport {
75 /// Every output page (rendered this build or reused from cache).
76 pub pages: Vec<Utf8PathBuf>,
77 /// Pages actually re-rendered and written this build (the invalidation set).
78 pub rendered: Vec<Utf8PathBuf>,
79 /// Pages whose existing on-disk output was reused unchanged (spec §4.1 skip rule).
80 pub skipped: Vec<Utf8PathBuf>,
81 pub assets: Vec<Utf8PathBuf>,
82 /// Unresolved internal links: `(page, target)`. Warnings, not failures (spec §4.3.4).
83 pub broken: Vec<(Utf8PathBuf, TargetId)>,
84 /// Parse diagnostics: `(source file, diagnostic)`, in file then line order.
85 pub diagnostics: Vec<(Utf8PathBuf, Diagnostic)>,
86}
87
88impl SiteReport {
89 /// Every diagnostic and broken link, formatted one per line as
90 /// `file:line: message` — the form an editor can jump to.
91 pub fn warnings(&self) -> Vec<String> {
92 let mut out: Vec<String> = self
93 .diagnostics
94 .iter()
95 .map(|(path, d)| format!("{path}:{}: {}", d.line, d.message))
96 .collect();
97 out.extend(
98 self.broken
99 .iter()
100 .map(|(page, target)| format!("{page}: unresolved link {target}")),
101 );
102 out
103 }
104}
105
106/// Everything a build needs about one page *before* the decision to render it: its
107/// hashes, its resolved element tree, and the dependency edges it participates in.
108struct PagePrep {
109 source: Utf8PathBuf,
110 output: Utf8PathBuf,
111 title: String,
112 content_hash: ContentHash,
113 resolved: crate::resolve::ResolvedDoc,
114 used: HashSet<TargetId>,
115 defines: HashSet<TargetId>,
116 broken: Vec<TargetId>,
117 diagnostics: Vec<Diagnostic>,
118 nav: Vec<NavItem>,
119 context: PageContext,
120 /// The layout this page renders through: `#+TEMPLATE:`, a `[[pages]]` rule, or
121 /// `base.html` (see [`config::page_template`]).
122 template: String,
123}
124
125/// A generated page, resolved against the pages it lists.
126struct Listing {
127 output: Utf8PathBuf,
128 template: String,
129 title: String,
130 /// The pages it lists, already sorted. Empty for a group index, which lists groups.
131 entries: Vec<PageContext>,
132 /// The group this page is for, when it belongs to a grouped collection.
133 group: Option<GroupContext>,
134 /// Every group of the owning collection. The content of a group index, and context
135 /// for a group page.
136 groups: Vec<GroupContext>,
137 /// Set when this is one page of a paginated listing.
138 paginator: Option<Paginator>,
139 /// Render each entry's body into `entry.content` (see
140 /// [`Collection::include_content`](crate::config::Collection::include_content)).
141 include_content: bool,
142 /// Content hash of each entry's source, in `entries` order. Not shown to templates —
143 /// it is how a content-carrying listing notices that a body it embeds has changed,
144 /// without rendering every body to find out.
145 entry_hashes: Vec<ContentHash>,
146}
147
148/// Split one listing's entries across numbered pages, appending each as its own
149/// [`Listing`].
150///
151/// Page 1 keeps `output`, so a section's canonical URL never moves as its page count
152/// changes — only pages 2..N are named by `paginate_output`. An empty listing still
153/// emits page 1, because a section that exists but has nothing in it should be a page
154/// saying so rather than a 404.
155#[allow(clippy::too_many_arguments)]
156fn push_paginated(
157 listings: &mut Vec<Listing>,
158 collection: &config::Collection,
159 output: Utf8PathBuf,
160 title: String,
161 entries: Vec<PageContext>,
162 entry_hashes: Vec<ContentHash>,
163 group: Option<GroupContext>,
164 groups: Vec<GroupContext>,
165) {
166 let per_page = collection.paginate;
167 if per_page == 0 {
168 listings.push(Listing {
169 output,
170 template: collection.template.clone(),
171 title,
172 entries,
173 group,
174 groups,
175 paginator: None,
176 include_content: collection.include_content,
177 entry_hashes: entry_hashes.clone(),
178 });
179 return;
180 }
181
182 let total_entries = entries.len();
183 let total = entries.len().div_ceil(per_page).max(1);
184 let slug = group.as_ref().map(|g| g.slug.clone()).unwrap_or_default();
185 let page_output = |n: usize| -> Utf8PathBuf {
186 if n == 1 {
187 return output.clone();
188 }
189 Utf8PathBuf::from(
190 collection
191 .paginate_output
192 .as_str()
193 .replace(config::GROUP_PLACEHOLDER, &slug)
194 .replace(config::PAGE_PLACEHOLDER, &n.to_string()),
195 )
196 };
197 let outputs: Vec<Utf8PathBuf> = (1..=total).map(page_output).collect();
198
199 for (idx, chunk) in entries.chunks(per_page).chain(
200 // `chunks` yields nothing for an empty slice; page 1 still has to exist.
201 std::iter::once(&[][..]).take(usize::from(total_entries == 0)),
202 ) .enumerate()
203 {
204 let current = idx + 1;
205 let here = &outputs[idx];
206 let url_to = |n: usize| output_url(here, &outputs[n - 1], None);
207 listings.push(Listing {
208 output: here.clone(),
209 template: collection.template.clone(),
210 include_content: collection.include_content,
211 entry_hashes: entry_hashes.clone(),
212 title: title.clone(),
213 entries: chunk.to_vec(),
214 group: group.clone(),
215 groups: groups.clone(),
216 paginator: Some(Paginator {
217 current,
218 total,
219 per_page,
220 total_entries,
221 prev_url: (current > 1).then(|| url_to(current - 1)),
222 next_url: (current < total).then(|| url_to(current + 1)),
223 first_url: url_to(1),
224 last_url: url_to(total),
225 pages: (1..=total)
226 .map(|n| PaginatorPage {
227 number: n,
228 url: url_to(n),
229 current: n == current,
230 })
231 .collect(),
232 }),
233 });
234 }
235}
236
237/// Build the listing pages a config asks for, each with its entries sorted.
238fn build_listings(config: &Config, preps: &[PagePrep]) -> Result<Vec<Listing>> {
239 let mut listings = Vec::new();
240 let hashes: HashMap<&str, ContentHash> = preps
241 .iter()
242 .map(|p| (p.source.as_str(), p.content_hash))
243 .collect();
244 let hashes_of = |entries: &[PageContext]| -> Vec<ContentHash> {
245 entries
246 .iter()
247 .filter_map(|e| hashes.get(e.source.as_str()).copied())
248 .collect()
249 };
250 for collection in &config.collections {
251 let mut entries: Vec<PageContext> = preps
252 .iter()
253 .filter(|p| {
254 collection.source.as_str().is_empty() || p.source.starts_with(&collection.source)
255 })
256 .map(|p| p.context.clone())
257 .collect();
258
259 // Sort ascending first, then reverse for `desc`, so the two orders are exact
260 // mirrors of one another rather than two separately-written comparisons.
261 match collection.sort {
262 SortKey::Title => entries.sort_by(|a, b| a.title.cmp(&b.title)),
263 SortKey::Path => entries.sort_by(|a, b| a.url.cmp(&b.url)),
264 // Undated pages sort last in the final order regardless of direction: a
265 // draft with no date should not lead an archive.
266 SortKey::Date => entries.sort_by(|a, b| {
267 // Date *and* time: org records when a note was written, and two notes
268 // from the same day have an order that the day alone cannot express.
269 let key = |p: &PageContext| {
270 p.date_iso.as_ref().map(|d| {
271 let time = p
272 .date
273 .as_deref()
274 .and_then(iso_time)
275 .unwrap_or_else(|| "00:00:00".to_string());
276 format!("{d}T{time}")
277 })
278 };
279 match (key(a), key(b)) {
280 (Some(x), Some(y)) => x.cmp(&y).then_with(|| a.url.cmp(&b.url)),
281 (Some(_), None) => std::cmp::Ordering::Greater,
282 (None, Some(_)) => std::cmp::Ordering::Less,
283 (None, None) => a.url.cmp(&b.url),
284 }
285 }),
286 }
287 if collection.order == SortOrder::Desc {
288 entries.reverse();
289 }
290
291 if collection.group_by.is_empty() {
292 let entry_hashes = hashes_of(&entries);
293 push_paginated(
294 &mut listings,
295 collection,
296 collection.output.clone(),
297 collection.title.clone(),
298 entries,
299 entry_hashes,
300 None,
301 Vec::new(),
302 );
303 continue;
304 }
305
306 // Grouped: one page per distinct term. `entries` is already sorted, and grouping
307 // preserves that order within each group.
308 let mut terms: Vec<String> = Vec::new();
309 let mut members: HashMap<String, Vec<PageContext>> = HashMap::new();
310 for entry in &entries {
311 for term in group_terms(entry, &collection.group_by) {
312 if !members.contains_key(&term) {
313 terms.push(term.clone());
314 }
315 members.entry(term).or_default().push(entry.clone());
316 }
317 }
318 // Terms are discovered in page order, which is arbitrary from a reader's point of
319 // view; sort so a tag index reads alphabetically and hashes deterministically.
320 terms.sort();
321
322 let mut groups: Vec<GroupContext> = Vec::new();
323 let mut slugs: HashMap<String, String> = HashMap::new();
324 for term in &terms {
325 let slug = slugify(term);
326 if slug.is_empty() {
327 anyhow::bail!(
328 "the {} value {term:?} has no URL-safe form; it cannot name a page",
329 collection.group_by
330 );
331 }
332 // `C++` and `C ++` both slugify to `c`, and one would silently overwrite the
333 // other's page.
334 if let Some(other) = slugs.insert(slug.clone(), term.clone()) {
335 anyhow::bail!(
336 "the {} values {other:?} and {term:?} both become {slug:?} in a URL; \
337 rename one so their pages do not collide",
338 collection.group_by
339 );
340 }
341 groups.push(GroupContext {
342 name: term.clone(),
343 slug: slug.clone(),
344 url: if collection.output.as_str().is_empty() {
345 String::new()
346 } else {
347 collection
348 .output
349 .as_str()
350 .replace(config::GROUP_PLACEHOLDER, &slug)
351 }
352 .to_string(),
353 count: members.get(term).map(Vec::len).unwrap_or(0),
354 });
355 }
356
357 if !collection.output.as_str().is_empty() {
358 for group in &groups {
359 let members_of = members.get(&group.name).cloned().unwrap_or_default();
360 let entry_hashes = hashes_of(&members_of);
361 push_paginated(
362 &mut listings,
363 collection,
364 Utf8PathBuf::from(&group.url),
365 collection
366 .title
367 .replace(config::GROUP_PLACEHOLDER, &group.name),
368 members_of,
369 entry_hashes,
370 Some(group.clone()),
371 // Deliberately not the whole group list. A page that can see every
372 // group depends on every group, so one new post would re-render every
373 // tag page — cost that scales with tag count, to support a tag cloud
374 // nobody has asked for. A tag page depends on its own posts, and the
375 // group index is where the group list belongs.
376 Vec::new(),
377 );
378 }
379 }
380 if !collection.index_output.as_str().is_empty() {
381 listings.push(Listing {
382 output: collection.index_output.clone(),
383 template: collection.index_template.clone(),
384 title: collection.index_title.clone(),
385 entries: Vec::new(),
386 group: None,
387 groups: groups.clone(),
388 paginator: None,
389 // A group index lists groups, not pages; there are no bodies to carry.
390 include_content: false,
391 entry_hashes: Vec::new(),
392 });
393 }
394 }
395
396 // A generated page writing over a real page would silently replace it. Group pages
397 // make this easy to hit by accident, since their paths come from content.
398 for listing in &listings {
399 if let Some(clash) = preps.iter().find(|p| p.output == listing.output) {
400 anyhow::bail!(
401 "collection output {} collides with the page built from {}",
402 listing.output,
403 clash.source
404 );
405 }
406 }
407 let mut claimed: HashMap<&Utf8PathBuf, ()> = HashMap::new();
408 for listing in &listings {
409 if claimed.insert(&listing.output, ()).is_some() {
410 anyhow::bail!("two generated pages both write to {}", listing.output);
411 }
412 }
413 Ok(listings)
414}
415
416/// The `(output, title)` a collection contributes to the nav. A grouped collection
417/// offers its index; an ungrouped one offers its single page.
418fn nav_target(collection: &config::Collection) -> (Utf8PathBuf, String) {
419 if !collection.group_by.is_empty() {
420 return (
421 collection.index_output.clone(),
422 collection.index_title.clone(),
423 );
424 }
425 (collection.output.clone(), collection.title.clone())
426}
427
428/// The group terms a page belongs to. `tags` is multi-valued — a page appears under
429/// every tag it carries — while any other key names a single-valued `#+KEYWORD:`.
430fn group_terms(page: &PageContext, group_by: &str) -> Vec<String> {
431 if group_by.eq_ignore_ascii_case("tags") {
432 return page.tags.clone();
433 }
434 page.keywords
435 .get(&group_by.to_lowercase())
436 .map(|v| v.trim())
437 .filter(|v| !v.is_empty())
438 .map(|v| vec![v.to_string()])
439 .unwrap_or_default()
440}
441
442/// Everything a listing template can see about its entries, hashed. This is the listing
443/// page's whole dependency: if none of these change, its output cannot have changed.
444///
445/// Entries are hashed through their *serialization* rather than a hand-picked set of
446/// fields. Picking fields means the hash drifts from what a template can read the moment
447/// one is added — which it had: the excerpt was missing, so rewriting a post's first
448/// paragraph left the old excerpt on the index until something else invalidated it.
449fn listing_entries_hash(listing: &Listing) -> Hash {
450 let mut fields: Vec<(String, String)> = listing
451 .entries
452 .iter()
453 .map(|e| {
454 (
455 e.url.clone(),
456 serde_json::to_string(e).unwrap_or_else(|_| e.title.clone()),
457 )
458 })
459 // A group index has no entries at all — its content *is* the group list, so the
460 // groups have to be in the hash or a tag index would never notice a new tag.
461 .chain(
462 listing
463 .groups
464 .iter()
465 .map(|g| (g.url.clone(), format!("{}\u{0}{}", g.name, g.count))),
466 )
467 .chain(listing.paginator.iter().map(|p| {
468 (
469 format!("{}/{}", p.current, p.total),
470 format!("{:?}|{:?}", p.prev_url, p.next_url),
471 )
472 }))
473 .chain([(listing.title.clone(), listing.template.clone())])
474 .collect();
475
476 // A listing that embeds its entries' bodies depends on those bodies. The source hash
477 // stands in for the rendered HTML, so noticing a change does not cost a render of
478 // every page listed.
479 if listing.include_content {
480 fields.extend(
481 listing
482 .entry_hashes
483 .iter()
484 .map(|h| ("content".to_string(), format!("{h:?}"))),
485 );
486 }
487
488 // Entry *order* is meaningful in a listing, so this hashes the sorted-by-us sequence
489 // rather than a set: a re-ordering is a real change to the page.
490 site_structure_hash_ordered(&fields)
491}
492
493/// A listing's entries with their rendered bodies attached, for a template that asked
494/// for them — a full-content feed being the case that needs it.
495///
496/// An entry whose source is not among the prepared pages keeps `content: none` rather
497/// than failing: the listing is still a valid page, and a feed item without a body is a
498/// better outcome than no feed.
499fn entries_with_content(
500 entries: &[PageContext],
501 preps: &[PagePrep],
502 highlighter: &SyntectHighlighter,
503 config: &Config,
504) -> Vec<PageContext> {
505 let by_source: HashMap<&str, &PagePrep> =
506 preps.iter().map(|p| (p.source.as_str(), p)).collect();
507 let opts = render_options(config);
508 entries
509 .par_iter()
510 .map(|entry| {
511 let mut entry = entry.clone();
512 if let Some(prep) = by_source.get(entry.source.as_str()) {
513 let Html(html) = render_with(&prep.resolved, highlighter, &opts);
514 entry.content = Some(html);
515 }
516 entry
517 })
518 .collect()
519}
520
521/// The nav a listing page shows: whatever the site's nav is, relativized to this
522/// listing's own location.
523fn listing_nav(preps: &[PagePrep], output: &Utf8Path) -> Vec<NavItem> {
524 let Some(first) = preps.first() else {
525 return Vec::new();
526 };
527 first
528 .nav
529 .iter()
530 .map(|item| {
531 // Nav URLs on `preps[0]` are relative to that page; re-resolve them against
532 // the site root, then against this listing's depth.
533 let absolute = resolve_relative(&first.output, &item.url);
534 NavItem {
535 title: item.title.clone(),
536 url: output_url(output, &absolute, None),
537 }
538 })
539 .collect()
540}
541
542/// Turn a URL relative to `from` back into a site-root-relative path.
543fn resolve_relative(from: &Utf8Path, url: &str) -> Utf8PathBuf {
544 if url == "#" {
545 return from.to_owned();
546 }
547 let base = from.parent().unwrap_or_else(|| Utf8Path::new(""));
548 let mut stack: Vec<&str> = base.components().map(|c| c.as_str()).collect();
549 for part in url.split('/') {
550 match part {
551 "." | "" => {}
552 ".." => {
553 stack.pop();
554 }
555 other => stack.push(other),
556 }
557 }
558 Utf8PathBuf::from(stack.join("/"))
559}
560
561/// The `PageContext` a listing page presents for *itself*.
562fn listing_context(listing: &Listing) -> PageContext {
563 PageContext {
564 title: listing.title.clone(),
565 url: listing.output.to_string(),
566 source: String::new(),
567 date: None,
568 date_iso: None,
569 year: None,
570 tags: Vec::new(),
571 excerpt: String::new(),
572 content: None,
573 word_count: 0,
574 reading_time: 0,
575 keywords: Default::default(),
576 toc: Vec::new(),
577 }
578}
579
580/// Which pages the configured [`NavMode`] selects, in nav order.
581fn nav_selection<'a>(config: &Config, candidates: &'a [NavCandidate]) -> Vec<&'a NavCandidate> {
582 match config.nav.mode {
583 NavMode::None => Vec::new(),
584 NavMode::All => candidates.iter().collect(),
585 // Generated pages are a section's landing page, which is what a nav entry should
586 // point at whatever depth the section lives at.
587 NavMode::TopLevel => candidates
588 .iter()
589 .filter(|c| c.generated || is_top_level(&c.output))
590 .collect(),
591 // Configured order wins over discovery order — a hand-written nav is a designed
592 // sequence, not an alphabetical one.
593 NavMode::Explicit => {
594 let mut chosen: Vec<&NavCandidate> = config
595 .nav
596 .pages
597 .iter()
598 .filter_map(|want| candidates.iter().find(|c| c.matches(want)))
599 .collect();
600 // A collection that asked for the nav but was not listed is appended rather
601 // than dropped, so `nav = true` never silently does nothing. Listing it puts
602 // it exactly where you said instead.
603 for candidate in candidates.iter().filter(|c| c.generated) {
604 if !chosen.iter().any(|c| c.output == candidate.output) {
605 chosen.push(candidate);
606 }
607 }
608 chosen
609 }
610 }
611}
612
613/// A page the navigation could contain: one written as `.org`, or one generated by a
614/// collection.
615struct NavCandidate {
616 /// The source path of an authored page. Empty for a generated one, which has none.
617 source: Utf8PathBuf,
618 output: Utf8PathBuf,
619 title: String,
620 /// Generated pages are appended when an explicit nav does not name them.
621 generated: bool,
622}
623
624impl NavCandidate {
625 /// Does `name` in `nav.pages` refer to this entry?
626 ///
627 /// Authored pages are named by their source — `about.org` — because that is the file
628 /// you wrote and its output path may be moved by `#+SLUG:`. Generated pages have no
629 /// source, so they are named by their output — `blog/index.html`. Either spelling is
630 /// accepted for either, so a config that names an output path still works.
631 fn matches(&self, name: &Utf8Path) -> bool {
632 (!self.source.as_str().is_empty() && self.source == name) || self.output == name
633 }
634}
635
636/// Every page the navigation could contain, authored pages first.
637fn nav_candidates(
638 config: &Config,
639 pages: &[(Utf8PathBuf, Utf8PathBuf, String)],
640) -> Vec<NavCandidate> {
641 let mut candidates: Vec<NavCandidate> = pages
642 .iter()
643 .map(|(source, output, title)| NavCandidate {
644 source: source.clone(),
645 output: output.clone(),
646 title: title.clone(),
647 generated: false,
648 })
649 .collect();
650 for collection in config.collections.iter().filter(|c| c.nav) {
651 let (output, title) = nav_target(collection);
652 candidates.push(NavCandidate {
653 source: Utf8PathBuf::new(),
654 output,
655 title,
656 generated: true,
657 });
658 }
659 candidates
660}
661
662/// DISCOVER + PARSE + INDEX + RESOLVE the whole site, returning per-page prep and the
663/// global symbol table. RENDER/TEMPLATE is deferred to the caller so the incremental
664/// build can render only the pages it must. PARSE/INDEX/RESOLVE are cheap and pure, so
665/// they run for every file each build; the incremental win is on RENDER + EMIT (spec §4.4).
666fn prepare_pages(
667 src: &Utf8Path,
668 config: &Config,
669 out: Option<&Utf8Path>,
670) -> Result<(Vec<PagePrep>, SymbolTable)> {
671 let (org_rel, _assets) = discover(src, config, out)?;
672
673 // PARSE every file (relative paths keep snapshots and links machine-independent).
674 // PARSE is a pure function of one file's bytes (spec §2.1), which is exactly the
675 // property that makes it safe to run in parallel. `par_iter().collect()` preserves
676 // input order, so the document list — and everything downstream of it — is identical
677 // to the sequential build regardless of how the work was scheduled.
678 let docs: Vec<Document> = org_rel
679 .par_iter()
680 .map(|rel| {
681 let abs = src.join(rel);
682 let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?;
683 parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))
684 })
685 .collect::<Result<Vec<_>>>()?;
686
687 // Drop drafts before anything else sees them. Removing them here rather than at emit
688 // time means they are absent from listings, the nav and the symbol table too — so a
689 // link *to* a draft is reported as broken, which is exactly what it would be on the
690 // published site.
691 let docs: Vec<Document> = docs
692 .into_iter()
693 .filter(|d| config.build.drafts || !is_draft(&d.keywords))
694 .collect();
695
696 // INDEX: collect every link target across the corpus.
697 let mut symbols = SymbolTable::new();
698 for doc in &docs {
699 symbols.index_document(doc);
700 }
701
702 // `(source, output, title)` for every page. Titles come from #+TITLE (falling back to
703 // the file stem) and URLs from each page's output path, which `#+SLUG:` can rename.
704 let all_pages: Vec<(Utf8PathBuf, Utf8PathBuf, String)> = docs
705 .iter()
706 .map(|d| {
707 (
708 d.source_path.clone(),
709 output_path(&d.source_path, &d.keywords),
710 page_title(d),
711 )
712 })
713 .collect();
714
715 // Two sources emitting one page would silently drop a page — and with slugs, a
716 // collision is a typo away and invisible in the source filenames.
717 let mut claimed: std::collections::HashMap<&Utf8PathBuf, &Utf8PathBuf> =
718 std::collections::HashMap::new();
719 for (source, out, _) in &all_pages {
720 if let Some(other) = claimed.insert(out, source) {
721 anyhow::bail!(
722 "output collision: {other} and {source} both build to {out} \
723 (check their #+SLUG:)"
724 );
725 }
726 }
727
728 // Authored pages and the landing pages collections generate, in one list so an
729 // explicit nav can order them together.
730 let candidates = nav_candidates(config, &all_pages);
731
732 // An explicit nav naming something that does not exist is a typo, and a silently
733 // shorter nav is a poor way to learn about it.
734 if config.nav.mode == NavMode::Explicit {
735 for want in &config.nav.pages {
736 if !candidates.iter().any(|c| c.matches(want)) {
737 anyhow::bail!(
738 "nav.pages lists {want}, which is neither a page in {src} nor a \
739 collection with `nav = true`"
740 );
741 }
742 }
743 }
744 let entries: Vec<(Utf8PathBuf, String)> = nav_selection(config, &candidates)
745 .into_iter()
746 .map(|c| (c.output.clone(), c.title.clone()))
747 .collect();
748
749 // RESOLVE reads the shared symbol table and writes only into its own page's output,
750 // so it parallelizes for free once INDEX has finished building the table.
751 let pages: Vec<PagePrep> = docs
752 .par_iter()
753 .map(|doc| {
754 let out = resolve(doc, &symbols);
755 let used: HashSet<TargetId> = out.used_targets.iter().cloned().collect();
756 let broken: Vec<TargetId> = out.broken.iter().map(|b| b.target.clone()).collect();
757 let defines: HashSet<TargetId> = document_targets(doc).into_iter().collect();
758
759 let output = output_path(&doc.source_path, &doc.keywords);
760
761 // Nav links are relative to *this* page (spec URL scheme, §8 Q3).
762 let nav: Vec<NavItem> = entries
763 .iter()
764 .map(|(path, title)| NavItem {
765 title: title.clone(),
766 url: output_url(&output, path, None),
767 })
768 .collect();
769
770 PagePrep {
771 context: page_context(doc, &output, config),
772 template: config::page_template(config, &doc.source_path, &doc.keywords),
773 source: doc.source_path.clone(),
774 output,
775 title: page_title(doc),
776 content_hash: doc.content_hash,
777 resolved: out.resolved,
778 used,
779 defines,
780 broken,
781 diagnostics: doc.diagnostics.clone(),
782 nav,
783 }
784 })
785 .collect();
786
787 Ok((pages, symbols))
788}
789
790/// Fail before rendering if any page names a template that does not exist.
791///
792/// minijinja would report the missing name on its own, but only once a page reaches it
793/// — and a typo in `#+TEMPLATE:` or a `[[pages]]` rule is worth naming together with the
794/// page that carries it and the templates that do exist.
795fn check_page_templates(templater: &Templater, preps: &[PagePrep]) -> Result<()> {
796 for p in preps {
797 if !templater.has(&p.template) {
798 let mut available = templater.names();
799 available.sort_unstable();
800 anyhow::bail!(
801 "{} renders through {}, which is not in the templates directory. \
802 Available: {}",
803 p.source,
804 p.template,
805 available.join(", ")
806 );
807 }
808 }
809 Ok(())
810}
811
812/// Parse + index + resolve + render + template a whole site *in memory*, without
813/// touching the output directory. Shared by the tests (full render, every page).
814pub fn render_site(src: &Utf8Path) -> Result<(Vec<BuiltPage>, BrokenLinks)> {
815 let config = Config::load(src)?;
816 config.validate()?;
817 let (preps, _symbols) = prepare_pages(src, &config, None)?;
818 let highlighter = SyntectHighlighter::new();
819 let templater = Templater::load(Some(&src.join(&config.templates.dir)), &config.site.base_url)?;
820 check_page_templates(&templater, &preps)?;
821 let site = site_context(&config);
822 let listing = page_listing(&config, &preps);
823
824 let mut pages = Vec::new();
825 let mut broken = Vec::new();
826 for p in &preps {
827 for t in &p.broken {
828 broken.push((p.source.clone(), t.clone()));
829 }
830 let html = render_page(&templater, &highlighter, &site, listing.as_deref(), &config, p)?;
831 pages.push(BuiltPage {
832 source: p.source.clone(),
833 output: p.output.clone(),
834 title: p.title.clone(),
835 html,
836 });
837 }
838 Ok((pages, broken))
839}
840
841/// The site's render options. A document's own `#+OPTIONS:` switches are applied by the
842/// renderer, so every caller gets them.
843fn render_options(config: &Config) -> RenderOptions {
844 RenderOptions {
845 heading_offset: config.html.heading_offset,
846 section_numbers: config.html.section_numbers,
847 special_strings: config.html.special_strings,
848 sub_superscript: config.html.sub_superscript,
849 entities: config.html.entities,
850 }
851}
852
853fn site_context(config: &Config) -> SiteContext {
854 SiteContext {
855 title: config.site.title.clone(),
856 base_url: config.site.base_url.clone(),
857 description: config.site.description.clone(),
858 language: config.site.language.clone(),
859 }
860}
861
862/// The `pages` list templates see, when configured to see one (see
863/// [`crate::config::Templates::expose_page_list`]).
864fn page_listing(config: &Config, preps: &[PagePrep]) -> Option<Vec<PageContext>> {
865 config
866 .templates
867 .expose_page_list
868 .then(|| preps.iter().map(|p| p.context.clone()).collect())
869}
870
871/// RENDER + TEMPLATE one prepared page into its final HTML string.
872#[allow(clippy::too_many_arguments)]
873fn render_page(
874 templater: &Templater,
875 highlighter: &SyntectHighlighter,
876 site: &SiteContext,
877 pages: Option<&[PageContext]>,
878 config: &Config,
879 p: &PagePrep,
880) -> Result<String> {
881 let opts = render_options(config);
882 let Html(fragment) = render_with(&p.resolved, highlighter, &opts);
883 // Relative to the *output* path, since `#+SLUG:` can move a page between depths.
884 let root = relative_root(&p.output);
885 let stylesheet = format!("{root}{SYNTAX_STYLESHEET}");
886 let theme = theme_href(config, &root);
887 let mut ctx = RenderContext::new(site, &p.context, &p.nav, &stylesheet, &root);
888 ctx.body = &fragment;
889 ctx.theme = &theme;
890 ctx.pages = pages;
891 templater
892 .render(&p.template, &ctx)
893 .with_context(|| format!("templating {} through {}", p.source, p.template))
894}
895
896/// Site-root-relative name of the generated syntax stylesheet. Every page links to it.
897pub const SYNTAX_STYLESHEET: &str = "syntax.css";
898
899/// Site-root-relative name of the built-in theme's stylesheet, written only when
900/// `site.theme` names one.
901pub const THEME_STYLESHEET: &str = "theme.css";
902
903/// Where a page at `root` finds `theme.css` — empty when the site has no theme, which
904/// is how a template knows to link nothing at all.
905fn theme_href(config: &Config, root: &str) -> String {
906 if config.site.theme.is_empty() {
907 String::new()
908 } else {
909 format!("{root}{THEME_STYLESHEET}")
910 }
911}
912
913/// Site-root-relative name of the generated sitemap.
914pub const SITEMAP: &str = "sitemap.xml";
915
916/// `sitemap.xml` for every HTML page in `pages`, in URL order.
917///
918/// Only HTML: a sitemap is a list of pages for a crawler to read, and a feed or a
919/// stylesheet is neither. `lastmod` is the page's own `#+DATE:` where it has one — the
920/// nearest honest thing available without trusting a filesystem timestamp that a fresh
921/// clone would reset.
922fn sitemap(base_url: &str, pages: &[Utf8PathBuf], dated: &HashMap<&Utf8Path, &str>) -> String {
923 let mut urls: Vec<&Utf8PathBuf> = pages
924 .iter()
925 .filter(|p| p.extension() == Some("html"))
926 .collect();
927 urls.sort();
928 urls.dedup();
929
930 let base = base_url.trim_end_matches('/');
931 let mut out = String::from(
932 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
933 <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n",
934 );
935 for url in urls {
936 out.push_str("<url>\n");
937 out.push_str(&format!("<loc>{base}/{}</loc>\n", escape_xml(url.as_str())));
938 if let Some(date) = dated.get(url.as_path()) {
939 out.push_str(&format!("<lastmod>{date}</lastmod>\n"));
940 }
941 out.push_str("</url>\n");
942 }
943 out.push_str("</urlset>\n");
944 out
945}
946
947/// The five XML predefined entities. A `&` in a URL is the common one, from a query
948/// string that survived into a filename.
949fn escape_xml(s: &str) -> String {
950 s.replace('&', "&")
951 .replace('<', "<")
952 .replace('>', ">")
953 .replace('"', """)
954 .replace('\'', "'")
955}
956
957/// Full site build with the incremental layer (spec §4). Renders only the pages whose
958/// `render_key` changed or that link into a changed file's targets; reuses the on-disk
959/// output of everything else; persists an updated cache manifest.
960pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result<SiteReport> {
961 let mut cfg = match &opts.config_path {
962 Some(path) => Config::load_file(path)?,
963 None => Config::load(src)?,
964 };
965 cfg.validate()?;
966 // The flag turns drafts on; it never turns off a config that asked for them.
967 cfg.build.drafts |= opts.drafts;
968
969 // Create the output directory up front so it can be recognised and excluded when it
970 // lives inside the source tree.
971 fs::create_dir_all(out).with_context(|| format!("creating {out}"))?;
972 let (_org_rel, source_assets) = discover(src, &cfg, Some(out))?;
973 let assets = collect_assets(src, &cfg, Some(out), &source_assets)?;
974 let (preps, symbols) = prepare_pages(src, &cfg, Some(out))?;
975
976 let templater = Templater::load(Some(&src.join(&cfg.templates.dir)), &cfg.site.base_url)?;
977 check_page_templates(&templater, &preps)?;
978 let syntax_css = render::syntax_stylesheet(&cfg.highlight)?;
979
980 // The global hash classes (spec §4.1): a change in any invalidates the site. The
981 // config hash is combined with a site-structure hash covering the global chrome each
982 // page carries, so a change to that chrome re-renders the pages showing it.
983 //
984 // Which pages belong in that hash depends on what a template can *see*. Normally it
985 // is the nav only — a nested page cannot change another page's nav, so adding a blog
986 // post should render one page, not the site. But `expose_page_list` hands every
987 // template every page's metadata, and then any page's output really can depend on
988 // any other page, so the hash has to widen to match. Keyed on output paths, since a
989 // `#+SLUG:` change moves a page's URL without moving its source.
990 let all_pages: Vec<(Utf8PathBuf, Utf8PathBuf, String)> = preps
991 .iter()
992 .map(|p| (p.source.clone(), p.output.clone(), p.title.clone()))
993 .collect();
994 let structure_hash = if cfg.templates.expose_page_list {
995 let entries: Vec<(String, String)> = all_pages
996 .iter()
997 .map(|(_, out, title)| (out.to_string(), title.clone()))
998 .collect();
999 site_structure_hash(&entries)
1000 } else {
1001 // The same selection the nav itself is built from, so the two can never drift.
1002 // Hashed in order, because the nav's order is itself part of every page.
1003 let entries: Vec<(String, String)> = nav_selection(&cfg, &nav_candidates(&cfg, &all_pages))
1004 .into_iter()
1005 .map(|c| (c.output.to_string(), c.title.clone()))
1006 .collect();
1007 site_structure_hash_ordered(&entries)
1008 };
1009 let cfg_hash = combine(config_hash(&cfg), structure_hash);
1010 // Per template rather than per site: a page's render key covers the layout it uses
1011 // and that layout's own includes, so editing `feed.xml` re-renders the feed.
1012 let tmpl_hash_for = |name: &str| template_hash(&templater.sources_for(name));
1013
1014 // Compose each page's render key and record its dependency edges.
1015 let mut new_graph = DepGraph::default();
1016 let mut new_records: Vec<(Utf8PathBuf, PageRecord, Hash)> = Vec::new();
1017 let listings = build_listings(&cfg, &preps)?;
1018 for p in &preps {
1019 let rlh = resolved_links_hash(&p.source, &p.used, &symbols);
1020 let key = render_key(p.content_hash, rlh, cfg_hash, tmpl_hash_for(&p.template));
1021 new_graph.defines.insert(p.source.clone(), p.defines.clone());
1022 new_graph.uses.insert(p.source.clone(), p.used.clone());
1023 new_records.push((
1024 p.source.clone(),
1025 PageRecord {
1026 content_hash: p.content_hash,
1027 render_key: key,
1028 output_path: p.output.clone(),
1029 },
1030 key,
1031 ));
1032 }
1033
1034 // Load the prior manifest (unless bypassed). Absent/corrupt/version-mismatch ⇒ None
1035 // ⇒ full rebuild (spec §4.5).
1036 let prior = if opts.no_cache {
1037 None
1038 } else {
1039 incremental::load_manifest(out)
1040 };
1041
1042 let rebuild: HashSet<Utf8PathBuf> = compute_rebuild_set(
1043 &preps,
1044 &new_records,
1045 &new_graph,
1046 cfg_hash,
1047 out,
1048 prior.as_ref(),
1049 );
1050
1051 // Delete outputs for pages that existed last build but are gone now (spec §4.3 step 1:
1052 // removed files). Their targets are already in the merged graph, so their linkers were
1053 // invalidated above.
1054 if let Some(prior) = &prior {
1055 // Keyed by source path for real pages and by output path for generated listings,
1056 // which is also how each records itself in the manifest. Listings have to be in
1057 // this set or the cleanup would delete the file it just decided to keep — and a
1058 // removed collection genuinely should have its output deleted.
1059 let mut current: HashSet<&Utf8PathBuf> = preps.iter().map(|p| &p.source).collect();
1060 current.extend(listings.iter().map(|l| &l.output));
1061 for (key, rec) in &prior.pages {
1062 if !current.contains(key) {
1063 let dest = out.join(&rec.output_path);
1064 let _ = fs::remove_file(&dest);
1065 }
1066 }
1067 }
1068
1069 let highlighter = SyntectHighlighter::with_syntaxes(Some(&src.join(&cfg.highlight.syntaxes_dir)));
1070 let site = site_context(&cfg);
1071 let listing = page_listing(&cfg, &preps);
1072 let mut report = SiteReport::default();
1073
1074 // RENDER + TEMPLATE + EMIT, in parallel. This is where a build's time actually goes
1075 // (syntect highlighting and templating dominate), and each page writes only its own
1076 // file, so the pages are independent.
1077 //
1078 // The parallel pass returns whether each page was written; the report is assembled
1079 // sequentially afterwards from `preps` order. Pushing to the report from inside the
1080 // parallel pass would make `rendered`/`skipped` ordering depend on thread scheduling,
1081 // which would be a non-deterministic build report over a deterministic build.
1082 let written: Vec<bool> = preps
1083 .par_iter()
1084 .map(|p| {
1085 if !rebuild.contains(&p.source) {
1086 // Skip: the on-disk output is already correct (spec §4.1). Leave it alone.
1087 return Ok(false);
1088 }
1089 let dest = out.join(&p.output);
1090 if let Some(parent) = dest.parent() {
1091 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1092 }
1093 let html = render_page(&templater, &highlighter, &site, listing.as_deref(), &cfg, p)?;
1094 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
1095 Ok(true)
1096 })
1097 .collect::<Result<Vec<_>>>()?;
1098
1099 for (p, was_written) in preps.iter().zip(&written) {
1100 for t in &p.broken {
1101 report.broken.push((p.source.clone(), t.clone()));
1102 }
1103 for d in &p.diagnostics {
1104 report.diagnostics.push((p.source.clone(), d.clone()));
1105 }
1106 report.pages.push(p.output.clone());
1107 if *was_written {
1108 report.rendered.push(p.output.clone());
1109 } else {
1110 report.skipped.push(p.output.clone());
1111 }
1112 }
1113
1114 // Generated listing pages (spec §2.1 EMIT). A listing has no source file, so it is
1115 // cached on the one thing it actually depends on: the entries it lists. Adding a post
1116 // therefore re-renders that section's index and nothing else — the same precision the
1117 // rest of the build gets from content hashing.
1118 for listing in &listings {
1119 let key = combine(
1120 listing_entries_hash(listing),
1121 combine(cfg_hash, tmpl_hash_for(&listing.template)),
1122 );
1123 let dest = out.join(&listing.output);
1124 let cached = prior
1125 .as_ref()
1126 .and_then(|m| m.pages.get(&listing.output))
1127 .map(|rec| rec.render_key == key)
1128 .unwrap_or(false);
1129
1130 report.pages.push(listing.output.clone());
1131 if cached && dest.exists() {
1132 report.skipped.push(listing.output.clone());
1133 } else {
1134 if let Some(parent) = dest.parent() {
1135 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1136 }
1137 let root = relative_root(&listing.output);
1138 let stylesheet = format!("{root}{SYNTAX_STYLESHEET}");
1139 let nav = listing_nav(&preps, &listing.output);
1140 let page_ctx = listing_context(listing);
1141 // Bodies are rendered here rather than when the listing was built, so a
1142 // cached feed costs nothing. This is the only place a page is rendered twice.
1143 let with_content = listing
1144 .include_content
1145 .then(|| entries_with_content(&listing.entries, &preps, &highlighter, &cfg));
1146 let theme = theme_href(&cfg, &root);
1147 let mut ctx = RenderContext::new(&site, &page_ctx, &nav, &stylesheet, &root);
1148 ctx.theme = &theme;
1149 ctx.pages = Some(with_content.as_deref().unwrap_or(&listing.entries));
1150 ctx.group = listing.group.as_ref();
1151 ctx.groups = &listing.groups;
1152 ctx.paginator = listing.paginator.as_ref();
1153 let html = templater
1154 .render(&listing.template, &ctx)
1155 .with_context(|| {
1156 format!(
1157 "rendering collection {} with template {} (available: {})",
1158 listing.output,
1159 listing.template,
1160 templater.names().join(", ")
1161 )
1162 })?;
1163 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
1164 report.rendered.push(listing.output.clone());
1165 }
1166
1167 new_records.push((
1168 listing.output.clone(),
1169 PageRecord {
1170 content_hash: key,
1171 render_key: key,
1172 output_path: listing.output.clone(),
1173 },
1174 key,
1175 ));
1176 }
1177
1178 // The syntax stylesheet the highlighter's CSS classes refer to. Written every build
1179 // (it is a few KB and depends only on the theme, which lives in the config hash).
1180 fs::write(out.join(SYNTAX_STYLESHEET), &syntax_css)
1181 .with_context(|| format!("writing {SYNTAX_STYLESHEET} under {out}"))?;
1182
1183 // The built-in theme, on the same terms: compiled in, written every build, and named
1184 // in the config hash so switching themes re-renders the pages that link it.
1185 if let Some(css) = theme::theme_css(&cfg.site.theme) {
1186 fs::write(out.join(THEME_STYLESHEET), css)
1187 .with_context(|| format!("writing {THEME_STYLESHEET} under {out}"))?;
1188 }
1189
1190 // A sitemap covers every page the build emits, authored and generated alike, so it is
1191 // written here rather than declared as a collection: a collection lists the pages it
1192 // was pointed at, and this one has to know about all of them including itself.
1193 if cfg.build.sitemap && !cfg.site.base_url.is_empty() {
1194 let dated: HashMap<&Utf8Path, &str> = preps
1195 .iter()
1196 .filter_map(|p| Some((p.output.as_path(), p.context.date_iso.as_deref()?)))
1197 .collect();
1198 let xml = sitemap(&cfg.site.base_url, &report.pages, &dated);
1199 fs::write(out.join(SITEMAP), xml)
1200 .with_context(|| format!("writing {SITEMAP} under {out}"))?;
1201 }
1202
1203 // Assets are a dumb copy in v0.3 (spec §8 Q11): copy every run. Cheap, and keeps the
1204 // full-vs-incremental byte equivalence trivially true for non-`.org` files.
1205 for asset in &assets {
1206 let dest = out.join(&asset.rel);
1207 if let Some(parent) = dest.parent() {
1208 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1209 }
1210 fs::copy(&asset.from, &dest)
1211 .with_context(|| format!("copying {} -> {dest}", asset.from))?;
1212 report.assets.push(asset.rel.clone());
1213 }
1214
1215 // Persist the manifest for the next build.
1216 let manifest = Manifest {
1217 format_version: CACHE_FORMAT_VERSION,
1218 config_hash: Some(cfg_hash),
1219 pages: new_records
1220 .into_iter()
1221 .map(|(src_path, rec, _)| (src_path, rec))
1222 .collect(),
1223 graph: new_graph,
1224 };
1225 incremental::save_manifest(out, &manifest)
1226 .with_context(|| format!("writing cache manifest under {out}"))?;
1227
1228 let warnings = report.warnings();
1229 if opts.strict && !warnings.is_empty() {
1230 for w in &warnings {
1231 eprintln!("error: {w}");
1232 }
1233 anyhow::bail!(
1234 "{} problem(s) under --strict ({} parse diagnostic(s), {} unresolved link(s))",
1235 warnings.len(),
1236 report.diagnostics.len(),
1237 report.broken.len()
1238 );
1239 }
1240 for w in &warnings {
1241 eprintln!("warning: {w}");
1242 }
1243
1244 Ok(report)
1245}
1246
1247/// The set of source files to (re)render this build (spec §4.3 invalidation algorithm),
1248/// as the union of:
1249/// - **no prior cache** (absent/corrupt/version-mismatch/`--no-cache`) ⇒ every page;
1250/// - a **global** config- or template-hash change ⇒ every page (spec §4.1);
1251/// - **content-changed** files ∪ pages that link into a changed file's targets, via the
1252/// dependency graph merged with the prior build's `defines` (spec §4.3, so a removed
1253/// target still invalidates its linkers);
1254/// - any page whose composed **render_key** differs from the cached one (catches URL
1255/// changes on linked targets precisely);
1256/// - any page whose **output file is missing** on disk.
1257fn compute_rebuild_set(
1258 preps: &[PagePrep],
1259 new_records: &[(Utf8PathBuf, PageRecord, Hash)],
1260 new_graph: &DepGraph,
1261 cfg_hash: Hash,
1262 out: &Utf8Path,
1263 prior: Option<&Manifest>,
1264) -> HashSet<Utf8PathBuf> {
1265 let all: HashSet<Utf8PathBuf> = preps.iter().map(|p| p.source.clone()).collect();
1266
1267 let Some(prior) = prior else {
1268 return all; // No usable cache ⇒ full rebuild.
1269 };
1270
1271 // A config change invalidates every page. Template changes do not come through here:
1272 // each page's render key carries the hash of the templates *it* uses, so the key
1273 // comparison below invalidates exactly the pages whose layout moved.
1274 if prior.config_hash != Some(cfg_hash) {
1275 return all;
1276 }
1277
1278 // Content-changed = hash differs from the cached record, or the file is new.
1279 let mut changed: HashSet<Utf8PathBuf> = HashSet::new();
1280 for p in preps {
1281 match prior.pages.get(&p.source) {
1282 Some(rec) if rec.content_hash == p.content_hash => {}
1283 _ => {
1284 changed.insert(p.source.clone());
1285 }
1286 }
1287 }
1288
1289 // Graph expansion: changed files ∪ pages that link into a changed file's targets.
1290 // Merge prior `defines` so a target a changed file removed still pulls its linkers.
1291 let merged = prior.graph.merged_defines_with(new_graph);
1292 let mut rebuild = incremental::invalidation_set(&changed, &merged);
1293
1294 // Precise render_key delta (catches a linked target's URL change; also a belt for the
1295 // graph). A page whose render_key matches the cache and whose output exists is correct.
1296 for (src_path, _rec, key) in new_records {
1297 let unchanged = prior
1298 .pages
1299 .get(src_path)
1300 .map(|old| old.render_key == *key)
1301 .unwrap_or(false);
1302 if !unchanged {
1303 rebuild.insert(src_path.clone());
1304 }
1305 }
1306
1307 // Any page whose output file is missing must be re-emitted regardless.
1308 for p in preps {
1309 if !out.join(&p.output).exists() {
1310 rebuild.insert(p.source.clone());
1311 }
1312 }
1313
1314 rebuild
1315}
1316
1317/// Walk `src`, returning `.org` source paths and non-`.org` asset paths, both relative
1318/// to `src` and sorted for deterministic output. The cache manifest is not an asset.
1319fn discover(
1320 src: &Utf8Path,
1321 config: &Config,
1322 out: Option<&Utf8Path>,
1323) -> Result<(Vec<Utf8PathBuf>, Vec<Utf8PathBuf>)> {
1324 let skip_dirs = excluded_dirs(src, config, out);
1325 let mut org = Vec::new();
1326 let mut assets = Vec::new();
1327
1328 let walker = WalkDir::new(src).sort_by_file_name().into_iter();
1329 for entry in walker.filter_entry(|e| {
1330 let Some(path) = Utf8Path::from_path(e.path()) else {
1331 return false;
1332 };
1333 let rel = path.strip_prefix(src).unwrap_or(path);
1334 // The source root itself always passes; `filter_entry` prunes whole subtrees.
1335 rel.as_str().is_empty() || !is_excluded(rel, &skip_dirs)
1336 }) {
1337 let entry = entry.with_context(|| format!("walking {src}"))?;
1338 if !entry.file_type().is_file() {
1339 continue;
1340 }
1341 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
1342 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
1343 let rel = abs
1344 .strip_prefix(src)
1345 .map(|p| p.to_owned())
1346 .unwrap_or_else(|_| abs.clone());
1347 if rel == config::CONFIG_FILE {
1348 continue;
1349 }
1350 if rel.extension() == Some("org") {
1351 org.push(rel);
1352 } else {
1353 assets.push(rel);
1354 }
1355 }
1356 org.sort();
1357 assets.sort();
1358 Ok((org, assets))
1359}
1360
1361/// One file to copy through to the output: where it is, and where it goes.
1362#[derive(Debug, Clone, PartialEq)]
1363pub struct Asset {
1364 /// Path to read from.
1365 pub from: Utf8PathBuf,
1366 /// Path to write, relative to the output root.
1367 pub rel: Utf8PathBuf,
1368}
1369
1370/// Every file to copy: the source directory's non-`.org` files, then each extra asset
1371/// root's contents, flattened onto the site root.
1372///
1373/// Two files claiming one output path is an error rather than a race — whichever won
1374/// would depend on directory order, and a site whose favicon changes when a file is
1375/// renamed elsewhere is worse than a build that stops.
1376fn collect_assets(
1377 src: &Utf8Path,
1378 config: &Config,
1379 out: Option<&Utf8Path>,
1380 from_source: &[Utf8PathBuf],
1381) -> Result<Vec<Asset>> {
1382 let mut assets: Vec<Asset> = from_source
1383 .iter()
1384 .map(|rel| Asset {
1385 from: src.join(rel),
1386 rel: rel.clone(),
1387 })
1388 .collect();
1389
1390 for root in &config.build.assets {
1391 let base = src.join(root);
1392 if !base.is_dir() {
1393 anyhow::bail!(
1394 "build.assets lists {root}, which is not a directory (looked in {base})"
1395 );
1396 }
1397 let base_canon = std::fs::canonicalize(&base)
1398 .ok()
1399 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok())
1400 .unwrap_or_else(|| base.clone());
1401 // An asset root that contains the output directory would copy the site into
1402 // itself, one build at a time.
1403 let out_canon = out
1404 .and_then(|out| std::fs::canonicalize(out).ok())
1405 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok());
1406 if out_canon.is_some_and(|o| o.starts_with(&base_canon)) {
1407 anyhow::bail!(
1408 "build.assets lists {root}, which contains the output directory {}",
1409 out.unwrap_or(Utf8Path::new("(none)"))
1410 );
1411 }
1412 for entry in WalkDir::new(&base).sort_by_file_name() {
1413 let entry = entry.with_context(|| format!("walking {base}"))?;
1414 if !entry.file_type().is_file() {
1415 continue;
1416 }
1417 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
1418 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
1419 let rel = abs
1420 .strip_prefix(&base)
1421 .map(|p| p.to_owned())
1422 .unwrap_or_else(|_| abs.clone());
1423 if rel.components().any(|c| is_hidden(c.as_str())) {
1424 continue;
1425 }
1426 assets.push(Asset { from: abs, rel });
1427 }
1428 }
1429
1430 let mut seen: HashMap<&Utf8Path, &Utf8Path> = HashMap::new();
1431 for asset in &assets {
1432 if let Some(first) = seen.insert(&asset.rel, &asset.from) {
1433 anyhow::bail!(
1434 "two files both publish to {}: {first} and {}",
1435 asset.rel,
1436 asset.from
1437 );
1438 }
1439 }
1440 assets.sort_by(|a, b| a.rel.cmp(&b.rel));
1441 Ok(assets)
1442}
1443
1444/// Source-relative directories that DISCOVER must not descend into: the template
1445/// directory (build input, not content) and the output directory when it lives inside
1446/// the source.
1447///
1448/// The output case is not a corner case — `orgo build . -o _site` is the obvious
1449/// thing to type, and without this the build copies its own output back into itself,
1450/// growing `_site/_site/_site/…` on every run.
1451fn excluded_dirs(src: &Utf8Path, config: &Config, out: Option<&Utf8Path>) -> Vec<Utf8PathBuf> {
1452 let mut dirs = vec![config.templates.dir.clone()];
1453 if let Some(out) = out {
1454 // Compare canonicalized paths so `.`, `./x` and an absolute path all agree.
1455 // The output may not exist yet, in which case it cannot contain anything and
1456 // the textual fallback is enough.
1457 let canon = |p: &Utf8Path| -> Option<Utf8PathBuf> {
1458 std::fs::canonicalize(p)
1459 .ok()
1460 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok())
1461 };
1462 match (canon(src), canon(out)) {
1463 (Some(src_abs), Some(out_abs)) => {
1464 if let Ok(rel) = out_abs.strip_prefix(&src_abs) {
1465 if !rel.as_str().is_empty() {
1466 dirs.push(rel.to_owned());
1467 }
1468 }
1469 }
1470 _ => {
1471 if let Ok(rel) = out.strip_prefix(src) {
1472 if !rel.as_str().is_empty() {
1473 dirs.push(rel.to_owned());
1474 }
1475 }
1476 }
1477 }
1478 }
1479 dirs
1480}
1481
1482/// Is this source-relative path excluded from discovery?
1483///
1484/// Dot-entries are skipped wholesale. That is the conventional rule for site generators,
1485/// Is this path component a dot-entry that must not be published?
1486///
1487/// Dot-directories are excluded because a source directory is very often a git repository,
1488/// and publishing `.git` — or `.env` — leaks a project's entire history alongside its
1489/// homepage. `.well-known` is the exception the web actually defines (RFC 8615): it holds
1490/// `security.txt`, ACME challenges, and other files whose entire purpose is to be served.
1491/// Excluding it is how a deploy quietly deletes a site's security contact.
1492fn is_hidden(component: &str) -> bool {
1493 component.starts_with('.')
1494 && component != "."
1495 && component != ".."
1496 && component != WELL_KNOWN
1497}
1498
1499/// The one dot-directory the web expects to be published.
1500const WELL_KNOWN: &str = ".well-known";
1501
1502/// and the reason is safety rather than tidiness: a source directory is very often a git
1503/// repository, and publishing `.git` — or `.env` — is a way to leak a project's entire
1504/// history alongside its homepage.
1505fn is_excluded(rel: &Utf8Path, skip_dirs: &[Utf8PathBuf]) -> bool {
1506 if rel.components().any(|c| is_hidden(c.as_str())) {
1507 return true;
1508 }
1509 skip_dirs
1510 .iter()
1511 .any(|dir| !dir.as_str().is_empty() && rel.starts_with(dir))
1512}
1513
1514/// Does this output path sit at the site root?
1515///
1516/// The nav is the site's global chrome, and listing *every* page in it makes an `n`-page
1517/// site emit `n²` nav links — 1,790 pages produced 284 MB of output, most of it nav. A
1518/// nav is a map of the site's top level, not an index of its contents, so it is built
1519/// from root-level pages only. Section pages reach their siblings through that section's
1520/// own landing page.
1521fn is_top_level(output: &Utf8Path) -> bool {
1522 output.parent().is_none_or(|p| p.as_str().is_empty())
1523}
1524
1525/// Everything a template can know about one page. Every `#+KEYWORD:` is passed through
1526/// under its lowercased name, so a template can use metadata this crate has never heard
1527/// of without the crate needing a release to support it.
1528fn page_context(doc: &Document, output: &Utf8Path, config: &Config) -> PageContext {
1529 let words = document_text(&doc.root).split_whitespace().count();
1530 let keyword = |name: &str| {
1531 doc.keywords
1532 .entries
1533 .iter()
1534 .find(|(k, _)| k.eq_ignore_ascii_case(name))
1535 .map(|(_, v)| v.clone())
1536 };
1537 PageContext {
1538 title: page_title(doc),
1539 url: output.to_string(),
1540 source: doc.source_path.to_string(),
1541 date_iso: keyword("DATE").as_deref().and_then(iso_date),
1542 year: keyword("DATE")
1543 .as_deref()
1544 .and_then(iso_date)
1545 .map(|d| d[..4].to_string()),
1546 date: keyword("DATE"),
1547 excerpt: keyword("DESCRIPTION")
1548 .filter(|d| !d.trim().is_empty())
1549 .or_else(|| first_paragraph(&doc.root))
1550 .unwrap_or_default(),
1551 content: None,
1552 word_count: words,
1553 reading_time: words.div_ceil(WORDS_PER_MINUTE).max(usize::from(words > 0)),
1554 toc: if option_enabled(&doc.keywords, "toc", config.html.toc) {
1555 table_of_contents(&doc.root)
1556 } else {
1557 Vec::new()
1558 },
1559 tags: keyword("FILETAGS")
1560 .unwrap_or_default()
1561 .split(':')
1562 .filter(|t| !t.trim().is_empty())
1563 .map(|t| t.trim().to_string())
1564 .collect(),
1565 keywords: doc
1566 .keywords
1567 .entries
1568 .iter()
1569 .map(|(k, v)| (k.to_lowercase(), v.clone()))
1570 .collect(),
1571 }
1572}
1573
1574fn page_title(doc: &Document) -> String {
1575 doc.keywords
1576 .entries
1577 .iter()
1578 .find(|(k, _)| k.eq_ignore_ascii_case("TITLE"))
1579 .map(|(_, v)| v.clone())
1580 .unwrap_or_else(|| {
1581 doc.source_path
1582 .file_stem()
1583 .unwrap_or("untitled")
1584 .to_string()
1585 })
1586}