krz/orgo
Lightning fast org-mode static site generator.
clone: git clone https://gitbay.org/krz/orgo.git
1//! Site build: walk a source directory, PARSE every `.org` file, INDEX their targets,
2//! then RESOLVE + RENDER + TEMPLATE each page into a linked static site, copying
3//! non-`.org` assets through unchanged (spec §2.1 DISCOVER…EMIT).
4//!
5//! v0.3 wires in the incremental layer (spec §4, [`crate::incremental`]): a persisted
6//! cache manifest lets a rebuild re-render only the pages whose composed `render_key`
7//! changed, plus the pages that *link into* a changed file's targets (the dependency
8//! graph, spec §4.3). Unchanged pages keep their existing on-disk output untouched.
9//! `--no-cache` forces a full rebuild; the cache is never a correctness dependency, so a
10//! full rebuild and an incremental rebuild produce byte-identical output.
11
12use std::collections::{HashMap, HashSet};
13use std::fs;
14
15use anyhow::{Context, Result};
16use camino::{Utf8Path, Utf8PathBuf};
17use rayon::prelude::*;
18use walkdir::WalkDir;
19
20use crate::incremental::{
21 self, combine, config_hash, render_key, resolved_links_hash, site_structure_hash,
22 site_structure_hash_ordered, template_hash, DepGraph, Hash, Manifest, PageRecord,
23 CACHE_FORMAT_VERSION,
24};
25use crate::index::{document_targets, SymbolTable, TargetId};
26use crate::model::{ContentHash, Diagnostic, Document};
27use crate::parser::parse;
28use crate::render::{self, render_with, Html, RenderOptions, SyntectHighlighter};
29use crate::resolve::resolve;
30use crate::config::{self, Config, NavMode, SortKey, SortOrder};
31use crate::template::{
32 GroupContext, NavItem, PageContext, Paginator, PaginatorPage, RenderContext, SiteContext,
33 Templater,
34};
35use crate::util::{
36 document_text, first_paragraph, is_draft, iso_date, iso_time, option_enabled,
37 output_path, output_url,
38 relative_root, slugify, table_of_contents,
39};
40
41/// Reading speed for [`PageContext::reading_time`]. 200 wpm is the conventional figure
42/// for prose on screen.
43const WORDS_PER_MINUTE: usize = 200;
44
45/// A fully built page: source and output paths (relative to their roots) and its
46/// final templated HTML.
47#[derive(Debug, Clone)]
48pub struct BuiltPage {
49 pub source: Utf8PathBuf,
50 pub output: Utf8PathBuf,
51 pub title: String,
52 pub html: String,
53}
54
55/// Unresolved internal links found during a build: `(page, target)` (spec §4.3.4).
56pub type BrokenLinks = Vec<(Utf8PathBuf, TargetId)>;
57
58/// Options controlling a site build.
59#[derive(Debug, Clone, Default)]
60pub struct BuildOptions {
61 /// Bypass the incremental cache and re-render every page (spec §4.5).
62 pub no_cache: bool,
63 /// Treat broken internal links as a build error rather than a warning (spec §4.3.4).
64 pub strict: bool,
65 /// Explicit config file, overriding `orgo.toml` in the source directory.
66 pub config_path: Option<Utf8PathBuf>,
67 /// Include pages marked `#+DRAFT:`, overriding `build.drafts` when set.
68 pub drafts: bool,
69}
70
71/// Summary of a site build.
72#[derive(Debug, Default)]
73pub struct SiteReport {
74 /// Every output page (rendered this build or reused from cache).
75 pub pages: Vec<Utf8PathBuf>,
76 /// Pages actually re-rendered and written this build (the invalidation set).
77 pub rendered: Vec<Utf8PathBuf>,
78 /// Pages whose existing on-disk output was reused unchanged (spec §4.1 skip rule).
79 pub skipped: Vec<Utf8PathBuf>,
80 pub assets: Vec<Utf8PathBuf>,
81 /// Unresolved internal links: `(page, target)`. Warnings, not failures (spec §4.3.4).
82 pub broken: Vec<(Utf8PathBuf, TargetId)>,
83 /// Parse diagnostics: `(source file, diagnostic)`, in file then line order.
84 pub diagnostics: Vec<(Utf8PathBuf, Diagnostic)>,
85}
86
87impl SiteReport {
88 /// Every diagnostic and broken link, formatted one per line as
89 /// `file:line: message` — the form an editor can jump to.
90 pub fn warnings(&self) -> Vec<String> {
91 let mut out: Vec<String> = self
92 .diagnostics
93 .iter()
94 .map(|(path, d)| format!("{path}:{}: {}", d.line, d.message))
95 .collect();
96 out.extend(
97 self.broken
98 .iter()
99 .map(|(page, target)| format!("{page}: unresolved link {target}")),
100 );
101 out
102 }
103}
104
105/// Everything a build needs about one page *before* the decision to render it: its
106/// hashes, its resolved element tree, and the dependency edges it participates in.
107struct PagePrep {
108 source: Utf8PathBuf,
109 output: Utf8PathBuf,
110 title: String,
111 content_hash: ContentHash,
112 resolved: crate::resolve::ResolvedDoc,
113 used: HashSet<TargetId>,
114 defines: HashSet<TargetId>,
115 broken: Vec<TargetId>,
116 diagnostics: Vec<Diagnostic>,
117 nav: Vec<NavItem>,
118 context: PageContext,
119 /// The layout this page renders through: `#+TEMPLATE:`, a `[[pages]]` rule, or
120 /// `base.html` (see [`config::page_template`]).
121 template: String,
122}
123
124/// A generated page, resolved against the pages it lists.
125struct Listing {
126 output: Utf8PathBuf,
127 template: String,
128 title: String,
129 /// The pages it lists, already sorted. Empty for a group index, which lists groups.
130 entries: Vec<PageContext>,
131 /// The group this page is for, when it belongs to a grouped collection.
132 group: Option<GroupContext>,
133 /// Every group of the owning collection. The content of a group index, and context
134 /// for a group page.
135 groups: Vec<GroupContext>,
136 /// Set when this is one page of a paginated listing.
137 paginator: Option<Paginator>,
138 /// Render each entry's body into `entry.content` (see
139 /// [`Collection::include_content`](crate::config::Collection::include_content)).
140 include_content: bool,
141 /// Content hash of each entry's source, in `entries` order. Not shown to templates —
142 /// it is how a content-carrying listing notices that a body it embeds has changed,
143 /// without rendering every body to find out.
144 entry_hashes: Vec<ContentHash>,
145}
146
147/// Split one listing's entries across numbered pages, appending each as its own
148/// [`Listing`].
149///
150/// Page 1 keeps `output`, so a section's canonical URL never moves as its page count
151/// changes — only pages 2..N are named by `paginate_output`. An empty listing still
152/// emits page 1, because a section that exists but has nothing in it should be a page
153/// saying so rather than a 404.
154#[allow(clippy::too_many_arguments)]
155fn push_paginated(
156 listings: &mut Vec<Listing>,
157 collection: &config::Collection,
158 output: Utf8PathBuf,
159 title: String,
160 entries: Vec<PageContext>,
161 entry_hashes: Vec<ContentHash>,
162 group: Option<GroupContext>,
163 groups: Vec<GroupContext>,
164) {
165 let per_page = collection.paginate;
166 if per_page == 0 {
167 listings.push(Listing {
168 output,
169 template: collection.template.clone(),
170 title,
171 entries,
172 group,
173 groups,
174 paginator: None,
175 include_content: collection.include_content,
176 entry_hashes: entry_hashes.clone(),
177 });
178 return;
179 }
180
181 let total_entries = entries.len();
182 let total = entries.len().div_ceil(per_page).max(1);
183 let slug = group.as_ref().map(|g| g.slug.clone()).unwrap_or_default();
184 let page_output = |n: usize| -> Utf8PathBuf {
185 if n == 1 {
186 return output.clone();
187 }
188 Utf8PathBuf::from(
189 collection
190 .paginate_output
191 .as_str()
192 .replace(config::GROUP_PLACEHOLDER, &slug)
193 .replace(config::PAGE_PLACEHOLDER, &n.to_string()),
194 )
195 };
196 let outputs: Vec<Utf8PathBuf> = (1..=total).map(page_output).collect();
197
198 for (idx, chunk) in entries.chunks(per_page).chain(
199 // `chunks` yields nothing for an empty slice; page 1 still has to exist.
200 std::iter::once(&[][..]).take(usize::from(total_entries == 0)),
201 ) .enumerate()
202 {
203 let current = idx + 1;
204 let here = &outputs[idx];
205 let url_to = |n: usize| output_url(here, &outputs[n - 1], None);
206 listings.push(Listing {
207 output: here.clone(),
208 template: collection.template.clone(),
209 include_content: collection.include_content,
210 entry_hashes: entry_hashes.clone(),
211 title: title.clone(),
212 entries: chunk.to_vec(),
213 group: group.clone(),
214 groups: groups.clone(),
215 paginator: Some(Paginator {
216 current,
217 total,
218 per_page,
219 total_entries,
220 prev_url: (current > 1).then(|| url_to(current - 1)),
221 next_url: (current < total).then(|| url_to(current + 1)),
222 first_url: url_to(1),
223 last_url: url_to(total),
224 pages: (1..=total)
225 .map(|n| PaginatorPage {
226 number: n,
227 url: url_to(n),
228 current: n == current,
229 })
230 .collect(),
231 }),
232 });
233 }
234}
235
236/// Build the listing pages a config asks for, each with its entries sorted.
237fn build_listings(config: &Config, preps: &[PagePrep]) -> Result<Vec<Listing>> {
238 let mut listings = Vec::new();
239 let hashes: HashMap<&str, ContentHash> = preps
240 .iter()
241 .map(|p| (p.source.as_str(), p.content_hash))
242 .collect();
243 let hashes_of = |entries: &[PageContext]| -> Vec<ContentHash> {
244 entries
245 .iter()
246 .filter_map(|e| hashes.get(e.source.as_str()).copied())
247 .collect()
248 };
249 for collection in &config.collections {
250 let mut entries: Vec<PageContext> = preps
251 .iter()
252 .filter(|p| {
253 collection.source.as_str().is_empty() || p.source.starts_with(&collection.source)
254 })
255 .map(|p| p.context.clone())
256 .collect();
257
258 // Sort ascending first, then reverse for `desc`, so the two orders are exact
259 // mirrors of one another rather than two separately-written comparisons.
260 match collection.sort {
261 SortKey::Title => entries.sort_by(|a, b| a.title.cmp(&b.title)),
262 SortKey::Path => entries.sort_by(|a, b| a.url.cmp(&b.url)),
263 // Undated pages sort last in the final order regardless of direction: a
264 // draft with no date should not lead an archive.
265 SortKey::Date => entries.sort_by(|a, b| {
266 // Date *and* time: org records when a note was written, and two notes
267 // from the same day have an order that the day alone cannot express.
268 let key = |p: &PageContext| {
269 p.date_iso.as_ref().map(|d| {
270 let time = p
271 .date
272 .as_deref()
273 .and_then(iso_time)
274 .unwrap_or_else(|| "00:00:00".to_string());
275 format!("{d}T{time}")
276 })
277 };
278 match (key(a), key(b)) {
279 (Some(x), Some(y)) => x.cmp(&y).then_with(|| a.url.cmp(&b.url)),
280 (Some(_), None) => std::cmp::Ordering::Greater,
281 (None, Some(_)) => std::cmp::Ordering::Less,
282 (None, None) => a.url.cmp(&b.url),
283 }
284 }),
285 }
286 if collection.order == SortOrder::Desc {
287 entries.reverse();
288 }
289
290 if collection.group_by.is_empty() {
291 let entry_hashes = hashes_of(&entries);
292 push_paginated(
293 &mut listings,
294 collection,
295 collection.output.clone(),
296 collection.title.clone(),
297 entries,
298 entry_hashes,
299 None,
300 Vec::new(),
301 );
302 continue;
303 }
304
305 // Grouped: one page per distinct term. `entries` is already sorted, and grouping
306 // preserves that order within each group.
307 let mut terms: Vec<String> = Vec::new();
308 let mut members: HashMap<String, Vec<PageContext>> = HashMap::new();
309 for entry in &entries {
310 for term in group_terms(entry, &collection.group_by) {
311 if !members.contains_key(&term) {
312 terms.push(term.clone());
313 }
314 members.entry(term).or_default().push(entry.clone());
315 }
316 }
317 // Terms are discovered in page order, which is arbitrary from a reader's point of
318 // view; sort so a tag index reads alphabetically and hashes deterministically.
319 terms.sort();
320
321 let mut groups: Vec<GroupContext> = Vec::new();
322 let mut slugs: HashMap<String, String> = HashMap::new();
323 for term in &terms {
324 let slug = slugify(term);
325 if slug.is_empty() {
326 anyhow::bail!(
327 "the {} value {term:?} has no URL-safe form; it cannot name a page",
328 collection.group_by
329 );
330 }
331 // `C++` and `C ++` both slugify to `c`, and one would silently overwrite the
332 // other's page.
333 if let Some(other) = slugs.insert(slug.clone(), term.clone()) {
334 anyhow::bail!(
335 "the {} values {other:?} and {term:?} both become {slug:?} in a URL; \
336 rename one so their pages do not collide",
337 collection.group_by
338 );
339 }
340 groups.push(GroupContext {
341 name: term.clone(),
342 slug: slug.clone(),
343 url: if collection.output.as_str().is_empty() {
344 String::new()
345 } else {
346 collection
347 .output
348 .as_str()
349 .replace(config::GROUP_PLACEHOLDER, &slug)
350 }
351 .to_string(),
352 count: members.get(term).map(Vec::len).unwrap_or(0),
353 });
354 }
355
356 if !collection.output.as_str().is_empty() {
357 for group in &groups {
358 let members_of = members.get(&group.name).cloned().unwrap_or_default();
359 let entry_hashes = hashes_of(&members_of);
360 push_paginated(
361 &mut listings,
362 collection,
363 Utf8PathBuf::from(&group.url),
364 collection
365 .title
366 .replace(config::GROUP_PLACEHOLDER, &group.name),
367 members_of,
368 entry_hashes,
369 Some(group.clone()),
370 // Deliberately not the whole group list. A page that can see every
371 // group depends on every group, so one new post would re-render every
372 // tag page — cost that scales with tag count, to support a tag cloud
373 // nobody has asked for. A tag page depends on its own posts, and the
374 // group index is where the group list belongs.
375 Vec::new(),
376 );
377 }
378 }
379 if !collection.index_output.as_str().is_empty() {
380 listings.push(Listing {
381 output: collection.index_output.clone(),
382 template: collection.index_template.clone(),
383 title: collection.index_title.clone(),
384 entries: Vec::new(),
385 group: None,
386 groups: groups.clone(),
387 paginator: None,
388 // A group index lists groups, not pages; there are no bodies to carry.
389 include_content: false,
390 entry_hashes: Vec::new(),
391 });
392 }
393 }
394
395 // A generated page writing over a real page would silently replace it. Group pages
396 // make this easy to hit by accident, since their paths come from content.
397 for listing in &listings {
398 if let Some(clash) = preps.iter().find(|p| p.output == listing.output) {
399 anyhow::bail!(
400 "collection output {} collides with the page built from {}",
401 listing.output,
402 clash.source
403 );
404 }
405 }
406 let mut claimed: HashMap<&Utf8PathBuf, ()> = HashMap::new();
407 for listing in &listings {
408 if claimed.insert(&listing.output, ()).is_some() {
409 anyhow::bail!("two generated pages both write to {}", listing.output);
410 }
411 }
412 Ok(listings)
413}
414
415/// The `(output, title)` a collection contributes to the nav. A grouped collection
416/// offers its index; an ungrouped one offers its single page.
417fn nav_target(collection: &config::Collection) -> (Utf8PathBuf, String) {
418 if !collection.group_by.is_empty() {
419 return (
420 collection.index_output.clone(),
421 collection.index_title.clone(),
422 );
423 }
424 (collection.output.clone(), collection.title.clone())
425}
426
427/// The group terms a page belongs to. `tags` is multi-valued — a page appears under
428/// every tag it carries — while any other key names a single-valued `#+KEYWORD:`.
429fn group_terms(page: &PageContext, group_by: &str) -> Vec<String> {
430 if group_by.eq_ignore_ascii_case("tags") {
431 return page.tags.clone();
432 }
433 page.keywords
434 .get(&group_by.to_lowercase())
435 .map(|v| v.trim())
436 .filter(|v| !v.is_empty())
437 .map(|v| vec![v.to_string()])
438 .unwrap_or_default()
439}
440
441/// Everything a listing template can see about its entries, hashed. This is the listing
442/// page's whole dependency: if none of these change, its output cannot have changed.
443///
444/// Entries are hashed through their *serialization* rather than a hand-picked set of
445/// fields. Picking fields means the hash drifts from what a template can read the moment
446/// one is added — which it had: the excerpt was missing, so rewriting a post's first
447/// paragraph left the old excerpt on the index until something else invalidated it.
448fn listing_entries_hash(listing: &Listing) -> Hash {
449 let mut fields: Vec<(String, String)> = listing
450 .entries
451 .iter()
452 .map(|e| {
453 (
454 e.url.clone(),
455 serde_json::to_string(e).unwrap_or_else(|_| e.title.clone()),
456 )
457 })
458 // A group index has no entries at all — its content *is* the group list, so the
459 // groups have to be in the hash or a tag index would never notice a new tag.
460 .chain(
461 listing
462 .groups
463 .iter()
464 .map(|g| (g.url.clone(), format!("{}\u{0}{}", g.name, g.count))),
465 )
466 .chain(listing.paginator.iter().map(|p| {
467 (
468 format!("{}/{}", p.current, p.total),
469 format!("{:?}|{:?}", p.prev_url, p.next_url),
470 )
471 }))
472 .chain([(listing.title.clone(), listing.template.clone())])
473 .collect();
474
475 // A listing that embeds its entries' bodies depends on those bodies. The source hash
476 // stands in for the rendered HTML, so noticing a change does not cost a render of
477 // every page listed.
478 if listing.include_content {
479 fields.extend(
480 listing
481 .entry_hashes
482 .iter()
483 .map(|h| ("content".to_string(), format!("{h:?}"))),
484 );
485 }
486
487 // Entry *order* is meaningful in a listing, so this hashes the sorted-by-us sequence
488 // rather than a set: a re-ordering is a real change to the page.
489 site_structure_hash_ordered(&fields)
490}
491
492/// A listing's entries with their rendered bodies attached, for a template that asked
493/// for them — a full-content feed being the case that needs it.
494///
495/// An entry whose source is not among the prepared pages keeps `content: none` rather
496/// than failing: the listing is still a valid page, and a feed item without a body is a
497/// better outcome than no feed.
498fn entries_with_content(
499 entries: &[PageContext],
500 preps: &[PagePrep],
501 highlighter: &SyntectHighlighter,
502 config: &Config,
503) -> Vec<PageContext> {
504 let by_source: HashMap<&str, &PagePrep> =
505 preps.iter().map(|p| (p.source.as_str(), p)).collect();
506 let opts = render_options(config);
507 entries
508 .par_iter()
509 .map(|entry| {
510 let mut entry = entry.clone();
511 if let Some(prep) = by_source.get(entry.source.as_str()) {
512 let Html(html) = render_with(&prep.resolved, highlighter, &opts);
513 entry.content = Some(html);
514 }
515 entry
516 })
517 .collect()
518}
519
520/// The nav a listing page shows: whatever the site's nav is, relativized to this
521/// listing's own location.
522fn listing_nav(preps: &[PagePrep], output: &Utf8Path) -> Vec<NavItem> {
523 let Some(first) = preps.first() else {
524 return Vec::new();
525 };
526 first
527 .nav
528 .iter()
529 .map(|item| {
530 // Nav URLs on `preps[0]` are relative to that page; re-resolve them against
531 // the site root, then against this listing's depth.
532 let absolute = resolve_relative(&first.output, &item.url);
533 NavItem {
534 title: item.title.clone(),
535 url: output_url(output, &absolute, None),
536 }
537 })
538 .collect()
539}
540
541/// Turn a URL relative to `from` back into a site-root-relative path.
542fn resolve_relative(from: &Utf8Path, url: &str) -> Utf8PathBuf {
543 if url == "#" {
544 return from.to_owned();
545 }
546 let base = from.parent().unwrap_or_else(|| Utf8Path::new(""));
547 let mut stack: Vec<&str> = base.components().map(|c| c.as_str()).collect();
548 for part in url.split('/') {
549 match part {
550 "." | "" => {}
551 ".." => {
552 stack.pop();
553 }
554 other => stack.push(other),
555 }
556 }
557 Utf8PathBuf::from(stack.join("/"))
558}
559
560/// The `PageContext` a listing page presents for *itself*.
561fn listing_context(listing: &Listing) -> PageContext {
562 PageContext {
563 title: listing.title.clone(),
564 url: listing.output.to_string(),
565 source: String::new(),
566 date: None,
567 date_iso: None,
568 year: None,
569 tags: Vec::new(),
570 excerpt: String::new(),
571 content: None,
572 word_count: 0,
573 reading_time: 0,
574 keywords: Default::default(),
575 toc: Vec::new(),
576 }
577}
578
579/// Which pages the configured [`NavMode`] selects, in nav order.
580fn nav_selection<'a>(config: &Config, candidates: &'a [NavCandidate]) -> Vec<&'a NavCandidate> {
581 match config.nav.mode {
582 NavMode::None => Vec::new(),
583 NavMode::All => candidates.iter().collect(),
584 // Generated pages are a section's landing page, which is what a nav entry should
585 // point at whatever depth the section lives at.
586 NavMode::TopLevel => candidates
587 .iter()
588 .filter(|c| c.generated || is_top_level(&c.output))
589 .collect(),
590 // Configured order wins over discovery order — a hand-written nav is a designed
591 // sequence, not an alphabetical one.
592 NavMode::Explicit => {
593 let mut chosen: Vec<&NavCandidate> = config
594 .nav
595 .pages
596 .iter()
597 .filter_map(|want| candidates.iter().find(|c| c.matches(want)))
598 .collect();
599 // A collection that asked for the nav but was not listed is appended rather
600 // than dropped, so `nav = true` never silently does nothing. Listing it puts
601 // it exactly where you said instead.
602 for candidate in candidates.iter().filter(|c| c.generated) {
603 if !chosen.iter().any(|c| c.output == candidate.output) {
604 chosen.push(candidate);
605 }
606 }
607 chosen
608 }
609 }
610}
611
612/// A page the navigation could contain: one written as `.org`, or one generated by a
613/// collection.
614struct NavCandidate {
615 /// The source path of an authored page. Empty for a generated one, which has none.
616 source: Utf8PathBuf,
617 output: Utf8PathBuf,
618 title: String,
619 /// Generated pages are appended when an explicit nav does not name them.
620 generated: bool,
621}
622
623impl NavCandidate {
624 /// Does `name` in `nav.pages` refer to this entry?
625 ///
626 /// Authored pages are named by their source — `about.org` — because that is the file
627 /// you wrote and its output path may be moved by `#+SLUG:`. Generated pages have no
628 /// source, so they are named by their output — `blog/index.html`. Either spelling is
629 /// accepted for either, so a config that names an output path still works.
630 fn matches(&self, name: &Utf8Path) -> bool {
631 (!self.source.as_str().is_empty() && self.source == name) || self.output == name
632 }
633}
634
635/// Every page the navigation could contain, authored pages first.
636fn nav_candidates(
637 config: &Config,
638 pages: &[(Utf8PathBuf, Utf8PathBuf, String)],
639) -> Vec<NavCandidate> {
640 let mut candidates: Vec<NavCandidate> = pages
641 .iter()
642 .map(|(source, output, title)| NavCandidate {
643 source: source.clone(),
644 output: output.clone(),
645 title: title.clone(),
646 generated: false,
647 })
648 .collect();
649 for collection in config.collections.iter().filter(|c| c.nav) {
650 let (output, title) = nav_target(collection);
651 candidates.push(NavCandidate {
652 source: Utf8PathBuf::new(),
653 output,
654 title,
655 generated: true,
656 });
657 }
658 candidates
659}
660
661/// DISCOVER + PARSE + INDEX + RESOLVE the whole site, returning per-page prep and the
662/// global symbol table. RENDER/TEMPLATE is deferred to the caller so the incremental
663/// build can render only the pages it must. PARSE/INDEX/RESOLVE are cheap and pure, so
664/// they run for every file each build; the incremental win is on RENDER + EMIT (spec §4.4).
665fn prepare_pages(
666 src: &Utf8Path,
667 config: &Config,
668 out: Option<&Utf8Path>,
669) -> Result<(Vec<PagePrep>, SymbolTable)> {
670 let (org_rel, _assets) = discover(src, config, out)?;
671
672 // PARSE every file (relative paths keep snapshots and links machine-independent).
673 // PARSE is a pure function of one file's bytes (spec §2.1), which is exactly the
674 // property that makes it safe to run in parallel. `par_iter().collect()` preserves
675 // input order, so the document list — and everything downstream of it — is identical
676 // to the sequential build regardless of how the work was scheduled.
677 let docs: Vec<Document> = org_rel
678 .par_iter()
679 .map(|rel| {
680 let abs = src.join(rel);
681 let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?;
682 parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))
683 })
684 .collect::<Result<Vec<_>>>()?;
685
686 // Drop drafts before anything else sees them. Removing them here rather than at emit
687 // time means they are absent from listings, the nav and the symbol table too — so a
688 // link *to* a draft is reported as broken, which is exactly what it would be on the
689 // published site.
690 let docs: Vec<Document> = docs
691 .into_iter()
692 .filter(|d| config.build.drafts || !is_draft(&d.keywords))
693 .collect();
694
695 // INDEX: collect every link target across the corpus.
696 let mut symbols = SymbolTable::new();
697 for doc in &docs {
698 symbols.index_document(doc);
699 }
700
701 // `(source, output, title)` for every page. Titles come from #+TITLE (falling back to
702 // the file stem) and URLs from each page's output path, which `#+SLUG:` can rename.
703 let all_pages: Vec<(Utf8PathBuf, Utf8PathBuf, String)> = docs
704 .iter()
705 .map(|d| {
706 (
707 d.source_path.clone(),
708 output_path(&d.source_path, &d.keywords),
709 page_title(d),
710 )
711 })
712 .collect();
713
714 // Two sources emitting one page would silently drop a page — and with slugs, a
715 // collision is a typo away and invisible in the source filenames.
716 let mut claimed: std::collections::HashMap<&Utf8PathBuf, &Utf8PathBuf> =
717 std::collections::HashMap::new();
718 for (source, out, _) in &all_pages {
719 if let Some(other) = claimed.insert(out, source) {
720 anyhow::bail!(
721 "output collision: {other} and {source} both build to {out} \
722 (check their #+SLUG:)"
723 );
724 }
725 }
726
727 // Authored pages and the landing pages collections generate, in one list so an
728 // explicit nav can order them together.
729 let candidates = nav_candidates(config, &all_pages);
730
731 // An explicit nav naming something that does not exist is a typo, and a silently
732 // shorter nav is a poor way to learn about it.
733 if config.nav.mode == NavMode::Explicit {
734 for want in &config.nav.pages {
735 if !candidates.iter().any(|c| c.matches(want)) {
736 anyhow::bail!(
737 "nav.pages lists {want}, which is neither a page in {src} nor a \
738 collection with `nav = true`"
739 );
740 }
741 }
742 }
743 let entries: Vec<(Utf8PathBuf, String)> = nav_selection(config, &candidates)
744 .into_iter()
745 .map(|c| (c.output.clone(), c.title.clone()))
746 .collect();
747
748 // RESOLVE reads the shared symbol table and writes only into its own page's output,
749 // so it parallelizes for free once INDEX has finished building the table.
750 let pages: Vec<PagePrep> = docs
751 .par_iter()
752 .map(|doc| {
753 let out = resolve(doc, &symbols);
754 let used: HashSet<TargetId> = out.used_targets.iter().cloned().collect();
755 let broken: Vec<TargetId> = out.broken.iter().map(|b| b.target.clone()).collect();
756 let defines: HashSet<TargetId> = document_targets(doc).into_iter().collect();
757
758 let output = output_path(&doc.source_path, &doc.keywords);
759
760 // Nav links are relative to *this* page (spec URL scheme, §8 Q3).
761 let nav: Vec<NavItem> = entries
762 .iter()
763 .map(|(path, title)| NavItem {
764 title: title.clone(),
765 url: output_url(&output, path, None),
766 })
767 .collect();
768
769 PagePrep {
770 context: page_context(doc, &output, config),
771 template: config::page_template(config, &doc.source_path, &doc.keywords),
772 source: doc.source_path.clone(),
773 output,
774 title: page_title(doc),
775 content_hash: doc.content_hash,
776 resolved: out.resolved,
777 used,
778 defines,
779 broken,
780 diagnostics: doc.diagnostics.clone(),
781 nav,
782 }
783 })
784 .collect();
785
786 Ok((pages, symbols))
787}
788
789/// Fail before rendering if any page names a template that does not exist.
790///
791/// minijinja would report the missing name on its own, but only once a page reaches it
792/// — and a typo in `#+TEMPLATE:` or a `[[pages]]` rule is worth naming together with the
793/// page that carries it and the templates that do exist.
794fn check_page_templates(templater: &Templater, preps: &[PagePrep]) -> Result<()> {
795 for p in preps {
796 if !templater.has(&p.template) {
797 let mut available = templater.names();
798 available.sort_unstable();
799 anyhow::bail!(
800 "{} renders through {}, which is not in the templates directory. \
801 Available: {}",
802 p.source,
803 p.template,
804 available.join(", ")
805 );
806 }
807 }
808 Ok(())
809}
810
811/// Parse + index + resolve + render + template a whole site *in memory*, without
812/// touching the output directory. Shared by the tests (full render, every page).
813pub fn render_site(src: &Utf8Path) -> Result<(Vec<BuiltPage>, BrokenLinks)> {
814 let config = Config::load(src)?;
815 config.validate()?;
816 let (preps, _symbols) = prepare_pages(src, &config, None)?;
817 let highlighter = SyntectHighlighter::new();
818 let templater = Templater::load(Some(&src.join(&config.templates.dir)), &config.site.base_url)?;
819 check_page_templates(&templater, &preps)?;
820 let site = site_context(&config);
821 let listing = page_listing(&config, &preps);
822
823 let mut pages = Vec::new();
824 let mut broken = Vec::new();
825 for p in &preps {
826 for t in &p.broken {
827 broken.push((p.source.clone(), t.clone()));
828 }
829 let html = render_page(&templater, &highlighter, &site, listing.as_deref(), &config, p)?;
830 pages.push(BuiltPage {
831 source: p.source.clone(),
832 output: p.output.clone(),
833 title: p.title.clone(),
834 html,
835 });
836 }
837 Ok((pages, broken))
838}
839
840/// The site's render options. A document's own `#+OPTIONS:` switches are applied by the
841/// renderer, so every caller gets them.
842fn render_options(config: &Config) -> RenderOptions {
843 RenderOptions {
844 heading_offset: config.html.heading_offset,
845 section_numbers: config.html.section_numbers,
846 special_strings: config.html.special_strings,
847 sub_superscript: config.html.sub_superscript,
848 entities: config.html.entities,
849 }
850}
851
852fn site_context(config: &Config) -> SiteContext {
853 SiteContext {
854 title: config.site.title.clone(),
855 base_url: config.site.base_url.clone(),
856 description: config.site.description.clone(),
857 language: config.site.language.clone(),
858 }
859}
860
861/// The `pages` list templates see, when configured to see one (see
862/// [`crate::config::Templates::expose_page_list`]).
863fn page_listing(config: &Config, preps: &[PagePrep]) -> Option<Vec<PageContext>> {
864 config
865 .templates
866 .expose_page_list
867 .then(|| preps.iter().map(|p| p.context.clone()).collect())
868}
869
870/// RENDER + TEMPLATE one prepared page into its final HTML string.
871#[allow(clippy::too_many_arguments)]
872fn render_page(
873 templater: &Templater,
874 highlighter: &SyntectHighlighter,
875 site: &SiteContext,
876 pages: Option<&[PageContext]>,
877 config: &Config,
878 p: &PagePrep,
879) -> Result<String> {
880 let opts = render_options(config);
881 let Html(fragment) = render_with(&p.resolved, highlighter, &opts);
882 // Relative to the *output* path, since `#+SLUG:` can move a page between depths.
883 let root = relative_root(&p.output);
884 let stylesheet = format!("{root}{SYNTAX_STYLESHEET}");
885 let mut ctx = RenderContext::new(site, &p.context, &p.nav, &stylesheet, &root);
886 ctx.body = &fragment;
887 ctx.pages = pages;
888 templater
889 .render(&p.template, &ctx)
890 .with_context(|| format!("templating {} through {}", p.source, p.template))
891}
892
893/// Site-root-relative name of the generated syntax stylesheet. Every page links to it.
894pub const SYNTAX_STYLESHEET: &str = "syntax.css";
895
896/// Site-root-relative name of the generated sitemap.
897pub const SITEMAP: &str = "sitemap.xml";
898
899/// `sitemap.xml` for every HTML page in `pages`, in URL order.
900///
901/// Only HTML: a sitemap is a list of pages for a crawler to read, and a feed or a
902/// stylesheet is neither. `lastmod` is the page's own `#+DATE:` where it has one — the
903/// nearest honest thing available without trusting a filesystem timestamp that a fresh
904/// clone would reset.
905fn sitemap(base_url: &str, pages: &[Utf8PathBuf], dated: &HashMap<&Utf8Path, &str>) -> String {
906 let mut urls: Vec<&Utf8PathBuf> = pages
907 .iter()
908 .filter(|p| p.extension() == Some("html"))
909 .collect();
910 urls.sort();
911 urls.dedup();
912
913 let base = base_url.trim_end_matches('/');
914 let mut out = String::from(
915 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
916 <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n",
917 );
918 for url in urls {
919 out.push_str("<url>\n");
920 out.push_str(&format!("<loc>{base}/{}</loc>\n", escape_xml(url.as_str())));
921 if let Some(date) = dated.get(url.as_path()) {
922 out.push_str(&format!("<lastmod>{date}</lastmod>\n"));
923 }
924 out.push_str("</url>\n");
925 }
926 out.push_str("</urlset>\n");
927 out
928}
929
930/// The five XML predefined entities. A `&` in a URL is the common one, from a query
931/// string that survived into a filename.
932fn escape_xml(s: &str) -> String {
933 s.replace('&', "&")
934 .replace('<', "<")
935 .replace('>', ">")
936 .replace('"', """)
937 .replace('\'', "'")
938}
939
940/// Full site build with the incremental layer (spec §4). Renders only the pages whose
941/// `render_key` changed or that link into a changed file's targets; reuses the on-disk
942/// output of everything else; persists an updated cache manifest.
943pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result<SiteReport> {
944 let mut cfg = match &opts.config_path {
945 Some(path) => Config::load_file(path)?,
946 None => Config::load(src)?,
947 };
948 cfg.validate()?;
949 // The flag turns drafts on; it never turns off a config that asked for them.
950 cfg.build.drafts |= opts.drafts;
951
952 // Create the output directory up front so it can be recognised and excluded when it
953 // lives inside the source tree.
954 fs::create_dir_all(out).with_context(|| format!("creating {out}"))?;
955 let (_org_rel, source_assets) = discover(src, &cfg, Some(out))?;
956 let assets = collect_assets(src, &cfg, Some(out), &source_assets)?;
957 let (preps, symbols) = prepare_pages(src, &cfg, Some(out))?;
958
959 let templater = Templater::load(Some(&src.join(&cfg.templates.dir)), &cfg.site.base_url)?;
960 check_page_templates(&templater, &preps)?;
961 let syntax_css = render::syntax_css(&cfg.highlight.theme).ok_or_else(|| {
962 anyhow::anyhow!(
963 "unknown highlight.theme {:?}. Available: {}",
964 cfg.highlight.theme,
965 render::available_themes().join(", ")
966 )
967 })?;
968
969 // The global hash classes (spec §4.1): a change in any invalidates the site. The
970 // config hash is combined with a site-structure hash covering the global chrome each
971 // page carries, so a change to that chrome re-renders the pages showing it.
972 //
973 // Which pages belong in that hash depends on what a template can *see*. Normally it
974 // is the nav only — a nested page cannot change another page's nav, so adding a blog
975 // post should render one page, not the site. But `expose_page_list` hands every
976 // template every page's metadata, and then any page's output really can depend on
977 // any other page, so the hash has to widen to match. Keyed on output paths, since a
978 // `#+SLUG:` change moves a page's URL without moving its source.
979 let all_pages: Vec<(Utf8PathBuf, Utf8PathBuf, String)> = preps
980 .iter()
981 .map(|p| (p.source.clone(), p.output.clone(), p.title.clone()))
982 .collect();
983 let structure_hash = if cfg.templates.expose_page_list {
984 let entries: Vec<(String, String)> = all_pages
985 .iter()
986 .map(|(_, out, title)| (out.to_string(), title.clone()))
987 .collect();
988 site_structure_hash(&entries)
989 } else {
990 // The same selection the nav itself is built from, so the two can never drift.
991 // Hashed in order, because the nav's order is itself part of every page.
992 let entries: Vec<(String, String)> = nav_selection(&cfg, &nav_candidates(&cfg, &all_pages))
993 .into_iter()
994 .map(|c| (c.output.to_string(), c.title.clone()))
995 .collect();
996 site_structure_hash_ordered(&entries)
997 };
998 let cfg_hash = combine(config_hash(&cfg), structure_hash);
999 // Per template rather than per site: a page's render key covers the layout it uses
1000 // and that layout's own includes, so editing `feed.xml` re-renders the feed.
1001 let tmpl_hash_for = |name: &str| template_hash(&templater.sources_for(name));
1002
1003 // Compose each page's render key and record its dependency edges.
1004 let mut new_graph = DepGraph::default();
1005 let mut new_records: Vec<(Utf8PathBuf, PageRecord, Hash)> = Vec::new();
1006 let listings = build_listings(&cfg, &preps)?;
1007 for p in &preps {
1008 let rlh = resolved_links_hash(&p.source, &p.used, &symbols);
1009 let key = render_key(p.content_hash, rlh, cfg_hash, tmpl_hash_for(&p.template));
1010 new_graph.defines.insert(p.source.clone(), p.defines.clone());
1011 new_graph.uses.insert(p.source.clone(), p.used.clone());
1012 new_records.push((
1013 p.source.clone(),
1014 PageRecord {
1015 content_hash: p.content_hash,
1016 render_key: key,
1017 output_path: p.output.clone(),
1018 },
1019 key,
1020 ));
1021 }
1022
1023 // Load the prior manifest (unless bypassed). Absent/corrupt/version-mismatch ⇒ None
1024 // ⇒ full rebuild (spec §4.5).
1025 let prior = if opts.no_cache {
1026 None
1027 } else {
1028 incremental::load_manifest(out)
1029 };
1030
1031 let rebuild: HashSet<Utf8PathBuf> = compute_rebuild_set(
1032 &preps,
1033 &new_records,
1034 &new_graph,
1035 cfg_hash,
1036 out,
1037 prior.as_ref(),
1038 );
1039
1040 // Delete outputs for pages that existed last build but are gone now (spec §4.3 step 1:
1041 // removed files). Their targets are already in the merged graph, so their linkers were
1042 // invalidated above.
1043 if let Some(prior) = &prior {
1044 // Keyed by source path for real pages and by output path for generated listings,
1045 // which is also how each records itself in the manifest. Listings have to be in
1046 // this set or the cleanup would delete the file it just decided to keep — and a
1047 // removed collection genuinely should have its output deleted.
1048 let mut current: HashSet<&Utf8PathBuf> = preps.iter().map(|p| &p.source).collect();
1049 current.extend(listings.iter().map(|l| &l.output));
1050 for (key, rec) in &prior.pages {
1051 if !current.contains(key) {
1052 let dest = out.join(&rec.output_path);
1053 let _ = fs::remove_file(&dest);
1054 }
1055 }
1056 }
1057
1058 let highlighter = SyntectHighlighter::with_syntaxes(Some(&src.join(&cfg.highlight.syntaxes_dir)));
1059 let site = site_context(&cfg);
1060 let listing = page_listing(&cfg, &preps);
1061 let mut report = SiteReport::default();
1062
1063 // RENDER + TEMPLATE + EMIT, in parallel. This is where a build's time actually goes
1064 // (syntect highlighting and templating dominate), and each page writes only its own
1065 // file, so the pages are independent.
1066 //
1067 // The parallel pass returns whether each page was written; the report is assembled
1068 // sequentially afterwards from `preps` order. Pushing to the report from inside the
1069 // parallel pass would make `rendered`/`skipped` ordering depend on thread scheduling,
1070 // which would be a non-deterministic build report over a deterministic build.
1071 let written: Vec<bool> = preps
1072 .par_iter()
1073 .map(|p| {
1074 if !rebuild.contains(&p.source) {
1075 // Skip: the on-disk output is already correct (spec §4.1). Leave it alone.
1076 return Ok(false);
1077 }
1078 let dest = out.join(&p.output);
1079 if let Some(parent) = dest.parent() {
1080 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1081 }
1082 let html = render_page(&templater, &highlighter, &site, listing.as_deref(), &cfg, p)?;
1083 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
1084 Ok(true)
1085 })
1086 .collect::<Result<Vec<_>>>()?;
1087
1088 for (p, was_written) in preps.iter().zip(&written) {
1089 for t in &p.broken {
1090 report.broken.push((p.source.clone(), t.clone()));
1091 }
1092 for d in &p.diagnostics {
1093 report.diagnostics.push((p.source.clone(), d.clone()));
1094 }
1095 report.pages.push(p.output.clone());
1096 if *was_written {
1097 report.rendered.push(p.output.clone());
1098 } else {
1099 report.skipped.push(p.output.clone());
1100 }
1101 }
1102
1103 // Generated listing pages (spec §2.1 EMIT). A listing has no source file, so it is
1104 // cached on the one thing it actually depends on: the entries it lists. Adding a post
1105 // therefore re-renders that section's index and nothing else — the same precision the
1106 // rest of the build gets from content hashing.
1107 for listing in &listings {
1108 let key = combine(
1109 listing_entries_hash(listing),
1110 combine(cfg_hash, tmpl_hash_for(&listing.template)),
1111 );
1112 let dest = out.join(&listing.output);
1113 let cached = prior
1114 .as_ref()
1115 .and_then(|m| m.pages.get(&listing.output))
1116 .map(|rec| rec.render_key == key)
1117 .unwrap_or(false);
1118
1119 report.pages.push(listing.output.clone());
1120 if cached && dest.exists() {
1121 report.skipped.push(listing.output.clone());
1122 } else {
1123 if let Some(parent) = dest.parent() {
1124 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1125 }
1126 let root = relative_root(&listing.output);
1127 let stylesheet = format!("{root}{SYNTAX_STYLESHEET}");
1128 let nav = listing_nav(&preps, &listing.output);
1129 let page_ctx = listing_context(listing);
1130 // Bodies are rendered here rather than when the listing was built, so a
1131 // cached feed costs nothing. This is the only place a page is rendered twice.
1132 let with_content = listing
1133 .include_content
1134 .then(|| entries_with_content(&listing.entries, &preps, &highlighter, &cfg));
1135 let mut ctx = RenderContext::new(&site, &page_ctx, &nav, &stylesheet, &root);
1136 ctx.pages = Some(with_content.as_deref().unwrap_or(&listing.entries));
1137 ctx.group = listing.group.as_ref();
1138 ctx.groups = &listing.groups;
1139 ctx.paginator = listing.paginator.as_ref();
1140 let html = templater
1141 .render(&listing.template, &ctx)
1142 .with_context(|| {
1143 format!(
1144 "rendering collection {} with template {} (available: {})",
1145 listing.output,
1146 listing.template,
1147 templater.names().join(", ")
1148 )
1149 })?;
1150 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
1151 report.rendered.push(listing.output.clone());
1152 }
1153
1154 new_records.push((
1155 listing.output.clone(),
1156 PageRecord {
1157 content_hash: key,
1158 render_key: key,
1159 output_path: listing.output.clone(),
1160 },
1161 key,
1162 ));
1163 }
1164
1165 // The syntax stylesheet the highlighter's CSS classes refer to. Written every build
1166 // (it is a few KB and depends only on the theme, which lives in the config hash).
1167 fs::write(out.join(SYNTAX_STYLESHEET), &syntax_css)
1168 .with_context(|| format!("writing {SYNTAX_STYLESHEET} under {out}"))?;
1169
1170 // A sitemap covers every page the build emits, authored and generated alike, so it is
1171 // written here rather than declared as a collection: a collection lists the pages it
1172 // was pointed at, and this one has to know about all of them including itself.
1173 if cfg.build.sitemap && !cfg.site.base_url.is_empty() {
1174 let dated: HashMap<&Utf8Path, &str> = preps
1175 .iter()
1176 .filter_map(|p| Some((p.output.as_path(), p.context.date_iso.as_deref()?)))
1177 .collect();
1178 let xml = sitemap(&cfg.site.base_url, &report.pages, &dated);
1179 fs::write(out.join(SITEMAP), xml)
1180 .with_context(|| format!("writing {SITEMAP} under {out}"))?;
1181 }
1182
1183 // Assets are a dumb copy in v0.3 (spec §8 Q11): copy every run. Cheap, and keeps the
1184 // full-vs-incremental byte equivalence trivially true for non-`.org` files.
1185 for asset in &assets {
1186 let dest = out.join(&asset.rel);
1187 if let Some(parent) = dest.parent() {
1188 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1189 }
1190 fs::copy(&asset.from, &dest)
1191 .with_context(|| format!("copying {} -> {dest}", asset.from))?;
1192 report.assets.push(asset.rel.clone());
1193 }
1194
1195 // Persist the manifest for the next build.
1196 let manifest = Manifest {
1197 format_version: CACHE_FORMAT_VERSION,
1198 config_hash: Some(cfg_hash),
1199 pages: new_records
1200 .into_iter()
1201 .map(|(src_path, rec, _)| (src_path, rec))
1202 .collect(),
1203 graph: new_graph,
1204 };
1205 incremental::save_manifest(out, &manifest)
1206 .with_context(|| format!("writing cache manifest under {out}"))?;
1207
1208 let warnings = report.warnings();
1209 if opts.strict && !warnings.is_empty() {
1210 for w in &warnings {
1211 eprintln!("error: {w}");
1212 }
1213 anyhow::bail!(
1214 "{} problem(s) under --strict ({} parse diagnostic(s), {} unresolved link(s))",
1215 warnings.len(),
1216 report.diagnostics.len(),
1217 report.broken.len()
1218 );
1219 }
1220 for w in &warnings {
1221 eprintln!("warning: {w}");
1222 }
1223
1224 Ok(report)
1225}
1226
1227/// The set of source files to (re)render this build (spec §4.3 invalidation algorithm),
1228/// as the union of:
1229/// - **no prior cache** (absent/corrupt/version-mismatch/`--no-cache`) ⇒ every page;
1230/// - a **global** config- or template-hash change ⇒ every page (spec §4.1);
1231/// - **content-changed** files ∪ pages that link into a changed file's targets, via the
1232/// dependency graph merged with the prior build's `defines` (spec §4.3, so a removed
1233/// target still invalidates its linkers);
1234/// - any page whose composed **render_key** differs from the cached one (catches URL
1235/// changes on linked targets precisely);
1236/// - any page whose **output file is missing** on disk.
1237fn compute_rebuild_set(
1238 preps: &[PagePrep],
1239 new_records: &[(Utf8PathBuf, PageRecord, Hash)],
1240 new_graph: &DepGraph,
1241 cfg_hash: Hash,
1242 out: &Utf8Path,
1243 prior: Option<&Manifest>,
1244) -> HashSet<Utf8PathBuf> {
1245 let all: HashSet<Utf8PathBuf> = preps.iter().map(|p| p.source.clone()).collect();
1246
1247 let Some(prior) = prior else {
1248 return all; // No usable cache ⇒ full rebuild.
1249 };
1250
1251 // A config change invalidates every page. Template changes do not come through here:
1252 // each page's render key carries the hash of the templates *it* uses, so the key
1253 // comparison below invalidates exactly the pages whose layout moved.
1254 if prior.config_hash != Some(cfg_hash) {
1255 return all;
1256 }
1257
1258 // Content-changed = hash differs from the cached record, or the file is new.
1259 let mut changed: HashSet<Utf8PathBuf> = HashSet::new();
1260 for p in preps {
1261 match prior.pages.get(&p.source) {
1262 Some(rec) if rec.content_hash == p.content_hash => {}
1263 _ => {
1264 changed.insert(p.source.clone());
1265 }
1266 }
1267 }
1268
1269 // Graph expansion: changed files ∪ pages that link into a changed file's targets.
1270 // Merge prior `defines` so a target a changed file removed still pulls its linkers.
1271 let merged = prior.graph.merged_defines_with(new_graph);
1272 let mut rebuild = incremental::invalidation_set(&changed, &merged);
1273
1274 // Precise render_key delta (catches a linked target's URL change; also a belt for the
1275 // graph). A page whose render_key matches the cache and whose output exists is correct.
1276 for (src_path, _rec, key) in new_records {
1277 let unchanged = prior
1278 .pages
1279 .get(src_path)
1280 .map(|old| old.render_key == *key)
1281 .unwrap_or(false);
1282 if !unchanged {
1283 rebuild.insert(src_path.clone());
1284 }
1285 }
1286
1287 // Any page whose output file is missing must be re-emitted regardless.
1288 for p in preps {
1289 if !out.join(&p.output).exists() {
1290 rebuild.insert(p.source.clone());
1291 }
1292 }
1293
1294 rebuild
1295}
1296
1297/// Walk `src`, returning `.org` source paths and non-`.org` asset paths, both relative
1298/// to `src` and sorted for deterministic output. The cache manifest is not an asset.
1299fn discover(
1300 src: &Utf8Path,
1301 config: &Config,
1302 out: Option<&Utf8Path>,
1303) -> Result<(Vec<Utf8PathBuf>, Vec<Utf8PathBuf>)> {
1304 let skip_dirs = excluded_dirs(src, config, out);
1305 let mut org = Vec::new();
1306 let mut assets = Vec::new();
1307
1308 let walker = WalkDir::new(src).sort_by_file_name().into_iter();
1309 for entry in walker.filter_entry(|e| {
1310 let Some(path) = Utf8Path::from_path(e.path()) else {
1311 return false;
1312 };
1313 let rel = path.strip_prefix(src).unwrap_or(path);
1314 // The source root itself always passes; `filter_entry` prunes whole subtrees.
1315 rel.as_str().is_empty() || !is_excluded(rel, &skip_dirs)
1316 }) {
1317 let entry = entry.with_context(|| format!("walking {src}"))?;
1318 if !entry.file_type().is_file() {
1319 continue;
1320 }
1321 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
1322 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
1323 let rel = abs
1324 .strip_prefix(src)
1325 .map(|p| p.to_owned())
1326 .unwrap_or_else(|_| abs.clone());
1327 if rel == config::CONFIG_FILE {
1328 continue;
1329 }
1330 if rel.extension() == Some("org") {
1331 org.push(rel);
1332 } else {
1333 assets.push(rel);
1334 }
1335 }
1336 org.sort();
1337 assets.sort();
1338 Ok((org, assets))
1339}
1340
1341/// One file to copy through to the output: where it is, and where it goes.
1342#[derive(Debug, Clone, PartialEq)]
1343pub struct Asset {
1344 /// Path to read from.
1345 pub from: Utf8PathBuf,
1346 /// Path to write, relative to the output root.
1347 pub rel: Utf8PathBuf,
1348}
1349
1350/// Every file to copy: the source directory's non-`.org` files, then each extra asset
1351/// root's contents, flattened onto the site root.
1352///
1353/// Two files claiming one output path is an error rather than a race — whichever won
1354/// would depend on directory order, and a site whose favicon changes when a file is
1355/// renamed elsewhere is worse than a build that stops.
1356fn collect_assets(
1357 src: &Utf8Path,
1358 config: &Config,
1359 out: Option<&Utf8Path>,
1360 from_source: &[Utf8PathBuf],
1361) -> Result<Vec<Asset>> {
1362 let mut assets: Vec<Asset> = from_source
1363 .iter()
1364 .map(|rel| Asset {
1365 from: src.join(rel),
1366 rel: rel.clone(),
1367 })
1368 .collect();
1369
1370 for root in &config.build.assets {
1371 let base = src.join(root);
1372 if !base.is_dir() {
1373 anyhow::bail!(
1374 "build.assets lists {root}, which is not a directory (looked in {base})"
1375 );
1376 }
1377 let base_canon = std::fs::canonicalize(&base)
1378 .ok()
1379 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok())
1380 .unwrap_or_else(|| base.clone());
1381 // An asset root that contains the output directory would copy the site into
1382 // itself, one build at a time.
1383 let out_canon = out
1384 .and_then(|out| std::fs::canonicalize(out).ok())
1385 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok());
1386 if out_canon.is_some_and(|o| o.starts_with(&base_canon)) {
1387 anyhow::bail!(
1388 "build.assets lists {root}, which contains the output directory {}",
1389 out.unwrap_or(Utf8Path::new("(none)"))
1390 );
1391 }
1392 for entry in WalkDir::new(&base).sort_by_file_name() {
1393 let entry = entry.with_context(|| format!("walking {base}"))?;
1394 if !entry.file_type().is_file() {
1395 continue;
1396 }
1397 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
1398 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
1399 let rel = abs
1400 .strip_prefix(&base)
1401 .map(|p| p.to_owned())
1402 .unwrap_or_else(|_| abs.clone());
1403 if rel.components().any(|c| is_hidden(c.as_str())) {
1404 continue;
1405 }
1406 assets.push(Asset { from: abs, rel });
1407 }
1408 }
1409
1410 let mut seen: HashMap<&Utf8Path, &Utf8Path> = HashMap::new();
1411 for asset in &assets {
1412 if let Some(first) = seen.insert(&asset.rel, &asset.from) {
1413 anyhow::bail!(
1414 "two files both publish to {}: {first} and {}",
1415 asset.rel,
1416 asset.from
1417 );
1418 }
1419 }
1420 assets.sort_by(|a, b| a.rel.cmp(&b.rel));
1421 Ok(assets)
1422}
1423
1424/// Source-relative directories that DISCOVER must not descend into: the template
1425/// directory (build input, not content) and the output directory when it lives inside
1426/// the source.
1427///
1428/// The output case is not a corner case — `orgo build . -o _site` is the obvious
1429/// thing to type, and without this the build copies its own output back into itself,
1430/// growing `_site/_site/_site/…` on every run.
1431fn excluded_dirs(src: &Utf8Path, config: &Config, out: Option<&Utf8Path>) -> Vec<Utf8PathBuf> {
1432 let mut dirs = vec![config.templates.dir.clone()];
1433 if let Some(out) = out {
1434 // Compare canonicalized paths so `.`, `./x` and an absolute path all agree.
1435 // The output may not exist yet, in which case it cannot contain anything and
1436 // the textual fallback is enough.
1437 let canon = |p: &Utf8Path| -> Option<Utf8PathBuf> {
1438 std::fs::canonicalize(p)
1439 .ok()
1440 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok())
1441 };
1442 match (canon(src), canon(out)) {
1443 (Some(src_abs), Some(out_abs)) => {
1444 if let Ok(rel) = out_abs.strip_prefix(&src_abs) {
1445 if !rel.as_str().is_empty() {
1446 dirs.push(rel.to_owned());
1447 }
1448 }
1449 }
1450 _ => {
1451 if let Ok(rel) = out.strip_prefix(src) {
1452 if !rel.as_str().is_empty() {
1453 dirs.push(rel.to_owned());
1454 }
1455 }
1456 }
1457 }
1458 }
1459 dirs
1460}
1461
1462/// Is this source-relative path excluded from discovery?
1463///
1464/// Dot-entries are skipped wholesale. That is the conventional rule for site generators,
1465/// Is this path component a dot-entry that must not be published?
1466///
1467/// Dot-directories are excluded because a source directory is very often a git repository,
1468/// and publishing `.git` — or `.env` — leaks a project's entire history alongside its
1469/// homepage. `.well-known` is the exception the web actually defines (RFC 8615): it holds
1470/// `security.txt`, ACME challenges, and other files whose entire purpose is to be served.
1471/// Excluding it is how a deploy quietly deletes a site's security contact.
1472fn is_hidden(component: &str) -> bool {
1473 component.starts_with('.')
1474 && component != "."
1475 && component != ".."
1476 && component != WELL_KNOWN
1477}
1478
1479/// The one dot-directory the web expects to be published.
1480const WELL_KNOWN: &str = ".well-known";
1481
1482/// and the reason is safety rather than tidiness: a source directory is very often a git
1483/// repository, and publishing `.git` — or `.env` — is a way to leak a project's entire
1484/// history alongside its homepage.
1485fn is_excluded(rel: &Utf8Path, skip_dirs: &[Utf8PathBuf]) -> bool {
1486 if rel.components().any(|c| is_hidden(c.as_str())) {
1487 return true;
1488 }
1489 skip_dirs
1490 .iter()
1491 .any(|dir| !dir.as_str().is_empty() && rel.starts_with(dir))
1492}
1493
1494/// Does this output path sit at the site root?
1495///
1496/// The nav is the site's global chrome, and listing *every* page in it makes an `n`-page
1497/// site emit `n²` nav links — 1,790 pages produced 284 MB of output, most of it nav. A
1498/// nav is a map of the site's top level, not an index of its contents, so it is built
1499/// from root-level pages only. Section pages reach their siblings through that section's
1500/// own landing page.
1501fn is_top_level(output: &Utf8Path) -> bool {
1502 output.parent().is_none_or(|p| p.as_str().is_empty())
1503}
1504
1505/// Everything a template can know about one page. Every `#+KEYWORD:` is passed through
1506/// under its lowercased name, so a template can use metadata this crate has never heard
1507/// of without the crate needing a release to support it.
1508fn page_context(doc: &Document, output: &Utf8Path, config: &Config) -> PageContext {
1509 let words = document_text(&doc.root).split_whitespace().count();
1510 let keyword = |name: &str| {
1511 doc.keywords
1512 .entries
1513 .iter()
1514 .find(|(k, _)| k.eq_ignore_ascii_case(name))
1515 .map(|(_, v)| v.clone())
1516 };
1517 PageContext {
1518 title: page_title(doc),
1519 url: output.to_string(),
1520 source: doc.source_path.to_string(),
1521 date_iso: keyword("DATE").as_deref().and_then(iso_date),
1522 year: keyword("DATE")
1523 .as_deref()
1524 .and_then(iso_date)
1525 .map(|d| d[..4].to_string()),
1526 date: keyword("DATE"),
1527 excerpt: keyword("DESCRIPTION")
1528 .filter(|d| !d.trim().is_empty())
1529 .or_else(|| first_paragraph(&doc.root))
1530 .unwrap_or_default(),
1531 content: None,
1532 word_count: words,
1533 reading_time: words.div_ceil(WORDS_PER_MINUTE).max(usize::from(words > 0)),
1534 toc: if option_enabled(&doc.keywords, "toc", config.html.toc) {
1535 table_of_contents(&doc.root)
1536 } else {
1537 Vec::new()
1538 },
1539 tags: keyword("FILETAGS")
1540 .unwrap_or_default()
1541 .split(':')
1542 .filter(|t| !t.trim().is_empty())
1543 .map(|t| t.trim().to_string())
1544 .collect(),
1545 keywords: doc
1546 .keywords
1547 .entries
1548 .iter()
1549 .map(|(k, v)| (k.to_lowercase(), v.clone()))
1550 .collect(),
1551 }
1552}
1553
1554fn page_title(doc: &Document) -> String {
1555 doc.keywords
1556 .entries
1557 .iter()
1558 .find(|(k, _)| k.eq_ignore_ascii_case("TITLE"))
1559 .map(|(_, v)| v.clone())
1560 .unwrap_or_else(|| {
1561 doc.source_path
1562 .file_stem()
1563 .unwrap_or("untitled")
1564 .to_string()
1565 })
1566}