src/site.rs
1745 lines · 67546 bytes
1//! Site build: walk a source directory, PARSE every `.org` file, INDEX their targets,
2//! then RESOLVE + RENDER + TEMPLATE each page into a linked static site, copying
3//! non-`.org` assets through unchanged (spec §2.1 DISCOVER…EMIT).
4//!
5//! v0.3 wires in the incremental layer (spec §4, [`crate::incremental`]): a persisted
6//! cache manifest lets a rebuild re-render only the pages whose composed `render_key`
7//! changed, plus the pages that *link into* a changed file's targets (the dependency
8//! graph, spec §4.3). Unchanged pages keep their existing on-disk output untouched.
9//! `--no-cache` forces a full rebuild; the cache is never a correctness dependency, so a
10//! full rebuild and an incremental rebuild produce byte-identical output.
11
12use std::collections::{BTreeSet, HashMap, HashSet};
13use std::fs;
14
15use anyhow::{Context, Result};
16use camino::{Utf8Path, Utf8PathBuf};
17use rayon::prelude::*;
18use walkdir::WalkDir;
19
20use crate::incremental::{
21 self, combine, config_hash, render_key, resolved_links_hash, site_structure_hash,
22 site_structure_hash_ordered, template_hash, DepGraph, Hash, Manifest, PageRecord,
23 CACHE_FORMAT_VERSION,
24};
25use crate::index::{document_targets, SymbolTable, TargetId};
26use crate::model::{ContentHash, Diagnostic, Document};
27use crate::parser::parse;
28use crate::render::{self, render_with, Html, RenderOptions, SyntectHighlighter};
29use crate::resolve::resolve;
30use crate::config::{self, Config, NavMode, SortKey, SortOrder};
31use crate::template::{
32 GroupContext, NavItem, PageContext, Paginator, PaginatorPage, RenderContext, SiteContext,
33 Templater,
34};
35use crate::theme;
36use crate::util::{
37 document_text, first_paragraph, is_draft, iso_date, iso_time, option_enabled,
38 output_path, output_url,
39 relative_root, slugify, table_of_contents,
40};
41
42/// Reading speed for [`PageContext::reading_time`]. 200 wpm is the conventional figure
43/// for prose on screen.
44const WORDS_PER_MINUTE: usize = 200;
45
46/// A fully built page: source and output paths (relative to their roots) and its
47/// final templated HTML.
48#[derive(Debug, Clone)]
49pub struct BuiltPage {
50 pub source: Utf8PathBuf,
51 pub output: Utf8PathBuf,
52 pub title: String,
53 pub html: String,
54}
55
56/// Unresolved internal links found during a build: `(page, target)` (spec §4.3.4).
57pub type BrokenLinks = Vec<(Utf8PathBuf, TargetId)>;
58
59/// Options controlling a site build.
60#[derive(Debug, Clone, Default)]
61pub struct BuildOptions {
62 /// Bypass the incremental cache and re-render every page (spec §4.5).
63 pub no_cache: bool,
64 /// Treat broken internal links as a build error rather than a warning (spec §4.3.4).
65 pub strict: bool,
66 /// Explicit config file, overriding `orgo.toml` in the source directory.
67 pub config_path: Option<Utf8PathBuf>,
68 /// Include pages marked `#+DRAFT:`, overriding `build.drafts` when set.
69 pub drafts: bool,
70}
71
72/// Summary of a site build.
73#[derive(Debug, Default)]
74pub struct SiteReport {
75 /// Every output page (rendered this build or reused from cache).
76 pub pages: Vec<Utf8PathBuf>,
77 /// Pages actually re-rendered and written this build (the invalidation set).
78 pub rendered: Vec<Utf8PathBuf>,
79 /// Pages whose existing on-disk output was reused unchanged (spec §4.1 skip rule).
80 pub skipped: Vec<Utf8PathBuf>,
81 pub assets: Vec<Utf8PathBuf>,
82 /// Unresolved internal links: `(page, target)`. Warnings, not failures (spec §4.3.4).
83 pub broken: Vec<(Utf8PathBuf, TargetId)>,
84 /// Parse diagnostics: `(source file, diagnostic)`, in file then line order.
85 pub diagnostics: Vec<(Utf8PathBuf, Diagnostic)>,
86}
87
88impl SiteReport {
89 /// Every diagnostic and broken link, formatted one per line as
90 /// `file:line: message` — the form an editor can jump to.
91 pub fn warnings(&self) -> Vec<String> {
92 let mut out: Vec<String> = self
93 .diagnostics
94 .iter()
95 .map(|(path, d)| format!("{path}:{}: {}", d.line, d.message))
96 .collect();
97 out.extend(
98 self.broken
99 .iter()
100 .map(|(page, target)| format!("{page}: unresolved link {target}")),
101 );
102 out
103 }
104}
105
106/// Everything a build needs about one page *before* the decision to render it: its
107/// hashes, its resolved element tree, and the dependency edges it participates in.
108struct PagePrep {
109 source: Utf8PathBuf,
110 output: Utf8PathBuf,
111 title: String,
112 content_hash: ContentHash,
113 resolved: crate::resolve::ResolvedDoc,
114 used: BTreeSet<TargetId>,
115 defines: BTreeSet<TargetId>,
116 broken: Vec<TargetId>,
117 diagnostics: Vec<Diagnostic>,
118 nav: Vec<NavItem>,
119 context: PageContext,
120 /// The layout this page renders through: `#+TEMPLATE:`, a `[[pages]]` rule, or
121 /// `base.html` (see [`config::page_template`]).
122 template: String,
123}
124
125/// A generated page, resolved against the pages it lists.
126struct Listing {
127 output: Utf8PathBuf,
128 template: String,
129 title: String,
130 /// The pages it lists, already sorted. Empty for a group index, which lists groups.
131 entries: Vec<PageContext>,
132 /// The group this page is for, when it belongs to a grouped collection.
133 group: Option<GroupContext>,
134 /// Every group of the owning collection. The content of a group index, and context
135 /// for a group page.
136 groups: Vec<GroupContext>,
137 /// Set when this is one page of a paginated listing.
138 paginator: Option<Paginator>,
139 /// Render each entry's body into `entry.content` (see
140 /// [`Collection::include_content`](crate::config::Collection::include_content)).
141 include_content: bool,
142 /// Content hash of each entry's source, in `entries` order. Not shown to templates —
143 /// it is how a content-carrying listing notices that a body it embeds has changed,
144 /// without rendering every body to find out.
145 entry_hashes: Vec<ContentHash>,
146}
147
148/// One listing before pagination: everything [`push_paginated`] needs besides the
149/// collection it came from.
150struct ListingPage {
151 output: Utf8PathBuf,
152 title: String,
153 entries: Vec<PageContext>,
154 entry_hashes: Vec<ContentHash>,
155 group: Option<GroupContext>,
156 groups: Vec<GroupContext>,
157}
158
159/// Split one listing's entries across numbered pages, appending each as its own
160/// [`Listing`].
161///
162/// Page 1 keeps `output`, so a section's canonical URL never moves as its page count
163/// changes — only pages 2..N are named by `paginate_output`. An empty listing still
164/// emits page 1, because a section that exists but has nothing in it should be a page
165/// saying so rather than a 404.
166fn push_paginated(listings: &mut Vec<Listing>, collection: &config::Collection, page: ListingPage) {
167 let ListingPage {
168 output,
169 title,
170 entries,
171 entry_hashes,
172 group,
173 groups,
174 } = page;
175 let per_page = collection.paginate;
176 if per_page == 0 {
177 listings.push(Listing {
178 output,
179 template: collection.template.clone(),
180 title,
181 entries,
182 group,
183 groups,
184 paginator: None,
185 include_content: collection.include_content,
186 entry_hashes: entry_hashes.clone(),
187 });
188 return;
189 }
190
191 let total_entries = entries.len();
192 let total = entries.len().div_ceil(per_page).max(1);
193 let slug = group.as_ref().map(|g| g.slug.clone()).unwrap_or_default();
194 let page_output = |n: usize| -> Utf8PathBuf {
195 if n == 1 {
196 return output.clone();
197 }
198 Utf8PathBuf::from(
199 collection
200 .paginate_output
201 .as_str()
202 .replace(config::GROUP_PLACEHOLDER, &slug)
203 .replace(config::PAGE_PLACEHOLDER, &n.to_string()),
204 )
205 };
206 let outputs: Vec<Utf8PathBuf> = (1..=total).map(page_output).collect();
207
208 for (idx, chunk) in entries.chunks(per_page).chain(
209 // `chunks` yields nothing for an empty slice; page 1 still has to exist.
210 std::iter::once(&[][..]).take(usize::from(total_entries == 0)),
211 ) .enumerate()
212 {
213 let current = idx + 1;
214 let here = &outputs[idx];
215 let url_to = |n: usize| output_url(here, &outputs[n - 1], None);
216 listings.push(Listing {
217 output: here.clone(),
218 template: collection.template.clone(),
219 include_content: collection.include_content,
220 entry_hashes: entry_hashes.clone(),
221 title: title.clone(),
222 entries: chunk.to_vec(),
223 group: group.clone(),
224 groups: groups.clone(),
225 paginator: Some(Paginator {
226 current,
227 total,
228 per_page,
229 total_entries,
230 prev_url: (current > 1).then(|| url_to(current - 1)),
231 next_url: (current < total).then(|| url_to(current + 1)),
232 first_url: url_to(1),
233 last_url: url_to(total),
234 pages: (1..=total)
235 .map(|n| PaginatorPage {
236 number: n,
237 url: url_to(n),
238 current: n == current,
239 })
240 .collect(),
241 }),
242 });
243 }
244}
245
246/// The pages a collection lists, in its configured order.
247fn collection_entries(collection: &config::Collection, preps: &[PagePrep]) -> Vec<PageContext> {
248 let mut entries: Vec<PageContext> = preps
249 .iter()
250 .filter(|p| {
251 collection.source.as_str().is_empty() || p.source.starts_with(&collection.source)
252 })
253 .map(|p| p.context.clone())
254 .collect();
255
256 // Sort ascending first, then reverse for `desc`, so the two orders are exact
257 // mirrors of one another rather than two separately-written comparisons.
258 match collection.sort {
259 SortKey::Title => entries.sort_by(|a, b| a.title.cmp(&b.title)),
260 SortKey::Path => entries.sort_by(|a, b| a.url.cmp(&b.url)),
261 SortKey::Date => entries.sort_by(compare_dates),
262 }
263 if collection.order == SortOrder::Desc {
264 entries.reverse();
265 }
266 entries
267}
268
269/// Ascending date order. Undated pages sort last in the final order regardless of
270/// direction: a draft with no date should not lead an archive.
271fn compare_dates(a: &PageContext, b: &PageContext) -> std::cmp::Ordering {
272 // Date *and* time: org records when a note was written, and two notes
273 // from the same day have an order that the day alone cannot express.
274 let key = |p: &PageContext| {
275 p.date_iso.as_ref().map(|d| {
276 let time = p
277 .date
278 .as_deref()
279 .and_then(iso_time)
280 .unwrap_or_else(|| "00:00:00".to_string());
281 format!("{d}T{time}")
282 })
283 };
284 match (key(a), key(b)) {
285 (Some(x), Some(y)) => x.cmp(&y).then_with(|| a.url.cmp(&b.url)),
286 (Some(_), None) => std::cmp::Ordering::Greater,
287 (None, Some(_)) => std::cmp::Ordering::Less,
288 (None, None) => a.url.cmp(&b.url),
289 }
290}
291
292/// Each distinct `group_by` term, sorted, with its members in `entries` order.
293fn group_members(
294 entries: &[PageContext],
295 group_by: &str,
296) -> (Vec<String>, HashMap<String, Vec<PageContext>>) {
297 let mut terms: Vec<String> = Vec::new();
298 let mut members: HashMap<String, Vec<PageContext>> = HashMap::new();
299 for entry in entries {
300 for term in group_terms(entry, group_by) {
301 if !members.contains_key(&term) {
302 terms.push(term.clone());
303 }
304 members.entry(term).or_default().push(entry.clone());
305 }
306 }
307 // Terms are discovered in page order, which is arbitrary from a reader's point of
308 // view; sort so a tag index reads alphabetically and hashes deterministically.
309 terms.sort();
310 (terms, members)
311}
312
313/// One [`GroupContext`] per term, rejecting terms that cannot name a page.
314fn group_contexts(
315 collection: &config::Collection,
316 terms: &[String],
317 members: &HashMap<String, Vec<PageContext>>,
318) -> Result<Vec<GroupContext>> {
319 let mut groups: Vec<GroupContext> = Vec::new();
320 let mut slugs: HashMap<String, String> = HashMap::new();
321 for term in terms {
322 let slug = slugify(term);
323 if slug.is_empty() {
324 anyhow::bail!(
325 "the {} value {term:?} has no URL-safe form; it cannot name a page",
326 collection.group_by
327 );
328 }
329 // `C++` and `C ++` both slugify to `c`, and one would silently overwrite the
330 // other's page.
331 if let Some(other) = slugs.insert(slug.clone(), term.clone()) {
332 anyhow::bail!(
333 "the {} values {other:?} and {term:?} both become {slug:?} in a URL; \
334 rename one so their pages do not collide",
335 collection.group_by
336 );
337 }
338 groups.push(GroupContext {
339 name: term.clone(),
340 slug: slug.clone(),
341 url: if collection.output.as_str().is_empty() {
342 String::new()
343 } else {
344 collection
345 .output
346 .as_str()
347 .replace(config::GROUP_PLACEHOLDER, &slug)
348 }
349 .to_string(),
350 count: members.get(term).map(Vec::len).unwrap_or(0),
351 });
352 }
353 Ok(groups)
354}
355
356/// Reject generated pages that would overwrite a real page or one another.
357fn check_listing_outputs(listings: &[Listing], preps: &[PagePrep]) -> Result<()> {
358 // A generated page writing over a real page would silently replace it. Group pages
359 // make this easy to hit by accident, since their paths come from content.
360 for listing in listings {
361 if let Some(clash) = preps.iter().find(|p| p.output == listing.output) {
362 anyhow::bail!(
363 "collection output {} collides with the page built from {}",
364 listing.output,
365 clash.source
366 );
367 }
368 }
369 let mut claimed: HashMap<&Utf8PathBuf, ()> = HashMap::new();
370 for listing in listings {
371 if claimed.insert(&listing.output, ()).is_some() {
372 anyhow::bail!("two generated pages both write to {}", listing.output);
373 }
374 }
375 Ok(())
376}
377
378/// Build the listing pages a config asks for, each with its entries sorted.
379fn build_listings(config: &Config, preps: &[PagePrep]) -> Result<Vec<Listing>> {
380 let mut listings = Vec::new();
381 let hashes: HashMap<&str, ContentHash> = preps
382 .iter()
383 .map(|p| (p.source.as_str(), p.content_hash))
384 .collect();
385 let hashes_of = |entries: &[PageContext]| -> Vec<ContentHash> {
386 entries
387 .iter()
388 .filter_map(|e| hashes.get(e.source.as_str()).copied())
389 .collect()
390 };
391 for collection in &config.collections {
392 let entries = collection_entries(collection, preps);
393
394 if collection.group_by.is_empty() {
395 let entry_hashes = hashes_of(&entries);
396 push_paginated(
397 &mut listings,
398 collection,
399 ListingPage {
400 output: collection.output.clone(),
401 title: collection.title.clone(),
402 entries,
403 entry_hashes,
404 group: None,
405 groups: Vec::new(),
406 },
407 );
408 continue;
409 }
410
411 // Grouped: one page per distinct term. `entries` is already sorted, and grouping
412 // preserves that order within each group.
413 let (terms, members) = group_members(&entries, &collection.group_by);
414 let groups = group_contexts(collection, &terms, &members)?;
415
416 if !collection.output.as_str().is_empty() {
417 for group in &groups {
418 let members_of = members.get(&group.name).cloned().unwrap_or_default();
419 let entry_hashes = hashes_of(&members_of);
420 push_paginated(
421 &mut listings,
422 collection,
423 ListingPage {
424 output: Utf8PathBuf::from(&group.url),
425 title: collection
426 .title
427 .replace(config::GROUP_PLACEHOLDER, &group.name),
428 entries: members_of,
429 entry_hashes,
430 group: Some(group.clone()),
431 // Deliberately not the whole group list. A page that can see every
432 // group depends on every group, so one new post would re-render every
433 // tag page — cost that scales with tag count, to support a tag cloud
434 // nobody has asked for. A tag page depends on its own posts, and the
435 // group index is where the group list belongs.
436 groups: Vec::new(),
437 },
438 );
439 }
440 }
441 if !collection.index_output.as_str().is_empty() {
442 listings.push(Listing {
443 output: collection.index_output.clone(),
444 template: collection.index_template.clone(),
445 title: collection.index_title.clone(),
446 entries: Vec::new(),
447 group: None,
448 groups: groups.clone(),
449 paginator: None,
450 // A group index lists groups, not pages; there are no bodies to carry.
451 include_content: false,
452 entry_hashes: Vec::new(),
453 });
454 }
455 }
456
457 check_listing_outputs(&listings, preps)?;
458 Ok(listings)
459}
460
461/// The `(output, title)` a collection contributes to the nav. A grouped collection
462/// offers its index; an ungrouped one offers its single page.
463fn nav_target(collection: &config::Collection) -> (Utf8PathBuf, String) {
464 if !collection.group_by.is_empty() {
465 return (
466 collection.index_output.clone(),
467 collection.index_title.clone(),
468 );
469 }
470 (collection.output.clone(), collection.title.clone())
471}
472
473/// The group terms a page belongs to. `tags` is multi-valued — a page appears under
474/// every tag it carries — while any other key names a single-valued `#+KEYWORD:`.
475fn group_terms(page: &PageContext, group_by: &str) -> Vec<String> {
476 if group_by.eq_ignore_ascii_case("tags") {
477 return page.tags.clone();
478 }
479 page.keywords
480 .get(&group_by.to_lowercase())
481 .map(|v| v.trim())
482 .filter(|v| !v.is_empty())
483 .map(|v| vec![v.to_string()])
484 .unwrap_or_default()
485}
486
487/// Everything a listing template can see about its entries, hashed. This is the listing
488/// page's whole dependency: if none of these change, its output cannot have changed.
489///
490/// Entries are hashed through their *serialization* rather than a hand-picked set of
491/// fields. Picking fields means the hash drifts from what a template can read the moment
492/// one is added — which it had: the excerpt was missing, so rewriting a post's first
493/// paragraph left the old excerpt on the index until something else invalidated it.
494fn listing_entries_hash(listing: &Listing) -> Hash {
495 let mut fields: Vec<(String, String)> = listing
496 .entries
497 .iter()
498 .map(|e| {
499 (
500 e.url.clone(),
501 serde_json::to_string(e).unwrap_or_else(|_| e.title.clone()),
502 )
503 })
504 // A group index has no entries at all — its content *is* the group list, so the
505 // groups have to be in the hash or a tag index would never notice a new tag.
506 .chain(
507 listing
508 .groups
509 .iter()
510 .map(|g| (g.url.clone(), format!("{}\u{0}{}", g.name, g.count))),
511 )
512 .chain(listing.paginator.iter().map(|p| {
513 (
514 format!("{}/{}", p.current, p.total),
515 format!("{:?}|{:?}", p.prev_url, p.next_url),
516 )
517 }))
518 .chain([(listing.title.clone(), listing.template.clone())])
519 .collect();
520
521 // A listing that embeds its entries' bodies depends on those bodies. The source hash
522 // stands in for the rendered HTML, so noticing a change does not cost a render of
523 // every page listed.
524 if listing.include_content {
525 fields.extend(
526 listing
527 .entry_hashes
528 .iter()
529 .map(|h| ("content".to_string(), format!("{h:?}"))),
530 );
531 }
532
533 // Entry *order* is meaningful in a listing, so this hashes the sorted-by-us sequence
534 // rather than a set: a re-ordering is a real change to the page.
535 site_structure_hash_ordered(&fields)
536}
537
538/// A listing's entries with their rendered bodies attached, for a template that asked
539/// for them — a full-content feed being the case that needs it.
540///
541/// An entry whose source is not among the prepared pages keeps `content: none` rather
542/// than failing: the listing is still a valid page, and a feed item without a body is a
543/// better outcome than no feed.
544fn entries_with_content(
545 entries: &[PageContext],
546 preps: &[PagePrep],
547 highlighter: &SyntectHighlighter,
548 config: &Config,
549) -> Vec<PageContext> {
550 let by_source: HashMap<&str, &PagePrep> =
551 preps.iter().map(|p| (p.source.as_str(), p)).collect();
552 let opts = render_options(config);
553 entries
554 .par_iter()
555 .map(|entry| {
556 let mut entry = entry.clone();
557 if let Some(prep) = by_source.get(entry.source.as_str()) {
558 let Html(html) = render_with(&prep.resolved, highlighter, &opts);
559 entry.content = Some(html);
560 }
561 entry
562 })
563 .collect()
564}
565
566/// The nav a listing page shows: whatever the site's nav is, relativized to this
567/// listing's own location.
568fn listing_nav(preps: &[PagePrep], output: &Utf8Path) -> Vec<NavItem> {
569 let Some(first) = preps.first() else {
570 return Vec::new();
571 };
572 first
573 .nav
574 .iter()
575 .map(|item| {
576 // Nav URLs on `preps[0]` are relative to that page; re-resolve them against
577 // the site root, then against this listing's depth.
578 let absolute = resolve_relative(&first.output, &item.url);
579 NavItem {
580 title: item.title.clone(),
581 url: output_url(output, &absolute, None),
582 }
583 })
584 .collect()
585}
586
587/// Turn a URL relative to `from` back into a site-root-relative path.
588fn resolve_relative(from: &Utf8Path, url: &str) -> Utf8PathBuf {
589 if url == "#" {
590 return from.to_owned();
591 }
592 let base = from.parent().unwrap_or_else(|| Utf8Path::new(""));
593 let mut stack: Vec<&str> = base.components().map(|c| c.as_str()).collect();
594 for part in url.split('/') {
595 match part {
596 "." | "" => {}
597 ".." => {
598 stack.pop();
599 }
600 other => stack.push(other),
601 }
602 }
603 Utf8PathBuf::from(stack.join("/"))
604}
605
606/// The `PageContext` a listing page presents for *itself*.
607fn listing_context(listing: &Listing) -> PageContext {
608 PageContext {
609 title: listing.title.clone(),
610 url: listing.output.to_string(),
611 source: String::new(),
612 date: None,
613 date_iso: None,
614 year: None,
615 tags: Vec::new(),
616 excerpt: String::new(),
617 content: None,
618 word_count: 0,
619 reading_time: 0,
620 keywords: Default::default(),
621 toc: Vec::new(),
622 }
623}
624
625/// Which pages the configured [`NavMode`] selects, in nav order.
626fn nav_selection<'a>(config: &Config, candidates: &'a [NavCandidate]) -> Vec<&'a NavCandidate> {
627 match config.nav.mode {
628 NavMode::None => Vec::new(),
629 NavMode::All => candidates.iter().collect(),
630 // Generated pages are a section's landing page, which is what a nav entry should
631 // point at whatever depth the section lives at.
632 NavMode::TopLevel => candidates
633 .iter()
634 .filter(|c| c.generated || is_top_level(&c.output))
635 .collect(),
636 // Configured order wins over discovery order — a hand-written nav is a designed
637 // sequence, not an alphabetical one.
638 NavMode::Explicit => {
639 let mut chosen: Vec<&NavCandidate> = config
640 .nav
641 .pages
642 .iter()
643 .filter_map(|want| candidates.iter().find(|c| c.matches(want)))
644 .collect();
645 // A collection that asked for the nav but was not listed is appended rather
646 // than dropped, so `nav = true` never silently does nothing. Listing it puts
647 // it exactly where you said instead.
648 for candidate in candidates.iter().filter(|c| c.generated) {
649 if !chosen.iter().any(|c| c.output == candidate.output) {
650 chosen.push(candidate);
651 }
652 }
653 chosen
654 }
655 }
656}
657
658/// A page the navigation could contain: one written as `.org`, or one generated by a
659/// collection.
660struct NavCandidate {
661 /// The source path of an authored page. Empty for a generated one, which has none.
662 source: Utf8PathBuf,
663 output: Utf8PathBuf,
664 title: String,
665 /// Generated pages are appended when an explicit nav does not name them.
666 generated: bool,
667}
668
669impl NavCandidate {
670 /// Does `name` in `nav.pages` refer to this entry?
671 ///
672 /// Authored pages are named by their source — `about.org` — because that is the file
673 /// you wrote and its output path may be moved by `#+SLUG:`. Generated pages have no
674 /// source, so they are named by their output — `blog/index.html`. Either spelling is
675 /// accepted for either, so a config that names an output path still works.
676 fn matches(&self, name: &Utf8Path) -> bool {
677 (!self.source.as_str().is_empty() && self.source == name) || self.output == name
678 }
679}
680
681/// Every page the navigation could contain, authored pages first.
682fn nav_candidates(
683 config: &Config,
684 pages: &[(Utf8PathBuf, Utf8PathBuf, String)],
685) -> Vec<NavCandidate> {
686 let mut candidates: Vec<NavCandidate> = pages
687 .iter()
688 .map(|(source, output, title)| NavCandidate {
689 source: source.clone(),
690 output: output.clone(),
691 title: title.clone(),
692 generated: false,
693 })
694 .collect();
695 for collection in config.collections.iter().filter(|c| c.nav) {
696 let (output, title) = nav_target(collection);
697 candidates.push(NavCandidate {
698 source: Utf8PathBuf::new(),
699 output,
700 title,
701 generated: true,
702 });
703 }
704 candidates
705}
706
707/// DISCOVER + PARSE + INDEX + RESOLVE the whole site, returning per-page prep and the
708/// global symbol table. RENDER/TEMPLATE is deferred to the caller so the incremental
709/// build can render only the pages it must. PARSE/INDEX/RESOLVE are cheap and pure, so
710/// they run for every file each build; the incremental win is on RENDER + EMIT (spec §4.4).
711fn prepare_pages(
712 src: &Utf8Path,
713 config: &Config,
714 out: Option<&Utf8Path>,
715) -> Result<(Vec<PagePrep>, SymbolTable)> {
716 let (org_rel, _assets) = discover(src, config, out)?;
717
718 // PARSE every file (relative paths keep snapshots and links machine-independent).
719 // PARSE is a pure function of one file's bytes (spec §2.1), which is exactly the
720 // property that makes it safe to run in parallel. `par_iter().collect()` preserves
721 // input order, so the document list — and everything downstream of it — is identical
722 // to the sequential build regardless of how the work was scheduled.
723 let docs: Vec<Document> = org_rel
724 .par_iter()
725 .map(|rel| {
726 let abs = src.join(rel);
727 let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?;
728 parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))
729 })
730 .collect::<Result<Vec<_>>>()?;
731
732 // Drop drafts before anything else sees them. Removing them here rather than at emit
733 // time means they are absent from listings, the nav and the symbol table too — so a
734 // link *to* a draft is reported as broken, which is exactly what it would be on the
735 // published site.
736 let docs: Vec<Document> = docs
737 .into_iter()
738 .filter(|d| config.build.drafts || !is_draft(&d.keywords))
739 .collect();
740
741 // INDEX: collect every link target across the corpus.
742 let mut symbols = SymbolTable::new();
743 for doc in &docs {
744 symbols.index_document(doc);
745 }
746
747 // `(source, output, title)` for every page. Titles come from #+TITLE (falling back to
748 // the file stem) and URLs from each page's output path, which `#+SLUG:` can rename.
749 let all_pages: Vec<(Utf8PathBuf, Utf8PathBuf, String)> = docs
750 .iter()
751 .map(|d| {
752 (
753 d.source_path.clone(),
754 output_path(&d.source_path, &d.keywords),
755 page_title(d),
756 )
757 })
758 .collect();
759
760 // Two sources emitting one page would silently drop a page — and with slugs, a
761 // collision is a typo away and invisible in the source filenames.
762 let mut claimed: std::collections::HashMap<&Utf8PathBuf, &Utf8PathBuf> =
763 std::collections::HashMap::new();
764 for (source, out, _) in &all_pages {
765 if let Some(other) = claimed.insert(out, source) {
766 anyhow::bail!(
767 "output collision: {other} and {source} both build to {out} \
768 (check their #+SLUG:)"
769 );
770 }
771 }
772
773 // Authored pages and the landing pages collections generate, in one list so an
774 // explicit nav can order them together.
775 let candidates = nav_candidates(config, &all_pages);
776
777 // An explicit nav naming something that does not exist is a typo, and a silently
778 // shorter nav is a poor way to learn about it.
779 if config.nav.mode == NavMode::Explicit {
780 for want in &config.nav.pages {
781 if !candidates.iter().any(|c| c.matches(want)) {
782 anyhow::bail!(
783 "nav.pages lists {want}, which is neither a page in {src} nor a \
784 collection with `nav = true`"
785 );
786 }
787 }
788 }
789 let entries: Vec<(Utf8PathBuf, String)> = nav_selection(config, &candidates)
790 .into_iter()
791 .map(|c| (c.output.clone(), c.title.clone()))
792 .collect();
793
794 // RESOLVE reads the shared symbol table and writes only into its own page's output,
795 // so it parallelizes for free once INDEX has finished building the table.
796 let pages: Vec<PagePrep> = docs
797 .par_iter()
798 .map(|doc| {
799 let out = resolve(doc, &symbols);
800 let used: BTreeSet<TargetId> = out.used_targets.iter().cloned().collect();
801 let broken: Vec<TargetId> = out.broken.iter().map(|b| b.target.clone()).collect();
802 let defines: BTreeSet<TargetId> = document_targets(doc).into_iter().collect();
803
804 let output = output_path(&doc.source_path, &doc.keywords);
805
806 // Nav links are relative to *this* page (spec URL scheme, §8 Q3).
807 let nav: Vec<NavItem> = entries
808 .iter()
809 .map(|(path, title)| NavItem {
810 title: title.clone(),
811 url: output_url(&output, path, None),
812 })
813 .collect();
814
815 PagePrep {
816 context: page_context(doc, &output, config),
817 template: config::page_template(config, &doc.source_path, &doc.keywords),
818 source: doc.source_path.clone(),
819 output,
820 title: page_title(doc),
821 content_hash: doc.content_hash,
822 resolved: out.resolved,
823 used,
824 defines,
825 broken,
826 diagnostics: doc.diagnostics.clone(),
827 nav,
828 }
829 })
830 .collect();
831
832 Ok((pages, symbols))
833}
834
835/// Fail before rendering if any page names a template that does not exist.
836///
837/// minijinja would report the missing name on its own, but only once a page reaches it
838/// — and a typo in `#+TEMPLATE:` or a `[[pages]]` rule is worth naming together with the
839/// page that carries it and the templates that do exist.
840fn check_page_templates(templater: &Templater, preps: &[PagePrep]) -> Result<()> {
841 for p in preps {
842 if !templater.has(&p.template) {
843 let mut available = templater.names();
844 available.sort_unstable();
845 anyhow::bail!(
846 "{} renders through {}, which is not in the templates directory. \
847 Available: {}",
848 p.source,
849 p.template,
850 available.join(", ")
851 );
852 }
853 }
854 Ok(())
855}
856
857/// Parse + index + resolve + render + template a whole site *in memory*, without
858/// touching the output directory. Shared by the tests (full render, every page).
859pub fn render_site(src: &Utf8Path) -> Result<(Vec<BuiltPage>, BrokenLinks)> {
860 let config = Config::load(src)?;
861 config.validate()?;
862 let (preps, _symbols) = prepare_pages(src, &config, None)?;
863 let highlighter = SyntectHighlighter::new();
864 let templater = Templater::load(Some(&src.join(&config.templates.dir)), &config.site.base_url)?;
865 check_page_templates(&templater, &preps)?;
866 let site = site_context(&config);
867 let listing = page_listing(&config, &preps);
868 let renderer = PageRenderer {
869 templater: &templater,
870 highlighter: &highlighter,
871 site: &site,
872 pages: listing.as_deref(),
873 config: &config,
874 preps: &preps,
875 };
876
877 let mut pages = Vec::new();
878 let mut broken = Vec::new();
879 for p in &preps {
880 for t in &p.broken {
881 broken.push((p.source.clone(), t.clone()));
882 }
883 let html = renderer.render_page(p)?;
884 pages.push(BuiltPage {
885 source: p.source.clone(),
886 output: p.output.clone(),
887 title: p.title.clone(),
888 html,
889 });
890 }
891 Ok((pages, broken))
892}
893
894/// The site's render options. A document's own `#+OPTIONS:` switches are applied by the
895/// renderer, so every caller gets them.
896fn render_options(config: &Config) -> RenderOptions {
897 RenderOptions {
898 heading_offset: config.html.heading_offset,
899 section_numbers: config.html.section_numbers,
900 special_strings: config.html.special_strings,
901 sub_superscript: config.html.sub_superscript,
902 entities: config.html.entities,
903 }
904}
905
906fn site_context(config: &Config) -> SiteContext {
907 SiteContext {
908 title: config.site.title.clone(),
909 base_url: config.site.base_url.clone(),
910 description: config.site.description.clone(),
911 language: config.site.language.clone(),
912 }
913}
914
915/// The `pages` list templates see, when configured to see one (see
916/// [`crate::config::Templates::expose_page_list`]).
917fn page_listing(config: &Config, preps: &[PagePrep]) -> Option<Vec<PageContext>> {
918 config
919 .templates
920 .expose_page_list
921 .then(|| preps.iter().map(|p| p.context.clone()).collect())
922}
923
924/// What every page render in one build shares.
925struct PageRenderer<'a> {
926 templater: &'a Templater,
927 highlighter: &'a SyntectHighlighter,
928 site: &'a SiteContext,
929 /// The `pages` list templates see (see [`page_listing`]).
930 pages: Option<&'a [PageContext]>,
931 config: &'a Config,
932 preps: &'a [PagePrep],
933}
934
935impl PageRenderer<'_> {
936 /// RENDER + TEMPLATE one prepared page into its final HTML string.
937 fn render_page(&self, p: &PagePrep) -> Result<String> {
938 let opts = render_options(self.config);
939 let Html(fragment) = render_with(&p.resolved, self.highlighter, &opts);
940 // Relative to the *output* path, since `#+SLUG:` can move a page between depths.
941 let root = relative_root(&p.output);
942 let stylesheet = format!("{root}{SYNTAX_STYLESHEET}");
943 let theme = theme_href(self.config, &root);
944 let mut ctx = RenderContext::new(self.site, &p.context, &p.nav, &stylesheet, &root);
945 ctx.body = &fragment;
946 ctx.theme = &theme;
947 ctx.pages = self.pages;
948 self.templater
949 .render(&p.template, &ctx)
950 .with_context(|| format!("templating {} through {}", p.source, p.template))
951 }
952
953 /// TEMPLATE one generated listing page into its final HTML string.
954 fn render_listing(&self, listing: &Listing) -> Result<String> {
955 let root = relative_root(&listing.output);
956 let stylesheet = format!("{root}{SYNTAX_STYLESHEET}");
957 let nav = listing_nav(self.preps, &listing.output);
958 let page_ctx = listing_context(listing);
959 // Bodies are rendered here rather than when the listing was built, so a
960 // cached feed costs nothing. This is the only place a page is rendered twice.
961 let with_content = listing.include_content.then(|| {
962 entries_with_content(&listing.entries, self.preps, self.highlighter, self.config)
963 });
964 let theme = theme_href(self.config, &root);
965 let mut ctx = RenderContext::new(self.site, &page_ctx, &nav, &stylesheet, &root);
966 ctx.theme = &theme;
967 ctx.pages = Some(with_content.as_deref().unwrap_or(&listing.entries));
968 ctx.group = listing.group.as_ref();
969 ctx.groups = &listing.groups;
970 ctx.paginator = listing.paginator.as_ref();
971 self.templater
972 .render(&listing.template, &ctx)
973 .with_context(|| {
974 format!(
975 "rendering collection {} with template {} (available: {})",
976 listing.output,
977 listing.template,
978 self.templater.names().join(", ")
979 )
980 })
981 }
982}
983
984/// Site-root-relative name of the generated syntax stylesheet. Every page links to it.
985pub const SYNTAX_STYLESHEET: &str = "syntax.css";
986
987/// Site-root-relative name of the built-in theme's stylesheet, written only when
988/// `site.theme` names one.
989pub const THEME_STYLESHEET: &str = "theme.css";
990
991/// Where a page at `root` finds `theme.css` — empty when the site has no theme, which
992/// is how a template knows to link nothing at all.
993fn theme_href(config: &Config, root: &str) -> String {
994 if config.site.theme.is_empty() {
995 String::new()
996 } else {
997 format!("{root}{THEME_STYLESHEET}")
998 }
999}
1000
1001/// Site-root-relative name of the generated sitemap.
1002pub const SITEMAP: &str = "sitemap.xml";
1003
1004/// `sitemap.xml` for every HTML page in `pages`, in URL order.
1005///
1006/// Only HTML: a sitemap is a list of pages for a crawler to read, and a feed or a
1007/// stylesheet is neither. `lastmod` is the page's own `#+DATE:` where it has one — the
1008/// nearest honest thing available without trusting a filesystem timestamp that a fresh
1009/// clone would reset.
1010fn sitemap(base_url: &str, pages: &[Utf8PathBuf], dated: &HashMap<&Utf8Path, &str>) -> String {
1011 let mut urls: Vec<&Utf8PathBuf> = pages
1012 .iter()
1013 .filter(|p| p.extension() == Some("html"))
1014 .collect();
1015 urls.sort();
1016 urls.dedup();
1017
1018 let base = base_url.trim_end_matches('/');
1019 let mut out = String::from(
1020 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
1021 <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n",
1022 );
1023 for url in urls {
1024 out.push_str("<url>\n");
1025 out.push_str(&format!("<loc>{base}/{}</loc>\n", escape_xml(url.as_str())));
1026 if let Some(date) = dated.get(url.as_path()) {
1027 out.push_str(&format!("<lastmod>{date}</lastmod>\n"));
1028 }
1029 out.push_str("</url>\n");
1030 }
1031 out.push_str("</urlset>\n");
1032 out
1033}
1034
1035/// The five XML predefined entities. A `&` in a URL is the common one, from a query
1036/// string that survived into a filename.
1037fn escape_xml(s: &str) -> String {
1038 s.replace('&', "&")
1039 .replace('<', "<")
1040 .replace('>', ">")
1041 .replace('"', """)
1042 .replace('\'', "'")
1043}
1044
1045/// Full site build with the incremental layer (spec §4). Renders only the pages whose
1046/// `render_key` changed or that link into a changed file's targets; reuses the on-disk
1047/// output of everything else; persists an updated cache manifest.
1048pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result<SiteReport> {
1049 let cfg = build_config(src, opts)?;
1050
1051 // Create the output directory up front so it can be recognised and excluded when it
1052 // lives inside the source tree.
1053 fs::create_dir_all(out).with_context(|| format!("creating {out}"))?;
1054 let (_org_rel, source_assets) = discover(src, &cfg, Some(out))?;
1055 let assets = collect_assets(src, &cfg, Some(out), &source_assets)?;
1056 let (preps, symbols) = prepare_pages(src, &cfg, Some(out))?;
1057
1058 let templater = Templater::load(Some(&src.join(&cfg.templates.dir)), &cfg.site.base_url)?;
1059 check_page_templates(&templater, &preps)?;
1060 let syntax_css = render::syntax_stylesheet(&cfg.highlight)?;
1061
1062 let cfg_hash = combine(config_hash(&cfg), structure_hash(&cfg, &preps));
1063 // Per template rather than per site: a page's render key covers the layout it uses
1064 // and that layout's own includes, so editing `feed.xml` re-renders the feed.
1065 let tmpl_hash_for = |name: &str| template_hash(&templater.sources_for(name));
1066
1067 let listings = build_listings(&cfg, &preps)?;
1068 let mut keys = page_keys(&preps, &symbols, cfg_hash, &tmpl_hash_for);
1069
1070 // Load the prior manifest (unless bypassed). Absent/corrupt/version-mismatch ⇒ None
1071 // ⇒ full rebuild (spec §4.5).
1072 let prior = if opts.no_cache {
1073 None
1074 } else {
1075 incremental::load_manifest(out)
1076 };
1077
1078 let rebuild = compute_rebuild_set(&preps, &keys, cfg_hash, out, prior.as_ref());
1079 if let Some(prior) = &prior {
1080 remove_deleted_outputs(prior, &preps, &listings, out);
1081 }
1082
1083 let highlighter = SyntectHighlighter::with_syntaxes(Some(&src.join(&cfg.highlight.syntaxes_dir)));
1084 let site = site_context(&cfg);
1085 let listing = page_listing(&cfg, &preps);
1086 let renderer = PageRenderer {
1087 templater: &templater,
1088 highlighter: &highlighter,
1089 site: &site,
1090 pages: listing.as_deref(),
1091 config: &cfg,
1092 preps: &preps,
1093 };
1094 let mut report = emit_pages(&renderer, &rebuild, out)?;
1095
1096 // Generated listing pages (spec §2.1 EMIT). A listing has no source file, so it is
1097 // cached on the one thing it actually depends on: the entries it lists. Adding a post
1098 // therefore re-renders that section's index and nothing else — the same precision the
1099 // rest of the build gets from content hashing.
1100 for listing in &listings {
1101 let key = combine(
1102 listing_entries_hash(listing),
1103 combine(cfg_hash, tmpl_hash_for(&listing.template)),
1104 );
1105 let dest = out.join(&listing.output);
1106 let cached = prior
1107 .as_ref()
1108 .and_then(|m| m.pages.get(&listing.output))
1109 .map(|rec| rec.render_key == key)
1110 .unwrap_or(false);
1111
1112 report.pages.push(listing.output.clone());
1113 if cached && dest.exists() {
1114 report.skipped.push(listing.output.clone());
1115 } else {
1116 create_parent(&dest)?;
1117 let html = renderer.render_listing(listing)?;
1118 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
1119 report.rendered.push(listing.output.clone());
1120 }
1121
1122 keys.records.push((
1123 listing.output.clone(),
1124 PageRecord {
1125 content_hash: key,
1126 render_key: key,
1127 output_path: listing.output.clone(),
1128 },
1129 key,
1130 ));
1131 }
1132
1133 write_site_files(&cfg, &preps, &syntax_css, &report.pages, out)?;
1134 copy_assets(&assets, out, &mut report)?;
1135
1136 // Persist the manifest for the next build.
1137 let manifest = Manifest {
1138 format_version: CACHE_FORMAT_VERSION,
1139 config_hash: Some(cfg_hash),
1140 pages: keys
1141 .records
1142 .into_iter()
1143 .map(|(src_path, rec, _)| (src_path, rec))
1144 .collect(),
1145 graph: keys.graph,
1146 };
1147 incremental::save_manifest(out, &manifest)
1148 .with_context(|| format!("writing cache manifest under {out}"))?;
1149
1150 report_warnings(&report, opts.strict)?;
1151 Ok(report)
1152}
1153
1154/// The config a build runs with: `--config` or the source directory's, validated, with
1155/// `--drafts` applied.
1156fn build_config(src: &Utf8Path, opts: &BuildOptions) -> Result<Config> {
1157 let mut cfg = match &opts.config_path {
1158 Some(path) => Config::load_file(path)?,
1159 None => Config::load(src)?,
1160 };
1161 cfg.validate()?;
1162 // The flag turns drafts on; it never turns off a config that asked for them.
1163 cfg.build.drafts |= opts.drafts;
1164 Ok(cfg)
1165}
1166
1167/// The site-structure half of the global hash class (spec §4.1): a change in any
1168/// invalidates the site. It covers the global chrome each page carries, so a change to
1169/// that chrome re-renders the pages showing it.
1170///
1171/// Which pages belong in that hash depends on what a template can *see*. Normally it
1172/// is the nav only — a nested page cannot change another page's nav, so adding a blog
1173/// post should render one page, not the site. But `expose_page_list` hands every
1174/// template every page's metadata, and then any page's output really can depend on
1175/// any other page, so the hash has to widen to match. Keyed on output paths, since a
1176/// `#+SLUG:` change moves a page's URL without moving its source.
1177fn structure_hash(cfg: &Config, preps: &[PagePrep]) -> Hash {
1178 let all_pages: Vec<(Utf8PathBuf, Utf8PathBuf, String)> = preps
1179 .iter()
1180 .map(|p| (p.source.clone(), p.output.clone(), p.title.clone()))
1181 .collect();
1182 if cfg.templates.expose_page_list {
1183 let entries: Vec<(String, String)> = all_pages
1184 .iter()
1185 .map(|(_, out, title)| (out.to_string(), title.clone()))
1186 .collect();
1187 site_structure_hash(&entries)
1188 } else {
1189 // The same selection the nav itself is built from, so the two can never drift.
1190 // Hashed in order, because the nav's order is itself part of every page.
1191 let entries: Vec<(String, String)> = nav_selection(cfg, &nav_candidates(cfg, &all_pages))
1192 .into_iter()
1193 .map(|c| (c.output.to_string(), c.title.clone()))
1194 .collect();
1195 site_structure_hash_ordered(&entries)
1196 }
1197}
1198
1199/// Each page's composed render key and dependency edges.
1200struct PageKeys {
1201 graph: DepGraph,
1202 /// `(key, record, render key)`, where the key is a page's source path or a
1203 /// listing's output path.
1204 records: Vec<(Utf8PathBuf, PageRecord, Hash)>,
1205}
1206
1207fn page_keys(
1208 preps: &[PagePrep],
1209 symbols: &SymbolTable,
1210 cfg_hash: Hash,
1211 tmpl_hash_for: &dyn Fn(&str) -> Hash,
1212) -> PageKeys {
1213 let mut keys = PageKeys {
1214 graph: DepGraph::default(),
1215 records: Vec::new(),
1216 };
1217 for p in preps {
1218 let rlh = resolved_links_hash(&p.source, &p.used, symbols);
1219 let key = render_key(p.content_hash, rlh, cfg_hash, tmpl_hash_for(&p.template));
1220 keys.graph.defines.insert(p.source.clone(), p.defines.clone());
1221 keys.graph.uses.insert(p.source.clone(), p.used.clone());
1222 keys.records.push((
1223 p.source.clone(),
1224 PageRecord {
1225 content_hash: p.content_hash,
1226 render_key: key,
1227 output_path: p.output.clone(),
1228 },
1229 key,
1230 ));
1231 }
1232 keys
1233}
1234
1235/// Delete outputs for pages that existed last build but are gone now (spec §4.3 step 1:
1236/// removed files). Their targets are already in the merged graph, so their linkers were
1237/// invalidated by [`compute_rebuild_set`].
1238fn remove_deleted_outputs(
1239 prior: &Manifest,
1240 preps: &[PagePrep],
1241 listings: &[Listing],
1242 out: &Utf8Path,
1243) {
1244 // Keyed by source path for real pages and by output path for generated listings,
1245 // which is also how each records itself in the manifest. Listings have to be in
1246 // this set or the cleanup would delete the file it just decided to keep — and a
1247 // removed collection genuinely should have its output deleted.
1248 let mut current: HashSet<&Utf8PathBuf> = preps.iter().map(|p| &p.source).collect();
1249 current.extend(listings.iter().map(|l| &l.output));
1250 for (key, rec) in &prior.pages {
1251 if !current.contains(key) {
1252 let dest = out.join(&rec.output_path);
1253 let _ = fs::remove_file(&dest);
1254 }
1255 }
1256}
1257
1258fn create_parent(dest: &Utf8Path) -> Result<()> {
1259 if let Some(parent) = dest.parent() {
1260 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
1261 }
1262 Ok(())
1263}
1264
1265/// RENDER + TEMPLATE + EMIT every page in `rebuild`, in parallel, and report on all of
1266/// them.
1267fn emit_pages(
1268 renderer: &PageRenderer,
1269 rebuild: &HashSet<Utf8PathBuf>,
1270 out: &Utf8Path,
1271) -> Result<SiteReport> {
1272 // This is where a build's time actually goes (syntect highlighting and templating
1273 // dominate), and each page writes only its own file, so the pages are independent.
1274 //
1275 // The parallel pass returns whether each page was written; the report is assembled
1276 // sequentially afterwards from `preps` order. Pushing to the report from inside the
1277 // parallel pass would make `rendered`/`skipped` ordering depend on thread scheduling,
1278 // which would be a non-deterministic build report over a deterministic build.
1279 let preps = renderer.preps;
1280 let written: Vec<bool> = preps
1281 .par_iter()
1282 .map(|p| {
1283 if !rebuild.contains(&p.source) {
1284 // Skip: the on-disk output is already correct (spec §4.1). Leave it alone.
1285 return Ok(false);
1286 }
1287 let dest = out.join(&p.output);
1288 create_parent(&dest)?;
1289 let html = renderer.render_page(p)?;
1290 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
1291 Ok(true)
1292 })
1293 .collect::<Result<Vec<_>>>()?;
1294
1295 let mut report = SiteReport::default();
1296 for (p, was_written) in preps.iter().zip(&written) {
1297 for t in &p.broken {
1298 report.broken.push((p.source.clone(), t.clone()));
1299 }
1300 for d in &p.diagnostics {
1301 report.diagnostics.push((p.source.clone(), d.clone()));
1302 }
1303 report.pages.push(p.output.clone());
1304 if *was_written {
1305 report.rendered.push(p.output.clone());
1306 } else {
1307 report.skipped.push(p.output.clone());
1308 }
1309 }
1310 Ok(report)
1311}
1312
1313/// The syntax stylesheet, the theme stylesheet, and the sitemap.
1314fn write_site_files(
1315 cfg: &Config,
1316 preps: &[PagePrep],
1317 syntax_css: &str,
1318 pages: &[Utf8PathBuf],
1319 out: &Utf8Path,
1320) -> Result<()> {
1321 // The syntax stylesheet the highlighter's CSS classes refer to. Written every build
1322 // (it is a few KB and depends only on the theme, which lives in the config hash).
1323 fs::write(out.join(SYNTAX_STYLESHEET), syntax_css)
1324 .with_context(|| format!("writing {SYNTAX_STYLESHEET} under {out}"))?;
1325
1326 // The built-in theme, on the same terms: compiled in, written every build, and named
1327 // in the config hash so switching themes re-renders the pages that link it.
1328 if let Some(css) = theme::theme_css(&cfg.site.theme) {
1329 fs::write(out.join(THEME_STYLESHEET), css)
1330 .with_context(|| format!("writing {THEME_STYLESHEET} under {out}"))?;
1331 }
1332
1333 // A sitemap covers every page the build emits, authored and generated alike, so it is
1334 // written here rather than declared as a collection: a collection lists the pages it
1335 // was pointed at, and this one has to know about all of them including itself.
1336 if cfg.build.sitemap && !cfg.site.base_url.is_empty() {
1337 let dated: HashMap<&Utf8Path, &str> = preps
1338 .iter()
1339 .filter_map(|p| Some((p.output.as_path(), p.context.date_iso.as_deref()?)))
1340 .collect();
1341 let xml = sitemap(&cfg.site.base_url, pages, &dated);
1342 fs::write(out.join(SITEMAP), xml)
1343 .with_context(|| format!("writing {SITEMAP} under {out}"))?;
1344 }
1345 Ok(())
1346}
1347
1348/// Assets are a dumb copy in v0.3 (spec §8 Q11): copy every run. Cheap, and keeps the
1349/// full-vs-incremental byte equivalence trivially true for non-`.org` files.
1350fn copy_assets(assets: &[Asset], out: &Utf8Path, report: &mut SiteReport) -> Result<()> {
1351 for asset in assets {
1352 let dest = out.join(&asset.rel);
1353 create_parent(&dest)?;
1354 fs::copy(&asset.from, &dest)
1355 .with_context(|| format!("copying {} -> {dest}", asset.from))?;
1356 report.assets.push(asset.rel.clone());
1357 }
1358 Ok(())
1359}
1360
1361/// Print the build's warnings, or fail with them under `--strict`.
1362fn report_warnings(report: &SiteReport, strict: bool) -> Result<()> {
1363 let warnings = report.warnings();
1364 if strict && !warnings.is_empty() {
1365 for w in &warnings {
1366 eprintln!("error: {w}");
1367 }
1368 anyhow::bail!(
1369 "{} problem(s) under --strict ({} parse diagnostic(s), {} unresolved link(s))",
1370 warnings.len(),
1371 report.diagnostics.len(),
1372 report.broken.len()
1373 );
1374 }
1375 for w in &warnings {
1376 eprintln!("warning: {w}");
1377 }
1378 Ok(())
1379}
1380
1381/// The set of source files to (re)render this build (spec §4.3 invalidation algorithm),
1382/// as the union of:
1383/// - **no prior cache** (absent/corrupt/version-mismatch/`--no-cache`) ⇒ every page;
1384/// - a **global** config- or template-hash change ⇒ every page (spec §4.1);
1385/// - **content-changed** files ∪ pages that link into a changed file's targets, via the
1386/// dependency graph merged with the prior build's `defines` (spec §4.3, so a removed
1387/// target still invalidates its linkers);
1388/// - any page whose composed **render_key** differs from the cached one (catches URL
1389/// changes on linked targets precisely);
1390/// - any page whose **output file is missing** on disk.
1391fn compute_rebuild_set(
1392 preps: &[PagePrep],
1393 keys: &PageKeys,
1394 cfg_hash: Hash,
1395 out: &Utf8Path,
1396 prior: Option<&Manifest>,
1397) -> HashSet<Utf8PathBuf> {
1398 let all: HashSet<Utf8PathBuf> = preps.iter().map(|p| p.source.clone()).collect();
1399
1400 let Some(prior) = prior else {
1401 return all; // No usable cache ⇒ full rebuild.
1402 };
1403
1404 // A config change invalidates every page. Template changes do not come through here:
1405 // each page's render key carries the hash of the templates *it* uses, so the key
1406 // comparison below invalidates exactly the pages whose layout moved.
1407 if prior.config_hash != Some(cfg_hash) {
1408 return all;
1409 }
1410
1411 // Content-changed = hash differs from the cached record, or the file is new.
1412 let mut changed: HashSet<Utf8PathBuf> = HashSet::new();
1413 for p in preps {
1414 match prior.pages.get(&p.source) {
1415 Some(rec) if rec.content_hash == p.content_hash => {}
1416 _ => {
1417 changed.insert(p.source.clone());
1418 }
1419 }
1420 }
1421
1422 // Graph expansion: changed files ∪ pages that link into a changed file's targets.
1423 // Merge prior `defines` so a target a changed file removed still pulls its linkers.
1424 let merged = prior.graph.merged_defines_with(&keys.graph);
1425 let mut rebuild = incremental::invalidation_set(&changed, &merged);
1426
1427 // Precise render_key delta (catches a linked target's URL change; also a belt for the
1428 // graph). A page whose render_key matches the cache and whose output exists is correct.
1429 for (src_path, _rec, key) in &keys.records {
1430 let unchanged = prior
1431 .pages
1432 .get(src_path)
1433 .map(|old| old.render_key == *key)
1434 .unwrap_or(false);
1435 if !unchanged {
1436 rebuild.insert(src_path.clone());
1437 }
1438 }
1439
1440 // Any page whose output file is missing must be re-emitted regardless.
1441 for p in preps {
1442 if !out.join(&p.output).exists() {
1443 rebuild.insert(p.source.clone());
1444 }
1445 }
1446
1447 rebuild
1448}
1449
1450/// Walk `src`, returning `.org` source paths and non-`.org` asset paths, both relative
1451/// to `src` and sorted for deterministic output. The cache manifest is not an asset.
1452fn discover(
1453 src: &Utf8Path,
1454 config: &Config,
1455 out: Option<&Utf8Path>,
1456) -> Result<(Vec<Utf8PathBuf>, Vec<Utf8PathBuf>)> {
1457 let skip_dirs = excluded_dirs(src, config, out);
1458 let mut org = Vec::new();
1459 let mut assets = Vec::new();
1460
1461 let walker = WalkDir::new(src).sort_by_file_name().into_iter();
1462 for entry in walker.filter_entry(|e| {
1463 let Some(path) = Utf8Path::from_path(e.path()) else {
1464 return false;
1465 };
1466 let rel = path.strip_prefix(src).unwrap_or(path);
1467 // The source root itself always passes; `filter_entry` prunes whole subtrees.
1468 if rel.as_str().is_empty() {
1469 return true;
1470 }
1471 !is_excluded(rel, &skip_dirs) && !is_build_output(e)
1472 }) {
1473 let entry = entry.with_context(|| format!("walking {src}"))?;
1474 if !entry.file_type().is_file() {
1475 continue;
1476 }
1477 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
1478 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
1479 let rel = abs
1480 .strip_prefix(src)
1481 .map(|p| p.to_owned())
1482 .unwrap_or_else(|_| abs.clone());
1483 if rel == config::CONFIG_FILE {
1484 continue;
1485 }
1486 if rel.extension() == Some("org") {
1487 org.push(rel);
1488 } else {
1489 assets.push(rel);
1490 }
1491 }
1492 org.sort();
1493 assets.sort();
1494 Ok((org, assets))
1495}
1496
1497/// One file to copy through to the output: where it is, and where it goes.
1498#[derive(Debug, Clone, PartialEq)]
1499pub struct Asset {
1500 /// Path to read from.
1501 pub from: Utf8PathBuf,
1502 /// Path to write, relative to the output root.
1503 pub rel: Utf8PathBuf,
1504}
1505
1506/// Every file to copy: the source directory's non-`.org` files, then each extra asset
1507/// root's contents, flattened onto the site root.
1508///
1509/// Two files claiming one output path is an error rather than a race — whichever won
1510/// would depend on directory order, and a site whose favicon changes when a file is
1511/// renamed elsewhere is worse than a build that stops.
1512fn collect_assets(
1513 src: &Utf8Path,
1514 config: &Config,
1515 out: Option<&Utf8Path>,
1516 from_source: &[Utf8PathBuf],
1517) -> Result<Vec<Asset>> {
1518 let mut assets: Vec<Asset> = from_source
1519 .iter()
1520 .map(|rel| Asset {
1521 from: src.join(rel),
1522 rel: rel.clone(),
1523 })
1524 .collect();
1525
1526 for root in &config.build.assets {
1527 let base = src.join(root);
1528 check_asset_root(root, &base, out)?;
1529 assets.extend(walk_asset_root(&base)?);
1530 }
1531
1532 let mut seen: HashMap<&Utf8Path, &Utf8Path> = HashMap::new();
1533 for asset in &assets {
1534 if let Some(first) = seen.insert(&asset.rel, &asset.from) {
1535 anyhow::bail!(
1536 "two files both publish to {}: {first} and {}",
1537 asset.rel,
1538 asset.from
1539 );
1540 }
1541 }
1542 assets.sort_by(|a, b| a.rel.cmp(&b.rel));
1543 Ok(assets)
1544}
1545
1546/// Reject an asset root that is missing or that would copy the site into itself.
1547fn check_asset_root(root: &Utf8Path, base: &Utf8Path, out: Option<&Utf8Path>) -> Result<()> {
1548 if !base.is_dir() {
1549 anyhow::bail!(
1550 "build.assets lists {root}, which is not a directory (looked in {base})"
1551 );
1552 }
1553 let base_canon = std::fs::canonicalize(base)
1554 .ok()
1555 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok())
1556 .unwrap_or_else(|| base.to_owned());
1557 // An asset root that contains the output directory would copy the site into
1558 // itself, one build at a time.
1559 let out_canon = out
1560 .and_then(|out| std::fs::canonicalize(out).ok())
1561 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok());
1562 if out_canon.is_some_and(|o| o.starts_with(&base_canon)) {
1563 anyhow::bail!(
1564 "build.assets lists {root}, which contains the output directory {}",
1565 out.unwrap_or(Utf8Path::new("(none)"))
1566 );
1567 }
1568 Ok(())
1569}
1570
1571/// Every non-hidden file under an asset root, relative to it, in file-name order.
1572fn walk_asset_root(base: &Utf8Path) -> Result<Vec<Asset>> {
1573 let mut assets = Vec::new();
1574 for entry in WalkDir::new(base).sort_by_file_name() {
1575 let entry = entry.with_context(|| format!("walking {base}"))?;
1576 if !entry.file_type().is_file() {
1577 continue;
1578 }
1579 let abs = Utf8PathBuf::from_path_buf(entry.into_path())
1580 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
1581 let rel = abs
1582 .strip_prefix(base)
1583 .map(|p| p.to_owned())
1584 .unwrap_or_else(|_| abs.clone());
1585 if rel.components().any(|c| is_hidden(c.as_str())) {
1586 continue;
1587 }
1588 assets.push(Asset { from: abs, rel });
1589 }
1590 Ok(assets)
1591}
1592
1593/// Source-relative directories that DISCOVER must not descend into: the template
1594/// directory (build input, not content) and the output directory when it lives inside
1595/// the source.
1596///
1597/// The output case is not a corner case — `orgo build . -o _site` is the obvious
1598/// thing to type, and without this the build copies its own output back into itself,
1599/// growing `_site/_site/_site/…` on every run.
1600fn excluded_dirs(src: &Utf8Path, config: &Config, out: Option<&Utf8Path>) -> Vec<Utf8PathBuf> {
1601 let mut dirs = vec![config.templates.dir.clone()];
1602 if let Some(out) = out {
1603 // Compare canonicalized paths so `.`, `./x` and an absolute path all agree.
1604 // The output may not exist yet, in which case it cannot contain anything and
1605 // the textual fallback is enough.
1606 let canon = |p: &Utf8Path| -> Option<Utf8PathBuf> {
1607 std::fs::canonicalize(p)
1608 .ok()
1609 .and_then(|p| Utf8PathBuf::from_path_buf(p).ok())
1610 };
1611 match (canon(src), canon(out)) {
1612 (Some(src_abs), Some(out_abs)) => {
1613 if let Ok(rel) = out_abs.strip_prefix(&src_abs) {
1614 if !rel.as_str().is_empty() {
1615 dirs.push(rel.to_owned());
1616 }
1617 }
1618 }
1619 _ => {
1620 if let Ok(rel) = out.strip_prefix(src) {
1621 if !rel.as_str().is_empty() {
1622 dirs.push(rel.to_owned());
1623 }
1624 }
1625 }
1626 }
1627 }
1628 dirs
1629}
1630
1631/// Is this path component a dot-entry that must not be published?
1632///
1633/// Dot-directories are excluded because a source directory is very often a git repository,
1634/// and publishing `.git` — or `.env` — leaks a project's entire history alongside its
1635/// homepage. `.well-known` is the exception the web actually defines (RFC 8615): it holds
1636/// `security.txt`, ACME challenges, and other files whose entire purpose is to be served.
1637/// Excluding it is how a deploy quietly deletes a site's security contact.
1638fn is_hidden(component: &str) -> bool {
1639 component.starts_with('.')
1640 && component != "."
1641 && component != ".."
1642 && component != WELL_KNOWN
1643}
1644
1645/// The one dot-directory the web expects to be published.
1646const WELL_KNOWN: &str = ".well-known";
1647
1648/// Is this directory a site orgo built earlier?
1649///
1650/// Every build writes `.orgo-cache.json` into its output directory, so a directory in the
1651/// source carrying one is output rather than content someone wrote. `excluded_dirs` only
1652/// covers an output directory nested in the source; without this, building into the
1653/// source (`-o _site`) and then previewing elsewhere (`-o /tmp/preview`) copies the whole
1654/// first site into the second, one asset at a time.
1655fn is_build_output(entry: &walkdir::DirEntry) -> bool {
1656 entry.file_type().is_dir() && entry.path().join(".orgo-cache.json").is_file()
1657}
1658
1659/// Is this source-relative path excluded from discovery?
1660///
1661/// Dot-entries are skipped wholesale, and the reason is safety rather than tidiness: a
1662/// source directory is very often a git repository, and publishing `.git` — or `.env` —
1663/// is a way to leak a project's entire history alongside its homepage.
1664fn is_excluded(rel: &Utf8Path, skip_dirs: &[Utf8PathBuf]) -> bool {
1665 if rel.components().any(|c| is_hidden(c.as_str())) {
1666 return true;
1667 }
1668 skip_dirs
1669 .iter()
1670 .any(|dir| !dir.as_str().is_empty() && rel.starts_with(dir))
1671}
1672
1673/// Does this output path sit at the site root?
1674///
1675/// The nav is the site's global chrome, and listing *every* page in it makes an `n`-page
1676/// site emit `n²` nav links — 1,790 pages produced 284 MB of output, most of it nav. A
1677/// nav is a map of the site's top level, not an index of its contents, so it is built
1678/// from root-level pages only. Section pages reach their siblings through that section's
1679/// own landing page.
1680fn is_top_level(output: &Utf8Path) -> bool {
1681 output.parent().is_none_or(|p| p.as_str().is_empty())
1682}
1683
1684/// Everything a template can know about one page. Every `#+KEYWORD:` is passed through
1685/// under its lowercased name, so a template can use metadata this crate has never heard
1686/// of without the crate needing a release to support it.
1687fn page_context(doc: &Document, output: &Utf8Path, config: &Config) -> PageContext {
1688 let words = document_text(&doc.root).split_whitespace().count();
1689 let keyword = |name: &str| {
1690 doc.keywords
1691 .entries
1692 .iter()
1693 .find(|(k, _)| k.eq_ignore_ascii_case(name))
1694 .map(|(_, v)| v.clone())
1695 };
1696 PageContext {
1697 title: page_title(doc),
1698 url: output.to_string(),
1699 source: doc.source_path.to_string(),
1700 date_iso: keyword("DATE").as_deref().and_then(iso_date),
1701 year: keyword("DATE")
1702 .as_deref()
1703 .and_then(iso_date)
1704 .map(|d| d[..4].to_string()),
1705 date: keyword("DATE"),
1706 excerpt: keyword("DESCRIPTION")
1707 .filter(|d| !d.trim().is_empty())
1708 .or_else(|| first_paragraph(&doc.root))
1709 .unwrap_or_default(),
1710 content: None,
1711 word_count: words,
1712 reading_time: words.div_ceil(WORDS_PER_MINUTE).max(usize::from(words > 0)),
1713 toc: if option_enabled(&doc.keywords, "toc", config.html.toc) {
1714 table_of_contents(&doc.root)
1715 } else {
1716 Vec::new()
1717 },
1718 tags: keyword("FILETAGS")
1719 .unwrap_or_default()
1720 .split(':')
1721 .filter(|t| !t.trim().is_empty())
1722 .map(|t| t.trim().to_string())
1723 .collect(),
1724 keywords: doc
1725 .keywords
1726 .entries
1727 .iter()
1728 .map(|(k, v)| (k.to_lowercase(), v.clone()))
1729 .collect(),
1730 }
1731}
1732
1733fn page_title(doc: &Document) -> String {
1734 doc.keywords
1735 .entries
1736 .iter()
1737 .find(|(k, _)| k.eq_ignore_ascii_case("TITLE"))
1738 .map(|(_, v)| v.clone())
1739 .unwrap_or_else(|| {
1740 doc.source_path
1741 .file_stem()
1742 .unwrap_or("untitled")
1743 .to_string()
1744 })
1745}