krz/orgo
Lightning fast org-mode static site generator.
clone: git clone https://gitbay.org/krz/orgo.git
f828151d5f68e4e7ad93e30c67c820225e50af6b
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-08-11T04:31:36Z
Cargo.lock | 54 ++++++++++++++++++++- Cargo.toml | 3 +- README.md | 66 ++++++++++++++++++++++--- src/main.rs | 8 ++- src/model.rs | 16 ++++++ src/parser.rs | 134 +++++++++++++++++++++++++++++++++++++++++---------- src/site.rs | 129 +++++++++++++++++++++++++++++++++++-------------- tests/constructs.rs | 99 +++++++++++++++++++++++++++++++++++++ tests/incremental.rs | 67 ++++++++++++++++++++++++++ 9 files changed, 504 insertions(+), 72 deletions(-) @@ -258,12 +258,43 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -538,7 +569,7 @@ dependencies = [ [[package]] name = "org-ssg" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "blake3", @@ -547,6 +578,7 @@ dependencies = [ "clap", "insta", "minijinja", + "rayon", "serde", "serde_json", "syntect", @@ -618,6 +650,26 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex-syntax" version = "0.8.11" @@ -1,6 +1,6 @@ [package] name = "org-ssg" -version = "0.4.0" +version = "0.5.0" edition = "2021" description = "Org-mode static site generator that renders the org element tree straight to HTML" license = "MIT" @@ -30,6 +30,7 @@ blake3 = "1" clap = { version = "4", features = ["derive"] } anyhow = "1" thiserror = "2" +rayon = "1.12.0" [dev-dependencies] insta = { version = "1", features = ["json"] } @@ -71,7 +71,7 @@ all-of-org. Phase 0 checked this line against a real 179-file corpus and found i | 4 | Rendering to HTML — tree walk, tables, footnote two-pass, minijinja templating, syntect highlighting | done | | 5 | Link resolution + symbol table (INDEX + RESOLVE, used-target list, broken-link reporting) | done | | 6 | Incremental build layer (hashing, dep graph, invalidation) done; `watch` is a simple poll loop | done | -| 7 | Hardening: rayon parallelism, error locations in parse diagnostics | todo | +| **7** | **Hardening: rayon parallelism, error locations in parse diagnostics** | **done** | ### v0.2 in / out @@ -164,9 +164,9 @@ excluded construct to a specific degradation: babel is never executed *and* a ch as literal text; drawers other than PROPERTIES are captured and dropped; unmodelled block types keep their content verbatim. -**Still out at v0.4:** rayon parallelism; parse errors carrying source locations; `#+TODO:` -per-file keyword sequences; planning lines (`SCHEDULED:`/`DEADLINE:`), which render as -ordinary paragraphs; fixed-width `: ` lines; and the `watch` fs-notify integration. +**Still out:** `#+TODO:` per-file keyword sequences; planning lines +(`SCHEDULED:`/`DEADLINE:`), which render as ordinary paragraphs; fixed-width `: ` lines; +and the `watch` fs-notify integration. ## Phase 0: the corpus audit and the Emacs oracle @@ -241,6 +241,59 @@ corrupted (it trimmed each of syntect's per-token text runs, turning `def greet` were measurement artifacts. A differential harness is a piece of software like any other, and the first divergences it reports are usually its own. +## Phase 7: hardening + +### Parse diagnostics (`file:line: message`) + +The parser's contract is that it always returns a document — out-of-scope and malformed +constructs degrade rather than crash. The gap was that they degraded *silently*, and in the +worst cases the degradation is severe: an unterminated `#+BEGIN_SRC` reads the rest of the +file as block content, and an unterminated drawer does the same but renders to nothing, so +one missing line deletes most of a page from a build that reports success. + +`parse` now returns `Document::diagnostics`, each carrying a 1-based source line, and the +build prints them as `file:line: message`. `--strict` turns them (and unresolved links) into +a non-zero exit. Line numbers are threaded as an absolute offset through every nested parse, +so a block inside a list item inside a section still reports its real file line — there is a +test for exactly that, because reconstructed and re-indented nested slices are precisely +where an off-by-N hides. The 179-file corpus produces zero diagnostics. + +### Parallelism + +PARSE, RESOLVE and RENDER/EMIT run under rayon. PARSE is a pure function of one file's bytes +and RESOLVE only reads the shared symbol table, which is what makes both safe to parallelize +at all; INDEX stays sequential. + +| corpus | before | after | speedup | +|---|---|---|---| +| 179 files (real) | 0.23s | 0.07s | 3.3× | +| 1,790 files (10× copy) | 3.98s | 0.82s | 4.9× | + +Measured on 12 cores. `RAYON_NUM_THREADS=1` reproduces the old 3.98s exactly, so the gain is +parallelism rather than incidental change, and the output is byte-identical to the sequential +build across the whole corpus. + +**Parallelism must not be observable in the result.** `par_iter().collect()` preserves input +order, so the emitted bytes are unaffected — but the build *report* is the fragile half: +pushing to `rendered`/`skipped` from inside the parallel pass would order them by thread +scheduling, producing a non-deterministic report over a deterministic site. The parallel pass +therefore returns only what was written, and the report is assembled sequentially afterwards. +`parallel_builds_are_deterministic_in_output_and_report_order` holds that line, and it was +verified by reintroducing the bug and watching it fail. + +### The real scaling limit is not the CPU + +Going 10× on corpus size cost 17× in time before parallelism, which is superlinear — and +parallelism moves that constant without fixing it. The cause is the nav bar: it lists **every** +page, so an *n*-page site emits *n*² nav links. At 1,790 pages each page carries 1,799 links +and the output is 284 MB, against 5.5 MB for the 179-page corpus — 52× the bytes for 10× the +input. Even at the real corpus size this is already visible: 18 KB pages whose nav dwarfs the +prose, where the live site's nav has about six links. + +This is a template and configuration question rather than a bug — *which* pages belong in a +nav is a decision this project has not made yet — so it is recorded here rather than guessed +at. Until it is made, a build's cost is dominated by chrome nobody asked for. + **From v0.1 (core subset):** headings with nesting and anchors (every heading is now anchored — `:CUSTOM_ID:`/`:ID:` else a slug of its text) and trailing tags; paragraphs; plain lists (unordered + ordered) with checkboxes; source blocks; inline markup (`*bold*`, @@ -251,8 +304,9 @@ plain lists (unordered + ordered) with checkboxes; source blocks; inline markup Parser is hand-written recursive descent (not `nom`/`chumsky`/`pest` — org is line-oriented and context-sensitive, not clean CFG). Key crates: `syntect` (syntax highlighting, behind a `Highlighter` trait so tree-sitter can be swapped in later), -`minijinja` (runtime templates), `blake3` (content/cache hashing), `chrono`, -`camino`, `walkdir`, `clap`, `anyhow`/`thiserror`. `insta` for snapshot tests. +`minijinja` (runtime templates), `blake3` (content/cache hashing), `rayon` (parallel +PARSE/RESOLVE/RENDER), `chrono`, `camino`, `walkdir`, `clap`, `anyhow`/`thiserror`. +`insta` for snapshot tests, and `emacs --batch` — optional, and only for the oracle. ## Build & test @@ -72,14 +72,15 @@ fn main() -> Result<()> { let opts = BuildOptions { no_cache, strict }; let report = build_site(&input, &out, &opts)?; println!( - "built {} page(s) ({} rendered, {} cached), copied {} asset(s) from {} -> {} ({} unresolved link(s))", + "built {} page(s) ({} rendered, {} cached), copied {} asset(s) from {} -> {} ({} unresolved link(s), {} diagnostic(s))", report.pages.len(), report.rendered.len(), report.skipped.len(), report.assets.len(), input, out, - report.broken.len() + report.broken.len(), + report.diagnostics.len() ); } else { let output = output.unwrap_or_else(|| input.with_extension("html")); @@ -170,6 +171,9 @@ fn build_file(input: &Utf8Path, output: &Utf8Path) -> Result<()> { let source = fs::read_to_string(input) .with_context(|| format!("reading source file {input}"))?; let document = parse(input, &source).with_context(|| format!("parsing {input}"))?; + for d in &document.diagnostics { + eprintln!("warning: {input}:{}: {}", d.line, d.message); + } let title = document .keywords @@ -36,6 +36,19 @@ pub struct TodoKeyword { pub done: bool, } +/// A problem found while parsing, carrying the 1-based source line it was found on. +/// +/// Diagnostics are warnings, not errors: the parser's contract is that it always returns +/// a document (spec §1 — out-of-scope constructs degrade, never crash). What a warning +/// buys is that degrading stops being *silent*, which matters most exactly where the +/// damage is largest — an unterminated `#+BEGIN_SRC` swallows the rest of the file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Diagnostic { + /// 1-based line number in the source file. + pub line: usize, + pub message: String, +} + /// One source file → one Document. This is the unit of parsing and caching (spec §2.3). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Document { @@ -44,6 +57,9 @@ pub struct Document { pub keywords: Keywords, /// Pre-first-heading content plus child headings. pub root: Section, + /// Non-fatal problems found while parsing this file. + #[serde(default)] + pub diagnostics: Vec<Diagnostic>, } /// A section = content directly under a heading (or the file preamble), followed by @@ -25,8 +25,8 @@ use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use crate::model::{ BlockParams, Bullet, Checkbox, ContentHash, Document, Element, Heading, Keywords, Link, - LinkTarget, List, ListItem, ListKind, Object, Properties, Section, Table, TableRow, Timestamp, - TodoKeyword, + Diagnostic, LinkTarget, List, ListItem, ListKind, Object, Properties, Section, Table, TableRow, + Timestamp, TodoKeyword, }; #[derive(Debug, thiserror::Error)] @@ -104,6 +104,7 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> { let lines: Vec<&str> = source.lines().collect(); let classes = line_lexer(source); + let mut diagnostics: Vec<Diagnostic> = Vec::new(); let mut keywords = Keywords::default(); let mut root = Section { heading: None, @@ -135,7 +136,7 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> { } } } - root.content = parse_elements(&lines[..first]); + root.content = parse_elements(&lines[..first], 0, &mut diagnostics); } // Each heading segment runs from its own line up to (but excluding) the next heading. @@ -144,7 +145,8 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> { let end = heading_idxs.get(k + 1).copied().unwrap_or(lines.len()); let heading = parse_heading(lines[h_idx]); let level = heading.level; - let (heading, content) = parse_section_body(heading, &lines[h_idx + 1..end]); + let (heading, content) = + parse_section_body(heading, &lines[h_idx + 1..end], h_idx + 1, &mut diagnostics); flat.push(( level, Section { @@ -158,11 +160,13 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> { let mut pos = 0; root.children = build_children(&mut flat, &mut pos, 0); + diagnostics.sort_by_key(|d| d.line); Ok(Document { source_path: path.to_owned(), content_hash, keywords, root, + diagnostics, }) } @@ -320,17 +324,25 @@ fn is_tag_cluster(s: &str) -> bool { // Section body: property drawer + block content // --------------------------------------------------------------------------- -fn parse_section_body(mut heading: Heading, body: &[&str]) -> (Heading, Vec<Element>) { +fn parse_section_body( + mut heading: Heading, + body: &[&str], + base: usize, + diags: &mut Vec<Diagnostic>, +) -> (Heading, Vec<Element>) { let mut idx = 0; while idx < body.len() && body[idx].trim().is_empty() { idx += 1; } if idx < body.len() && body[idx].trim().eq_ignore_ascii_case(":PROPERTIES:") { + let opened_at = base + idx; + let mut terminated = false; idx += 1; while idx < body.len() { let t = body[idx].trim(); if t.eq_ignore_ascii_case(":END:") { idx += 1; + terminated = true; break; } if let Some((k, v)) = parse_property(t) { @@ -343,8 +355,16 @@ fn parse_section_body(mut heading: Heading, body: &[&str]) -> (Heading, Vec<Elem } idx += 1; } + if !terminated { + diags.push(Diagnostic { + line: opened_at + 1, + message: "unterminated :PROPERTIES: drawer (no :END:); the rest of the \ + section was read as properties" + .to_string(), + }); + } } - let content = parse_elements(&body[idx..]); + let content = parse_elements(&body[idx..], base + idx, diags); (heading, content) } @@ -365,7 +385,10 @@ fn parse_property(line: &str) -> Option<(String, String)> { // Block-level element builder // --------------------------------------------------------------------------- -fn parse_elements(lines: &[&str]) -> Vec<Element> { +/// Build the block elements of `lines`. `base` is the absolute 0-based index of +/// `lines[0]` in the source file, so diagnostics can name a real line number however +/// deeply nested the construct is. +fn parse_elements(lines: &[&str], base: usize, diags: &mut Vec<Diagnostic>) -> Vec<Element> { let mut out = Vec::new(); // Affiliated keywords (`#+CAPTION:` and friends) belong to the element that follows // them, so they are held aside until that element is built. @@ -393,7 +416,7 @@ fn parse_elements(lines: &[&str]) -> Vec<Element> { i += 1; continue; } - let (element, next) = parse_one_element(lines, i); + let (element, next) = parse_one_element(lines, i, base, diags); i = next; if std::mem::take(&mut drop_next) { affiliated.clear(); @@ -409,17 +432,22 @@ fn parse_elements(lines: &[&str]) -> Vec<Element> { /// Build the single element starting at `lines[start]`, returning it with the index of /// the first line past it. `None` means the lines were consumed without producing an /// element. `start` is guaranteed non-blank and not an affiliated keyword. -fn parse_one_element(lines: &[&str], start: usize) -> (Option<Element>, usize) { +fn parse_one_element( + lines: &[&str], + start: usize, + base: usize, + diags: &mut Vec<Diagnostic>, +) -> (Option<Element>, usize) { let line = lines[start]; if let Some(text) = comment_text(line) { return (Some(Element::Comment(text)), start + 1); } if let Some((kind, after)) = block_begin(line) { - let (el, next) = parse_block(lines, start, &kind, &after); + let (el, next) = parse_block(lines, start, &kind, &after, base, diags); return (Some(el), next); } if let Some(name) = drawer_begin_name(line) { - let (el, next) = parse_drawer(lines, start, name); + let (el, next) = parse_drawer(lines, start, name, base, diags); return (Some(el), next); } if is_rule(line) { @@ -434,7 +462,7 @@ fn parse_one_element(lines: &[&str], start: usize) -> (Option<Element>, usize) { return (Some(def), next); } if is_list_item(line.trim_start()).is_some() { - let (list, next) = parse_list(lines, start); + let (list, next) = parse_list(lines, start, base, diags); return (Some(Element::List(list)), next); } // Paragraph: gather consecutive soft-wrapped text lines. @@ -449,8 +477,19 @@ fn parse_one_element(lines: &[&str], start: usize) -> (Option<Element>, usize) { i += 1; } if para.is_empty() { - // `is_structural` said this line begins a construct that no branch above claimed - // (a stray `#+END_`); skip it rather than looping forever. + // `is_structural` said this line begins a construct that no branch above claimed. + // In practice that is a stray `#+END_`: a block terminator with nothing open. + // Skip it rather than looping forever, but say so — it usually means a `#+BEGIN_` + // above it is misspelled, and silence would leave the author hunting. + if is_block_end(line) { + diags.push(Diagnostic { + line: base + start + 1, + message: format!( + "stray `{}` with no matching `#+BEGIN_`", + line.split_whitespace().next().unwrap_or("#+END_") + ), + }); + } return (None, start + 1); } (Some(Element::Paragraph(inline(¶.join(" ")))), i) @@ -478,13 +517,34 @@ fn is_structural(line: &str) -> bool { /// Consume `#+BEGIN_<KIND> … #+END_<KIND>`. Matching is on the *specific* kind so a /// source block can sit inside a quote block; an unterminated block runs to end of /// input rather than failing. -fn parse_block(lines: &[&str], start: usize, kind: &str, after: &str) -> (Element, usize) { +fn parse_block( + lines: &[&str], + start: usize, + kind: &str, + after: &str, + base: usize, + diags: &mut Vec<Diagnostic>, +) -> (Element, usize) { let mut inner: Vec<&str> = Vec::new(); let mut j = start + 1; while j < lines.len() && !is_block_end_of(lines[j], kind) { inner.push(lines[j]); j += 1; } + if j >= lines.len() { + // Everything to the end of input was swallowed by the block. This is the single + // most destructive malformation in org: one missing line silently deletes the + // rest of the document from the output. + diags.push(Diagnostic { + line: base + start + 1, + message: format!( + "unterminated `#+BEGIN_{}` block (no `#+END_{}`); \ + everything to the end of the file was read as block content", + kind.to_ascii_uppercase(), + kind.to_ascii_uppercase() + ), + }); + } let next = if j < lines.len() { j + 1 } else { j }; let element = match kind.to_ascii_uppercase().as_str() { "SRC" => { @@ -496,8 +556,8 @@ fn parse_block(lines: &[&str], start: usize, kind: &str, after: &str) -> (Elemen } } "EXAMPLE" => Element::ExampleBlock(inner.join("\n")), - "QUOTE" => Element::QuoteBlock(parse_elements(&inner)), - "CENTER" => Element::CenterBlock(parse_elements(&inner)), + "QUOTE" => Element::QuoteBlock(parse_elements(&inner, base + start + 1, diags)), + "CENTER" => Element::CenterBlock(parse_elements(&inner, base + start + 1, diags)), "EXPORT" => Element::ExportBlock { backend: after.split_whitespace().next().unwrap_or("").to_string(), raw: inner.join("\n"), @@ -512,18 +572,35 @@ fn parse_block(lines: &[&str], start: usize, kind: &str, after: &str) -> (Elemen /// `:NAME:` … `:END:` at block level. A PROPERTIES drawer directly under a heading is /// consumed by [`parse_section_body`]; anything reaching here is a generic drawer, /// which the renderer drops (README §OUT). -fn parse_drawer(lines: &[&str], start: usize, name: String) -> (Element, usize) { +fn parse_drawer( + lines: &[&str], + start: usize, + name: String, + base: usize, + diags: &mut Vec<Diagnostic>, +) -> (Element, usize) { let mut inner: Vec<&str> = Vec::new(); let mut j = start + 1; while j < lines.len() && !lines[j].trim().eq_ignore_ascii_case(":END:") { inner.push(lines[j]); j += 1; } + if j >= lines.len() { + // Drawers render to nothing, so an unterminated one deletes the rest of the file + // from the output just as thoroughly as an unterminated block — and more quietly. + diags.push(Diagnostic { + line: base + start + 1, + message: format!( + "unterminated `:{name}:` drawer (no `:END:`); everything to the end of \ + the file was read as drawer content and will not be rendered" + ), + }); + } let next = if j < lines.len() { j + 1 } else { j }; ( Element::Drawer { name, - content: parse_elements(&inner), + content: parse_elements(&inner, base + start + 1, diags), }, next, ) @@ -698,8 +775,13 @@ fn parse_footnote_def( /// column; everything indented further is that item's body, re-parsed as block content — /// which is what makes lists nest. A single blank line does not end a list, but a blank /// line followed by anything that is not a sibling bullet does. -fn parse_list(lines: &[&str], start: usize) -> (List, usize) { - let base = indent_of(lines[start]); +fn parse_list( + lines: &[&str], + start: usize, + base: usize, + diags: &mut Vec<Diagnostic>, +) -> (List, usize) { + let base_indent = indent_of(lines[start]); let family = bullet_family(&is_list_item(lines[start].trim_start()).expect("list item")); // A list is a description list when its FIRST item carries a `::` term separator. let kind = match (&family, split_term(item_text(lines[start].trim_start()))) { @@ -716,7 +798,7 @@ fn parse_list(lines: &[&str], start: usize) -> (List, usize) { while j < lines.len() && lines[j].trim().is_empty() { j += 1; } - if j >= lines.len() || indent_of(lines[j]) != base { + if j >= lines.len() || indent_of(lines[j]) != base_indent { break; } let Some(bullet) = is_list_item(lines[j].trim_start()) else { @@ -747,14 +829,14 @@ fn parse_list(lines: &[&str], start: usize) -> (List, usize) { while k < lines.len() && lines[k].trim().is_empty() { k += 1; } - if k < lines.len() && indent_of(lines[k]) > base { + if k < lines.len() && indent_of(lines[k]) > base_indent { body.resize(body.len() + (k - i), String::new()); i = k; continue; } break; } - if indent_of(lines[i]) <= base { + if indent_of(lines[i]) <= base_indent { break; } body.push(lines[i].to_string()); @@ -765,7 +847,9 @@ fn parse_list(lines: &[&str], start: usize) -> (List, usize) { bullet, checkbox, term, - content: parse_elements(&dedent(&body)), + // The item body starts at the bullet line, so `base + j` is exact even after + // the body has been dedented into fresh strings. + content: parse_elements(&dedent(&body), base + j, diags), }); } (List { kind, items }, i) @@ -14,6 +14,7 @@ use std::fs; use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; +use rayon::prelude::*; use walkdir::WalkDir; use crate::incremental::{ @@ -21,7 +22,7 @@ use crate::incremental::{ template_hash, BuildConfig, DepGraph, Hash, Manifest, PageRecord, CACHE_FORMAT_VERSION, }; use crate::index::{document_targets, SymbolTable, TargetId}; -use crate::model::{ContentHash, Document}; +use crate::model::{ContentHash, Diagnostic, Document}; use crate::parser::parse; use crate::render::{render, syntax_css, Html, SyntectHighlighter}; use crate::resolve::resolve; @@ -62,6 +63,26 @@ pub struct SiteReport { pub assets: Vec<Utf8PathBuf>, /// Unresolved internal links: `(page, target)`. Warnings, not failures (spec §4.3.4). pub broken: Vec<(Utf8PathBuf, TargetId)>, + /// Parse diagnostics: `(source file, diagnostic)`, in file then line order. + pub diagnostics: Vec<(Utf8PathBuf, Diagnostic)>, +} + +impl SiteReport { + /// Every diagnostic and broken link, formatted one per line as + /// `file:line: message` — the form an editor can jump to. + pub fn warnings(&self) -> Vec<String> { + let mut out: Vec<String> = self + .diagnostics + .iter() + .map(|(path, d)| format!("{path}:{}: {}", d.line, d.message)) + .collect(); + out.extend( + self.broken + .iter() + .map(|(page, target)| format!("{page}: unresolved link {target}")), + ); + out + } } /// Everything a build needs about one page *before* the decision to render it: its @@ -75,6 +96,7 @@ struct PagePrep { used: HashSet<TargetId>, defines: HashSet<TargetId>, broken: Vec<TargetId>, + diagnostics: Vec<Diagnostic>, nav: Vec<NavItem>, } @@ -86,13 +108,18 @@ fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> { let (org_rel, _assets) = discover(src)?; // PARSE every file (relative paths keep snapshots and links machine-independent). - let mut docs: Vec<Document> = Vec::new(); - for rel in &org_rel { - let abs = src.join(rel); - let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?; - let doc = parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))?; - docs.push(doc); - } + // PARSE is a pure function of one file's bytes (spec §2.1), which is exactly the + // property that makes it safe to run in parallel. `par_iter().collect()` preserves + // input order, so the document list — and everything downstream of it — is identical + // to the sequential build regardless of how the work was scheduled. + let docs: Vec<Document> = org_rel + .par_iter() + .map(|rel| { + let abs = src.join(rel); + let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?; + parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}")) + }) + .collect::<Result<Vec<_>>>()?; // INDEX: collect every link target across the corpus. let mut symbols = SymbolTable::new(); @@ -121,8 +148,11 @@ fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> { } } - let mut pages = Vec::new(); - for doc in &docs { + // RESOLVE reads the shared symbol table and writes only into its own page's output, + // so it parallelizes for free once INDEX has finished building the table. + let pages: Vec<PagePrep> = docs + .par_iter() + .map(|doc| { let out = resolve(doc, &symbols); let used: HashSet<TargetId> = out.used_targets.iter().cloned().collect(); let broken: Vec<TargetId> = out.broken.iter().map(|b| b.target.clone()).collect(); @@ -139,18 +169,20 @@ fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> { }) .collect(); - pages.push(PagePrep { - source: doc.source_path.clone(), - output, - title: page_title(doc), - content_hash: doc.content_hash, - resolved: out.resolved, - used, - defines, - broken, - nav, - }); - } + PagePrep { + source: doc.source_path.clone(), + output, + title: page_title(doc), + content_hash: doc.content_hash, + resolved: out.resolved, + used, + defines, + broken, + diagnostics: doc.diagnostics.clone(), + nav, + } + }) + .collect(); Ok((pages, symbols)) } @@ -270,22 +302,42 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result let templater = Templater::new(); let mut report = SiteReport::default(); - for p in &preps { - for t in &p.broken { - report.broken.push((p.source.clone(), t.clone())); - } - report.pages.push(p.output.clone()); - - let dest = out.join(&p.output); - if rebuild.contains(&p.source) { + // RENDER + TEMPLATE + EMIT, in parallel. This is where a build's time actually goes + // (syntect highlighting and templating dominate), and each page writes only its own + // file, so the pages are independent. + // + // The parallel pass returns whether each page was written; the report is assembled + // sequentially afterwards from `preps` order. Pushing to the report from inside the + // parallel pass would make `rendered`/`skipped` ordering depend on thread scheduling, + // which would be a non-deterministic build report over a deterministic build. + let written: Vec<bool> = preps + .par_iter() + .map(|p| { + if !rebuild.contains(&p.source) { + // Skip: the on-disk output is already correct (spec §4.1). Leave it alone. + return Ok(false); + } + let dest = out.join(&p.output); if let Some(parent) = dest.parent() { fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?; } let html = render_page(&templater, &highlighter, p)?; fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?; + Ok(true) + }) + .collect::<Result<Vec<_>>>()?; + + for (p, was_written) in preps.iter().zip(&written) { + for t in &p.broken { + report.broken.push((p.source.clone(), t.clone())); + } + for d in &p.diagnostics { + report.diagnostics.push((p.source.clone(), d.clone())); + } + report.pages.push(p.output.clone()); + if *was_written { report.rendered.push(p.output.clone()); } else { - // Skip: the on-disk output is already correct (spec §4.1). Leave it untouched. report.skipped.push(p.output.clone()); } } @@ -322,17 +374,20 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result incremental::save_manifest(out, &manifest) .with_context(|| format!("writing cache manifest under {out}"))?; - if opts.strict && !report.broken.is_empty() { - for (page, target) in &report.broken { - eprintln!("error: {page}: unresolved link {target}"); + let warnings = report.warnings(); + if opts.strict && !warnings.is_empty() { + for w in &warnings { + eprintln!("error: {w}"); } anyhow::bail!( - "{} unresolved internal link(s) under --strict", + "{} problem(s) under --strict ({} parse diagnostic(s), {} unresolved link(s))", + warnings.len(), + report.diagnostics.len(), report.broken.len() ); } - for (page, target) in &report.broken { - eprintln!("warning: {page}: unresolved link {target}"); + for w in &warnings { + eprintln!("warning: {w}"); } Ok(report) @@ -290,3 +290,102 @@ fn include_is_not_expanded() { "`#+INCLUDE:` must not be expanded or echoed:\n{html}" ); } + +// --------------------------------------------------------------------------- +// Parse diagnostics: degrading is fine, degrading *silently* is not +// --------------------------------------------------------------------------- + +fn diagnostics(source: &str) -> Vec<String> { + let document = parse(Utf8PathBuf::from("t.org").as_path(), source).expect("parse"); + document + .diagnostics + .iter() + .map(|d| format!("{}: {}", d.line, d.message)) + .collect() +} + +/// An unterminated block swallows the rest of the file. The parser's contract is to +/// degrade rather than crash, so it still returns a document — but a silent one would +/// mean a build that reports success while deleting most of a page. +#[test] +fn unterminated_block_is_reported_with_its_line() { + let source = "#+TITLE: T\n\nIntro.\n\n* Section\n\n#+BEGIN_SRC rust\nfn main() {}\n\n* Vanishes\n"; + let found = diagnostics(source); + assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}"); + assert!( + found[0].starts_with("7: unterminated `#+BEGIN_SRC` block"), + "must name the line the block opened on: {found:?}" + ); +} + +/// The same failure mode, and quieter: drawers render to nothing, so an unterminated one +/// deletes the rest of the file without even leaving a code block behind. +#[test] +fn unterminated_drawer_is_reported_with_its_line() { + let found = diagnostics("#+TITLE: T\n\n* Head\n:LOGBOOK:\nCLOCK: x\n\n* Lost\n"); + assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}"); + assert!( + found[0].starts_with("4: unterminated `:LOGBOOK:` drawer"), + "must name the drawer and its line: {found:?}" + ); +} + +/// A stray terminator usually means the matching `#+BEGIN_` above it is misspelled. +#[test] +fn stray_block_end_is_reported_with_its_line() { + let found = diagnostics("#+TITLE: T\n\nText.\n\n#+END_SRC\n\nMore.\n"); + assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}"); + assert!( + found[0].starts_with("5: stray `#+END_SRC`"), + "must name the stray terminator and its line: {found:?}" + ); +} + +/// Line numbers must survive nesting. A block inside a list item inside a section is +/// several levels of re-parsed, re-indented, reconstructed lines away from the file, and +/// a diagnostic that points at the wrong line is worse than none. +#[test] +fn diagnostic_lines_survive_nesting() { + let source = concat!( + "#+TITLE: T\n", // 1 + "\n", // 2 + "* Section\n", // 3 + "\n", // 4 + "- an item\n", // 5 + "\n", // 6 + " #+BEGIN_SRC sh\n", // 7 + " echo hi\n", // 8 + ); + let found = diagnostics(source); + assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}"); + assert!( + found[0].starts_with("7: unterminated"), + "the line must be the real file line, not an offset into a nested slice: {found:?}" + ); +} + +/// Every fixture that is meant to be well-formed must parse without complaint — +/// otherwise the diagnostics are crying wolf on ordinary documents. +#[test] +fn well_formed_fixtures_produce_no_diagnostics() { + for name in [ + "minimal.org", + "core.org", + "elements.org", + "table.org", + "footnote.org", + "headings.org", + "lists.org", + "blocks.org", + "timestamps.org", + "images.org", + "outofscope.org", + ] { + let document = parse_fixture(name); + assert!( + document.diagnostics.is_empty(), + "{name} should parse cleanly, got {:?}", + document.diagnostics + ); + } +} @@ -294,3 +294,70 @@ fn corrupt_cache_falls_back_without_crashing() { let r = build_site(&src, &out_dir, &BuildOptions::default()).unwrap(); assert_eq!(r.rendered.len(), 2, "a corrupt cache is never a correctness dependency"); } + +/// PARSE, RESOLVE and RENDER/EMIT all run in parallel (rayon). Parallelism must not be +/// observable in the result: the emitted bytes and the *ordering* of the build report +/// have to be identical run to run, or a build stops being reproducible. +/// +/// The report ordering is the fragile half. Pushing to `rendered`/`skipped` from inside +/// the parallel pass would order them by thread scheduling, giving a non-deterministic +/// report over a deterministic site — so the report is assembled sequentially afterwards, +/// and this test is what holds that line. Enough pages to make a race likely if one exists. +#[test] +fn parallel_builds_are_deterministic_in_output_and_report_order() { + let root = tmpdir("parallel"); + let src = root.join("src"); + std::fs::create_dir_all(src.join("deep")).unwrap(); + + for i in 0..40 { + // Cross-link every page to its neighbour so RESOLVE has real work, and give each + // a source block so RENDER does too. + let body = format!( + "#+TITLE: Page {i}\n#+SLUG: page-{i}\n\n\ + See [[#anchor-{next}][the next page]].\n\n\ + * Heading {i}\n:PROPERTIES:\n:CUSTOM_ID: anchor-{i}\n:END:\n\n\ + #+BEGIN_SRC rust\nfn page_{i}() -> u32 {{ {i} }}\n#+END_SRC\n", + next = (i + 1) % 40 + ); + let dir = if i % 3 == 0 { src.join("deep") } else { src.clone() }; + std::fs::write(dir.join(format!("p{i}.org")), body).unwrap(); + } + + let build = |out: &Utf8PathBuf| { + build_site( + &src, + out, + &BuildOptions { + no_cache: true, + strict: false, + }, + ) + .unwrap() + }; + + let first_out = root.join("first"); + let first = build(&first_out); + assert_eq!(first.rendered.len(), 40, "every page renders"); + + for _ in 0..3 { + let out = tmpdir("parallel-again").join("out"); + let again = build(&out); + assert_eq!( + first.pages, again.pages, + "page ordering in the report must be deterministic" + ); + assert_eq!( + first.rendered, again.rendered, + "rendered ordering in the report must be deterministic" + ); + assert_eq!( + first.skipped, again.skipped, + "skipped ordering in the report must be deterministic" + ); + assert_eq!( + output_files(&first_out), + output_files(&out), + "emitted bytes must be identical across runs" + ); + } +}