krz/orgo

Lightning fast org-mode static site generator. fast go org-mode static-site-generator

Commit f828151d5f

f828151d5f68e4e7ad93e30c67c820225e50af6b

parent: ba4152c75e

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-11 04:31 UTC

Phase 7: parse diagnostics with source locations, and rayon parallelism

Parse diagnostics:
- The parser always returns a document — malformed constructs degrade rather than
  crash. The gap was that they degraded silently, and the worst cases are severe: an
  unterminated #+BEGIN_SRC reads the rest of the file as block content, and an
  unterminated drawer does the same but renders to nothing, so one missing line deletes
  most of a page from a build that reports success.
- parse now returns Document::diagnostics, each carrying a 1-based source line, for
  unterminated blocks, unterminated drawers (including :PROPERTIES:), and stray #+END_
  with nothing open. The build prints file:line: message; --strict makes them, and
  unresolved links, a non-zero exit.
- Line numbers are threaded as an absolute offset through every nested parse, so a
  block inside a list item inside a section reports its real file line. There is a test
  for exactly that, because reconstructed and re-indented nested slices are where an
  off-by-N hides. Threading the offset surfaced a shadowing bug: parse_list already had
  a local `base` for the indent column, so the line offset was silently shadowed;
  renamed to base_indent.
- The 179-file corpus produces zero diagnostics.

Parallelism (rayon over PARSE, RESOLVE and RENDER/EMIT):
- 179 files: 0.23s -> 0.07s. 1,790 files: 3.98s -> 0.82s (4.9x on 12 cores).
  RAYON_NUM_THREADS=1 reproduces the old 3.98s exactly, so the gain is parallelism and
  not incidental change, and output is byte-identical to the sequential build across
  the whole corpus.
- PARSE is pure over one file's bytes and RESOLVE only reads the shared symbol table,
  which is what makes them safe to parallelize; INDEX stays sequential.
- Parallelism must not be observable in the result. par_iter().collect() preserves
  order so the bytes are unaffected, but the report is the fragile half: pushing to
  rendered/skipped from inside the parallel pass would order them by thread scheduling,
  a non-deterministic report over a deterministic site. The parallel pass returns only
  what was written and the report is assembled sequentially after. The new determinism
  test was verified by reintroducing that bug and watching it fail.

Measured but deliberately not fixed: the build is O(n^2) because the nav bar lists every
page. At 1,790 pages each page carries 1,799 nav links and the output is 284 MB against
5.5 MB for 179 pages — 52x the bytes for 10x the input. Parallelism moves that constant
without fixing it. Which pages belong in a nav is a product decision this project has
not made, so it is recorded in the README rather than guessed at.

Layout: unified · split

Cargo.lock +53 −1
@@ -258,12 +258,43 @@ dependencies = [
258258 "cfg-if",
259259]
260260
261[[package]]
262name = "crossbeam-deque"
263version = "0.8.7"
264source = "registry+https://github.com/rust-lang/crates.io-index"
265checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
266dependencies = [
267 "crossbeam-epoch",
268 "crossbeam-utils",
269]
270
271[[package]]
272name = "crossbeam-epoch"
273version = "0.9.20"
274source = "registry+https://github.com/rust-lang/crates.io-index"
275checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
276dependencies = [
277 "crossbeam-utils",
278]
279
280[[package]]
281name = "crossbeam-utils"
282version = "0.8.22"
283source = "registry+https://github.com/rust-lang/crates.io-index"
284checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
285
261286[[package]]
262287name = "deranged"
263288version = "0.5.8"
264289source = "registry+https://github.com/rust-lang/crates.io-index"
265290checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
266291
292[[package]]
293name = "either"
294version = "1.17.0"
295source = "registry+https://github.com/rust-lang/crates.io-index"
296checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
297
267298[[package]]
268299name = "encode_unicode"
269300version = "1.0.0"
@@ -538,7 +569,7 @@ dependencies = [
538569
539570[[package]]
540571name = "org-ssg"
541version = "0.4.0"
572version = "0.5.0"
542573dependencies = [
543574 "anyhow",
544575 "blake3",
@@ -547,6 +578,7 @@ dependencies = [
547578 "clap",
548579 "insta",
549580 "minijinja",
581 "rayon",
550582 "serde",
551583 "serde_json",
552584 "syntect",
@@ -618,6 +650,26 @@ version = "6.0.0"
618650source = "registry+https://github.com/rust-lang/crates.io-index"
619651checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
620652
653[[package]]
654name = "rayon"
655version = "1.12.0"
656source = "registry+https://github.com/rust-lang/crates.io-index"
657checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
658dependencies = [
659 "either",
660 "rayon-core",
661]
662
663[[package]]
664name = "rayon-core"
665version = "1.13.0"
666source = "registry+https://github.com/rust-lang/crates.io-index"
667checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
668dependencies = [
669 "crossbeam-deque",
670 "crossbeam-utils",
671]
672
621673[[package]]
622674name = "regex-syntax"
623675version = "0.8.11"
Cargo.toml +2 −1
@@ -1,6 +1,6 @@
11[package]
22name = "org-ssg"
3version = "0.4.0"
3version = "0.5.0"
44edition = "2021"
55description = "Org-mode static site generator that renders the org element tree straight to HTML"
66license = "MIT"
@@ -30,6 +30,7 @@ blake3 = "1"
3030clap = { version = "4", features = ["derive"] }
3131anyhow = "1"
3232thiserror = "2"
33rayon = "1.12.0"
3334
3435[dev-dependencies]
3536insta = { version = "1", features = ["json"] }
README.md +60 −6
@@ -71,7 +71,7 @@ all-of-org. Phase 0 checked this line against a real 179-file corpus and found i
7171| 4 | Rendering to HTML — tree walk, tables, footnote two-pass, minijinja templating, syntect highlighting | done |
7272| 5 | Link resolution + symbol table (INDEX + RESOLVE, used-target list, broken-link reporting) | done |
7373| 6 | Incremental build layer (hashing, dep graph, invalidation) done; `watch` is a simple poll loop | done |
74| 7 | Hardening: rayon parallelism, error locations in parse diagnostics | todo |
74| **7** | **Hardening: rayon parallelism, error locations in parse diagnostics** | **done** |
7575
7676### v0.2 in / out
7777
@@ -164,9 +164,9 @@ excluded construct to a specific degradation: babel is never executed *and* a ch
164164as literal text; drawers other than PROPERTIES are captured and dropped; unmodelled block
165165types keep their content verbatim.
166166
167**Still out at v0.4:** rayon parallelism; parse errors carrying source locations; `#+TODO:`
168per-file keyword sequences; planning lines (`SCHEDULED:`/`DEADLINE:`), which render as
169ordinary paragraphs; fixed-width `: ` lines; and the `watch` fs-notify integration.
167**Still out:** `#+TODO:` per-file keyword sequences; planning lines
168(`SCHEDULED:`/`DEADLINE:`), which render as ordinary paragraphs; fixed-width `: ` lines;
169and the `watch` fs-notify integration.
170170
171171## Phase 0: the corpus audit and the Emacs oracle
172172
@@ -241,6 +241,59 @@ corrupted (it trimmed each of syntect's per-token text runs, turning `def greet`
241241were measurement artifacts. A differential harness is a piece of software like any other,
242242and the first divergences it reports are usually its own.
243243
244## Phase 7: hardening
245
246### Parse diagnostics (`file:line: message`)
247
248The parser's contract is that it always returns a document — out-of-scope and malformed
249constructs degrade rather than crash. The gap was that they degraded *silently*, and in the
250worst cases the degradation is severe: an unterminated `#+BEGIN_SRC` reads the rest of the
251file as block content, and an unterminated drawer does the same but renders to nothing, so
252one missing line deletes most of a page from a build that reports success.
253
254`parse` now returns `Document::diagnostics`, each carrying a 1-based source line, and the
255build prints them as `file:line: message`. `--strict` turns them (and unresolved links) into
256a non-zero exit. Line numbers are threaded as an absolute offset through every nested parse,
257so a block inside a list item inside a section still reports its real file line — there is a
258test for exactly that, because reconstructed and re-indented nested slices are precisely
259where an off-by-N hides. The 179-file corpus produces zero diagnostics.
260
261### Parallelism
262
263PARSE, RESOLVE and RENDER/EMIT run under rayon. PARSE is a pure function of one file's bytes
264and RESOLVE only reads the shared symbol table, which is what makes both safe to parallelize
265at all; INDEX stays sequential.
266
267| corpus | before | after | speedup |
268|---|---|---|---|
269| 179 files (real) | 0.23s | 0.07s | 3.3× |
270| 1,790 files (10× copy) | 3.98s | 0.82s | 4.9× |
271
272Measured on 12 cores. `RAYON_NUM_THREADS=1` reproduces the old 3.98s exactly, so the gain is
273parallelism rather than incidental change, and the output is byte-identical to the sequential
274build across the whole corpus.
275
276**Parallelism must not be observable in the result.** `par_iter().collect()` preserves input
277order, so the emitted bytes are unaffected — but the build *report* is the fragile half:
278pushing to `rendered`/`skipped` from inside the parallel pass would order them by thread
279scheduling, producing a non-deterministic report over a deterministic site. The parallel pass
280therefore returns only what was written, and the report is assembled sequentially afterwards.
281`parallel_builds_are_deterministic_in_output_and_report_order` holds that line, and it was
282verified by reintroducing the bug and watching it fail.
283
284### The real scaling limit is not the CPU
285
286Going 10× on corpus size cost 17× in time before parallelism, which is superlinear — and
287parallelism moves that constant without fixing it. The cause is the nav bar: it lists **every**
288page, so an *n*-page site emits *n*² nav links. At 1,790 pages each page carries 1,799 links
289and the output is 284 MB, against 5.5 MB for the 179-page corpus — 52× the bytes for 10× the
290input. Even at the real corpus size this is already visible: 18 KB pages whose nav dwarfs the
291prose, where the live site's nav has about six links.
292
293This is a template and configuration question rather than a bug — *which* pages belong in a
294nav is a decision this project has not made yet — so it is recorded here rather than guessed
295at. Until it is made, a build's cost is dominated by chrome nobody asked for.
296
244297**From v0.1 (core subset):** headings with nesting and anchors (every heading is now
245298anchored — `:CUSTOM_ID:`/`:ID:` else a slug of its text) and trailing tags; paragraphs;
246299plain lists (unordered + ordered) with checkboxes; source blocks; inline markup (`*bold*`,
@@ -251,8 +304,9 @@ plain lists (unordered + ordered) with checkboxes; source blocks; inline markup
251304Parser is hand-written recursive descent (not `nom`/`chumsky`/`pest` — org is
252305line-oriented and context-sensitive, not clean CFG). Key crates: `syntect` (syntax
253306highlighting, behind a `Highlighter` trait so tree-sitter can be swapped in later),
254`minijinja` (runtime templates), `blake3` (content/cache hashing), `chrono`,
255`camino`, `walkdir`, `clap`, `anyhow`/`thiserror`. `insta` for snapshot tests.
307`minijinja` (runtime templates), `blake3` (content/cache hashing), `rayon` (parallel
308PARSE/RESOLVE/RENDER), `chrono`, `camino`, `walkdir`, `clap`, `anyhow`/`thiserror`.
309`insta` for snapshot tests, and `emacs --batch` — optional, and only for the oracle.
256310
257311## Build & test
258312
src/main.rs +6 −2
@@ -72,14 +72,15 @@ fn main() -> Result<()> {
7272 let opts = BuildOptions { no_cache, strict };
7373 let report = build_site(&input, &out, &opts)?;
7474 println!(
75 "built {} page(s) ({} rendered, {} cached), copied {} asset(s) from {} -> {} ({} unresolved link(s))",
75 "built {} page(s) ({} rendered, {} cached), copied {} asset(s) from {} -> {} ({} unresolved link(s), {} diagnostic(s))",
7676 report.pages.len(),
7777 report.rendered.len(),
7878 report.skipped.len(),
7979 report.assets.len(),
8080 input,
8181 out,
82 report.broken.len()
82 report.broken.len(),
83 report.diagnostics.len()
8384 );
8485 } else {
8586 let output = output.unwrap_or_else(|| input.with_extension("html"));
@@ -170,6 +171,9 @@ fn build_file(input: &Utf8Path, output: &Utf8Path) -> Result<()> {
170171 let source = fs::read_to_string(input)
171172 .with_context(|| format!("reading source file {input}"))?;
172173 let document = parse(input, &source).with_context(|| format!("parsing {input}"))?;
174 for d in &document.diagnostics {
175 eprintln!("warning: {input}:{}: {}", d.line, d.message);
176 }
173177
174178 let title = document
175179 .keywords
src/model.rs +16
@@ -36,6 +36,19 @@ pub struct TodoKeyword {
3636 pub done: bool,
3737}
3838
39/// A problem found while parsing, carrying the 1-based source line it was found on.
40///
41/// Diagnostics are warnings, not errors: the parser's contract is that it always returns
42/// a document (spec §1 — out-of-scope constructs degrade, never crash). What a warning
43/// buys is that degrading stops being *silent*, which matters most exactly where the
44/// damage is largest — an unterminated `#+BEGIN_SRC` swallows the rest of the file.
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
46pub struct Diagnostic {
47 /// 1-based line number in the source file.
48 pub line: usize,
49 pub message: String,
50}
51
3952/// One source file → one Document. This is the unit of parsing and caching (spec §2.3).
4053#[derive(Debug, Clone, Serialize, Deserialize)]
4154pub struct Document {
@@ -44,6 +57,9 @@ pub struct Document {
4457 pub keywords: Keywords,
4558 /// Pre-first-heading content plus child headings.
4659 pub root: Section,
60 /// Non-fatal problems found while parsing this file.
61 #[serde(default)]
62 pub diagnostics: Vec<Diagnostic>,
4763}
4864
4965/// A section = content directly under a heading (or the file preamble), followed by
src/parser.rs +109 −25
@@ -25,8 +25,8 @@ use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
2525
2626use crate::model::{
2727 BlockParams, Bullet, Checkbox, ContentHash, Document, Element, Heading, Keywords, Link,
28 LinkTarget, List, ListItem, ListKind, Object, Properties, Section, Table, TableRow, Timestamp,
29 TodoKeyword,
28 Diagnostic, LinkTarget, List, ListItem, ListKind, Object, Properties, Section, Table, TableRow,
29 Timestamp, TodoKeyword,
3030};
3131
3232#[derive(Debug, thiserror::Error)]
@@ -104,6 +104,7 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> {
104104 let lines: Vec<&str> = source.lines().collect();
105105 let classes = line_lexer(source);
106106
107 let mut diagnostics: Vec<Diagnostic> = Vec::new();
107108 let mut keywords = Keywords::default();
108109 let mut root = Section {
109110 heading: None,
@@ -135,7 +136,7 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> {
135136 }
136137 }
137138 }
138 root.content = parse_elements(&lines[..first]);
139 root.content = parse_elements(&lines[..first], 0, &mut diagnostics);
139140 }
140141
141142 // Each heading segment runs from its own line up to (but excluding) the next heading.
@@ -144,7 +145,8 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> {
144145 let end = heading_idxs.get(k + 1).copied().unwrap_or(lines.len());
145146 let heading = parse_heading(lines[h_idx]);
146147 let level = heading.level;
147 let (heading, content) = parse_section_body(heading, &lines[h_idx + 1..end]);
148 let (heading, content) =
149 parse_section_body(heading, &lines[h_idx + 1..end], h_idx + 1, &mut diagnostics);
148150 flat.push((
149151 level,
150152 Section {
@@ -158,11 +160,13 @@ pub fn parse(path: &Utf8Path, source: &str) -> Result<Document, ParseError> {
158160 let mut pos = 0;
159161 root.children = build_children(&mut flat, &mut pos, 0);
160162
163 diagnostics.sort_by_key(|d| d.line);
161164 Ok(Document {
162165 source_path: path.to_owned(),
163166 content_hash,
164167 keywords,
165168 root,
169 diagnostics,
166170 })
167171}
168172
@@ -320,17 +324,25 @@ fn is_tag_cluster(s: &str) -> bool {
320324// Section body: property drawer + block content
321325// ---------------------------------------------------------------------------
322326
323fn parse_section_body(mut heading: Heading, body: &[&str]) -> (Heading, Vec<Element>) {
327fn parse_section_body(
328 mut heading: Heading,
329 body: &[&str],
330 base: usize,
331 diags: &mut Vec<Diagnostic>,
332) -> (Heading, Vec<Element>) {
324333 let mut idx = 0;
325334 while idx < body.len() && body[idx].trim().is_empty() {
326335 idx += 1;
327336 }
328337 if idx < body.len() && body[idx].trim().eq_ignore_ascii_case(":PROPERTIES:") {
338 let opened_at = base + idx;
339 let mut terminated = false;
329340 idx += 1;
330341 while idx < body.len() {
331342 let t = body[idx].trim();
332343 if t.eq_ignore_ascii_case(":END:") {
333344 idx += 1;
345 terminated = true;
334346 break;
335347 }
336348 if let Some((k, v)) = parse_property(t) {
@@ -343,8 +355,16 @@ fn parse_section_body(mut heading: Heading, body: &[&str]) -> (Heading, Vec<Elem
343355 }
344356 idx += 1;
345357 }
358 if !terminated {
359 diags.push(Diagnostic {
360 line: opened_at + 1,
361 message: "unterminated :PROPERTIES: drawer (no :END:); the rest of the \
362 section was read as properties"
363 .to_string(),
364 });
365 }
346366 }
347 let content = parse_elements(&body[idx..]);
367 let content = parse_elements(&body[idx..], base + idx, diags);
348368 (heading, content)
349369}
350370
@@ -365,7 +385,10 @@ fn parse_property(line: &str) -> Option<(String, String)> {
365385// Block-level element builder
366386// ---------------------------------------------------------------------------
367387
368fn parse_elements(lines: &[&str]) -> Vec<Element> {
388/// Build the block elements of `lines`. `base` is the absolute 0-based index of
389/// `lines[0]` in the source file, so diagnostics can name a real line number however
390/// deeply nested the construct is.
391fn parse_elements(lines: &[&str], base: usize, diags: &mut Vec<Diagnostic>) -> Vec<Element> {
369392 let mut out = Vec::new();
370393 // Affiliated keywords (`#+CAPTION:` and friends) belong to the element that follows
371394 // them, so they are held aside until that element is built.
@@ -393,7 +416,7 @@ fn parse_elements(lines: &[&str]) -> Vec<Element> {
393416 i += 1;
394417 continue;
395418 }
396 let (element, next) = parse_one_element(lines, i);
419 let (element, next) = parse_one_element(lines, i, base, diags);
397420 i = next;
398421 if std::mem::take(&mut drop_next) {
399422 affiliated.clear();
@@ -409,17 +432,22 @@ fn parse_elements(lines: &[&str]) -> Vec<Element> {
409432/// Build the single element starting at `lines[start]`, returning it with the index of
410433/// the first line past it. `None` means the lines were consumed without producing an
411434/// element. `start` is guaranteed non-blank and not an affiliated keyword.
412fn parse_one_element(lines: &[&str], start: usize) -> (Option<Element>, usize) {
435fn parse_one_element(
436 lines: &[&str],
437 start: usize,
438 base: usize,
439 diags: &mut Vec<Diagnostic>,
440) -> (Option<Element>, usize) {
413441 let line = lines[start];
414442 if let Some(text) = comment_text(line) {
415443 return (Some(Element::Comment(text)), start + 1);
416444 }
417445 if let Some((kind, after)) = block_begin(line) {
418 let (el, next) = parse_block(lines, start, &kind, &after);
446 let (el, next) = parse_block(lines, start, &kind, &after, base, diags);
419447 return (Some(el), next);
420448 }
421449 if let Some(name) = drawer_begin_name(line) {
422 let (el, next) = parse_drawer(lines, start, name);
450 let (el, next) = parse_drawer(lines, start, name, base, diags);
423451 return (Some(el), next);
424452 }
425453 if is_rule(line) {
@@ -434,7 +462,7 @@ fn parse_one_element(lines: &[&str], start: usize) -> (Option<Element>, usize) {
434462 return (Some(def), next);
435463 }
436464 if is_list_item(line.trim_start()).is_some() {
437 let (list, next) = parse_list(lines, start);
465 let (list, next) = parse_list(lines, start, base, diags);
438466 return (Some(Element::List(list)), next);
439467 }
440468 // Paragraph: gather consecutive soft-wrapped text lines.
@@ -449,8 +477,19 @@ fn parse_one_element(lines: &[&str], start: usize) -> (Option<Element>, usize) {
449477 i += 1;
450478 }
451479 if para.is_empty() {
452 // `is_structural` said this line begins a construct that no branch above claimed
453 // (a stray `#+END_`); skip it rather than looping forever.
480 // `is_structural` said this line begins a construct that no branch above claimed.
481 // In practice that is a stray `#+END_`: a block terminator with nothing open.
482 // Skip it rather than looping forever, but say so — it usually means a `#+BEGIN_`
483 // above it is misspelled, and silence would leave the author hunting.
484 if is_block_end(line) {
485 diags.push(Diagnostic {
486 line: base + start + 1,
487 message: format!(
488 "stray `{}` with no matching `#+BEGIN_`",
489 line.split_whitespace().next().unwrap_or("#+END_")
490 ),
491 });
492 }
454493 return (None, start + 1);
455494 }
456495 (Some(Element::Paragraph(inline(&para.join(" ")))), i)
@@ -478,13 +517,34 @@ fn is_structural(line: &str) -> bool {
478517/// Consume `#+BEGIN_<KIND> … #+END_<KIND>`. Matching is on the *specific* kind so a
479518/// source block can sit inside a quote block; an unterminated block runs to end of
480519/// input rather than failing.
481fn parse_block(lines: &[&str], start: usize, kind: &str, after: &str) -> (Element, usize) {
520fn parse_block(
521 lines: &[&str],
522 start: usize,
523 kind: &str,
524 after: &str,
525 base: usize,
526 diags: &mut Vec<Diagnostic>,
527) -> (Element, usize) {
482528 let mut inner: Vec<&str> = Vec::new();
483529 let mut j = start + 1;
484530 while j < lines.len() && !is_block_end_of(lines[j], kind) {
485531 inner.push(lines[j]);
486532 j += 1;
487533 }
534 if j >= lines.len() {
535 // Everything to the end of input was swallowed by the block. This is the single
536 // most destructive malformation in org: one missing line silently deletes the
537 // rest of the document from the output.
538 diags.push(Diagnostic {
539 line: base + start + 1,
540 message: format!(
541 "unterminated `#+BEGIN_{}` block (no `#+END_{}`); \
542 everything to the end of the file was read as block content",
543 kind.to_ascii_uppercase(),
544 kind.to_ascii_uppercase()
545 ),
546 });
547 }
488548 let next = if j < lines.len() { j + 1 } else { j };
489549 let element = match kind.to_ascii_uppercase().as_str() {
490550 "SRC" => {
@@ -496,8 +556,8 @@ fn parse_block(lines: &[&str], start: usize, kind: &str, after: &str) -> (Elemen
496556 }
497557 }
498558 "EXAMPLE" => Element::ExampleBlock(inner.join("\n")),
499 "QUOTE" => Element::QuoteBlock(parse_elements(&inner)),
500 "CENTER" => Element::CenterBlock(parse_elements(&inner)),
559 "QUOTE" => Element::QuoteBlock(parse_elements(&inner, base + start + 1, diags)),
560 "CENTER" => Element::CenterBlock(parse_elements(&inner, base + start + 1, diags)),
501561 "EXPORT" => Element::ExportBlock {
502562 backend: after.split_whitespace().next().unwrap_or("").to_string(),
503563 raw: inner.join("\n"),
@@ -512,18 +572,35 @@ fn parse_block(lines: &[&str], start: usize, kind: &str, after: &str) -> (Elemen
512572/// `:NAME:` … `:END:` at block level. A PROPERTIES drawer directly under a heading is
513573/// consumed by [`parse_section_body`]; anything reaching here is a generic drawer,
514574/// which the renderer drops (README §OUT).
515fn parse_drawer(lines: &[&str], start: usize, name: String) -> (Element, usize) {
575fn parse_drawer(
576 lines: &[&str],
577 start: usize,
578 name: String,
579 base: usize,
580 diags: &mut Vec<Diagnostic>,
581) -> (Element, usize) {
516582 let mut inner: Vec<&str> = Vec::new();
517583 let mut j = start + 1;
518584 while j < lines.len() && !lines[j].trim().eq_ignore_ascii_case(":END:") {
519585 inner.push(lines[j]);
520586 j += 1;
521587 }
588 if j >= lines.len() {
589 // Drawers render to nothing, so an unterminated one deletes the rest of the file
590 // from the output just as thoroughly as an unterminated block — and more quietly.
591 diags.push(Diagnostic {
592 line: base + start + 1,
593 message: format!(
594 "unterminated `:{name}:` drawer (no `:END:`); everything to the end of \
595 the file was read as drawer content and will not be rendered"
596 ),
597 });
598 }
522599 let next = if j < lines.len() { j + 1 } else { j };
523600 (
524601 Element::Drawer {
525602 name,
526 content: parse_elements(&inner),
603 content: parse_elements(&inner, base + start + 1, diags),
527604 },
528605 next,
529606 )
@@ -698,8 +775,13 @@ fn parse_footnote_def(
698775/// column; everything indented further is that item's body, re-parsed as block content —
699776/// which is what makes lists nest. A single blank line does not end a list, but a blank
700777/// line followed by anything that is not a sibling bullet does.
701fn parse_list(lines: &[&str], start: usize) -> (List, usize) {
702 let base = indent_of(lines[start]);
778fn parse_list(
779 lines: &[&str],
780 start: usize,
781 base: usize,
782 diags: &mut Vec<Diagnostic>,
783) -> (List, usize) {
784 let base_indent = indent_of(lines[start]);
703785 let family = bullet_family(&is_list_item(lines[start].trim_start()).expect("list item"));
704786 // A list is a description list when its FIRST item carries a `::` term separator.
705787 let kind = match (&family, split_term(item_text(lines[start].trim_start()))) {
@@ -716,7 +798,7 @@ fn parse_list(lines: &[&str], start: usize) -> (List, usize) {
716798 while j < lines.len() && lines[j].trim().is_empty() {
717799 j += 1;
718800 }
719 if j >= lines.len() || indent_of(lines[j]) != base {
801 if j >= lines.len() || indent_of(lines[j]) != base_indent {
720802 break;
721803 }
722804 let Some(bullet) = is_list_item(lines[j].trim_start()) else {
@@ -747,14 +829,14 @@ fn parse_list(lines: &[&str], start: usize) -> (List, usize) {
747829 while k < lines.len() && lines[k].trim().is_empty() {
748830 k += 1;
749831 }
750 if k < lines.len() && indent_of(lines[k]) > base {
832 if k < lines.len() && indent_of(lines[k]) > base_indent {
751833 body.resize(body.len() + (k - i), String::new());
752834 i = k;
753835 continue;
754836 }
755837 break;
756838 }
757 if indent_of(lines[i]) <= base {
839 if indent_of(lines[i]) <= base_indent {
758840 break;
759841 }
760842 body.push(lines[i].to_string());
@@ -765,7 +847,9 @@ fn parse_list(lines: &[&str], start: usize) -> (List, usize) {
765847 bullet,
766848 checkbox,
767849 term,
768 content: parse_elements(&dedent(&body)),
850 // The item body starts at the bullet line, so `base + j` is exact even after
851 // the body has been dedented into fresh strings.
852 content: parse_elements(&dedent(&body), base + j, diags),
769853 });
770854 }
771855 (List { kind, items }, i)
src/site.rs +92 −37
@@ -14,6 +14,7 @@ use std::fs;
1414
1515use anyhow::{Context, Result};
1616use camino::{Utf8Path, Utf8PathBuf};
17use rayon::prelude::*;
1718use walkdir::WalkDir;
1819
1920use crate::incremental::{
@@ -21,7 +22,7 @@ use crate::incremental::{
2122 template_hash, BuildConfig, DepGraph, Hash, Manifest, PageRecord, CACHE_FORMAT_VERSION,
2223};
2324use crate::index::{document_targets, SymbolTable, TargetId};
24use crate::model::{ContentHash, Document};
25use crate::model::{ContentHash, Diagnostic, Document};
2526use crate::parser::parse;
2627use crate::render::{render, syntax_css, Html, SyntectHighlighter};
2728use crate::resolve::resolve;
@@ -62,6 +63,26 @@ pub struct SiteReport {
6263 pub assets: Vec<Utf8PathBuf>,
6364 /// Unresolved internal links: `(page, target)`. Warnings, not failures (spec §4.3.4).
6465 pub broken: Vec<(Utf8PathBuf, TargetId)>,
66 /// Parse diagnostics: `(source file, diagnostic)`, in file then line order.
67 pub diagnostics: Vec<(Utf8PathBuf, Diagnostic)>,
68}
69
70impl SiteReport {
71 /// Every diagnostic and broken link, formatted one per line as
72 /// `file:line: message` — the form an editor can jump to.
73 pub fn warnings(&self) -> Vec<String> {
74 let mut out: Vec<String> = self
75 .diagnostics
76 .iter()
77 .map(|(path, d)| format!("{path}:{}: {}", d.line, d.message))
78 .collect();
79 out.extend(
80 self.broken
81 .iter()
82 .map(|(page, target)| format!("{page}: unresolved link {target}")),
83 );
84 out
85 }
6586}
6687
6788/// Everything a build needs about one page *before* the decision to render it: its
@@ -75,6 +96,7 @@ struct PagePrep {
7596 used: HashSet<TargetId>,
7697 defines: HashSet<TargetId>,
7798 broken: Vec<TargetId>,
99 diagnostics: Vec<Diagnostic>,
78100 nav: Vec<NavItem>,
79101}
80102
@@ -86,13 +108,18 @@ fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> {
86108 let (org_rel, _assets) = discover(src)?;
87109
88110 // PARSE every file (relative paths keep snapshots and links machine-independent).
89 let mut docs: Vec<Document> = Vec::new();
90 for rel in &org_rel {
91 let abs = src.join(rel);
92 let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?;
93 let doc = parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))?;
94 docs.push(doc);
95 }
111 // PARSE is a pure function of one file's bytes (spec §2.1), which is exactly the
112 // property that makes it safe to run in parallel. `par_iter().collect()` preserves
113 // input order, so the document list — and everything downstream of it — is identical
114 // to the sequential build regardless of how the work was scheduled.
115 let docs: Vec<Document> = org_rel
116 .par_iter()
117 .map(|rel| {
118 let abs = src.join(rel);
119 let source = fs::read_to_string(&abs).with_context(|| format!("reading {abs}"))?;
120 parse(rel.as_path(), &source).with_context(|| format!("parsing {rel}"))
121 })
122 .collect::<Result<Vec<_>>>()?;
96123
97124 // INDEX: collect every link target across the corpus.
98125 let mut symbols = SymbolTable::new();
@@ -121,8 +148,11 @@ fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> {
121148 }
122149 }
123150
124 let mut pages = Vec::new();
125 for doc in &docs {
151 // RESOLVE reads the shared symbol table and writes only into its own page's output,
152 // so it parallelizes for free once INDEX has finished building the table.
153 let pages: Vec<PagePrep> = docs
154 .par_iter()
155 .map(|doc| {
126156 let out = resolve(doc, &symbols);
127157 let used: HashSet<TargetId> = out.used_targets.iter().cloned().collect();
128158 let broken: Vec<TargetId> = out.broken.iter().map(|b| b.target.clone()).collect();
@@ -139,18 +169,20 @@ fn prepare_pages(src: &Utf8Path) -> Result<(Vec<PagePrep>, SymbolTable)> {
139169 })
140170 .collect();
141171
142 pages.push(PagePrep {
143 source: doc.source_path.clone(),
144 output,
145 title: page_title(doc),
146 content_hash: doc.content_hash,
147 resolved: out.resolved,
148 used,
149 defines,
150 broken,
151 nav,
152 });
153 }
172 PagePrep {
173 source: doc.source_path.clone(),
174 output,
175 title: page_title(doc),
176 content_hash: doc.content_hash,
177 resolved: out.resolved,
178 used,
179 defines,
180 broken,
181 diagnostics: doc.diagnostics.clone(),
182 nav,
183 }
184 })
185 .collect();
154186
155187 Ok((pages, symbols))
156188}
@@ -270,22 +302,42 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result
270302 let templater = Templater::new();
271303 let mut report = SiteReport::default();
272304
273 for p in &preps {
274 for t in &p.broken {
275 report.broken.push((p.source.clone(), t.clone()));
276 }
277 report.pages.push(p.output.clone());
278
279 let dest = out.join(&p.output);
280 if rebuild.contains(&p.source) {
305 // RENDER + TEMPLATE + EMIT, in parallel. This is where a build's time actually goes
306 // (syntect highlighting and templating dominate), and each page writes only its own
307 // file, so the pages are independent.
308 //
309 // The parallel pass returns whether each page was written; the report is assembled
310 // sequentially afterwards from `preps` order. Pushing to the report from inside the
311 // parallel pass would make `rendered`/`skipped` ordering depend on thread scheduling,
312 // which would be a non-deterministic build report over a deterministic build.
313 let written: Vec<bool> = preps
314 .par_iter()
315 .map(|p| {
316 if !rebuild.contains(&p.source) {
317 // Skip: the on-disk output is already correct (spec §4.1). Leave it alone.
318 return Ok(false);
319 }
320 let dest = out.join(&p.output);
281321 if let Some(parent) = dest.parent() {
282322 fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?;
283323 }
284324 let html = render_page(&templater, &highlighter, p)?;
285325 fs::write(&dest, &html).with_context(|| format!("writing {dest}"))?;
326 Ok(true)
327 })
328 .collect::<Result<Vec<_>>>()?;
329
330 for (p, was_written) in preps.iter().zip(&written) {
331 for t in &p.broken {
332 report.broken.push((p.source.clone(), t.clone()));
333 }
334 for d in &p.diagnostics {
335 report.diagnostics.push((p.source.clone(), d.clone()));
336 }
337 report.pages.push(p.output.clone());
338 if *was_written {
286339 report.rendered.push(p.output.clone());
287340 } else {
288 // Skip: the on-disk output is already correct (spec §4.1). Leave it untouched.
289341 report.skipped.push(p.output.clone());
290342 }
291343 }
@@ -322,17 +374,20 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result
322374 incremental::save_manifest(out, &manifest)
323375 .with_context(|| format!("writing cache manifest under {out}"))?;
324376
325 if opts.strict && !report.broken.is_empty() {
326 for (page, target) in &report.broken {
327 eprintln!("error: {page}: unresolved link {target}");
377 let warnings = report.warnings();
378 if opts.strict && !warnings.is_empty() {
379 for w in &warnings {
380 eprintln!("error: {w}");
328381 }
329382 anyhow::bail!(
330 "{} unresolved internal link(s) under --strict",
383 "{} problem(s) under --strict ({} parse diagnostic(s), {} unresolved link(s))",
384 warnings.len(),
385 report.diagnostics.len(),
331386 report.broken.len()
332387 );
333388 }
334 for (page, target) in &report.broken {
335 eprintln!("warning: {page}: unresolved link {target}");
389 for w in &warnings {
390 eprintln!("warning: {w}");
336391 }
337392
338393 Ok(report)
tests/constructs.rs +99
@@ -290,3 +290,102 @@ fn include_is_not_expanded() {
290290 "`#+INCLUDE:` must not be expanded or echoed:\n{html}"
291291 );
292292}
293
294// ---------------------------------------------------------------------------
295// Parse diagnostics: degrading is fine, degrading *silently* is not
296// ---------------------------------------------------------------------------
297
298fn diagnostics(source: &str) -> Vec<String> {
299 let document = parse(Utf8PathBuf::from("t.org").as_path(), source).expect("parse");
300 document
301 .diagnostics
302 .iter()
303 .map(|d| format!("{}: {}", d.line, d.message))
304 .collect()
305}
306
307/// An unterminated block swallows the rest of the file. The parser's contract is to
308/// degrade rather than crash, so it still returns a document — but a silent one would
309/// mean a build that reports success while deleting most of a page.
310#[test]
311fn unterminated_block_is_reported_with_its_line() {
312 let source = "#+TITLE: T\n\nIntro.\n\n* Section\n\n#+BEGIN_SRC rust\nfn main() {}\n\n* Vanishes\n";
313 let found = diagnostics(source);
314 assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}");
315 assert!(
316 found[0].starts_with("7: unterminated `#+BEGIN_SRC` block"),
317 "must name the line the block opened on: {found:?}"
318 );
319}
320
321/// The same failure mode, and quieter: drawers render to nothing, so an unterminated one
322/// deletes the rest of the file without even leaving a code block behind.
323#[test]
324fn unterminated_drawer_is_reported_with_its_line() {
325 let found = diagnostics("#+TITLE: T\n\n* Head\n:LOGBOOK:\nCLOCK: x\n\n* Lost\n");
326 assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}");
327 assert!(
328 found[0].starts_with("4: unterminated `:LOGBOOK:` drawer"),
329 "must name the drawer and its line: {found:?}"
330 );
331}
332
333/// A stray terminator usually means the matching `#+BEGIN_` above it is misspelled.
334#[test]
335fn stray_block_end_is_reported_with_its_line() {
336 let found = diagnostics("#+TITLE: T\n\nText.\n\n#+END_SRC\n\nMore.\n");
337 assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}");
338 assert!(
339 found[0].starts_with("5: stray `#+END_SRC`"),
340 "must name the stray terminator and its line: {found:?}"
341 );
342}
343
344/// Line numbers must survive nesting. A block inside a list item inside a section is
345/// several levels of re-parsed, re-indented, reconstructed lines away from the file, and
346/// a diagnostic that points at the wrong line is worse than none.
347#[test]
348fn diagnostic_lines_survive_nesting() {
349 let source = concat!(
350 "#+TITLE: T\n", // 1
351 "\n", // 2
352 "* Section\n", // 3
353 "\n", // 4
354 "- an item\n", // 5
355 "\n", // 6
356 " #+BEGIN_SRC sh\n", // 7
357 " echo hi\n", // 8
358 );
359 let found = diagnostics(source);
360 assert_eq!(found.len(), 1, "exactly one diagnostic: {found:?}");
361 assert!(
362 found[0].starts_with("7: unterminated"),
363 "the line must be the real file line, not an offset into a nested slice: {found:?}"
364 );
365}
366
367/// Every fixture that is meant to be well-formed must parse without complaint —
368/// otherwise the diagnostics are crying wolf on ordinary documents.
369#[test]
370fn well_formed_fixtures_produce_no_diagnostics() {
371 for name in [
372 "minimal.org",
373 "core.org",
374 "elements.org",
375 "table.org",
376 "footnote.org",
377 "headings.org",
378 "lists.org",
379 "blocks.org",
380 "timestamps.org",
381 "images.org",
382 "outofscope.org",
383 ] {
384 let document = parse_fixture(name);
385 assert!(
386 document.diagnostics.is_empty(),
387 "{name} should parse cleanly, got {:?}",
388 document.diagnostics
389 );
390 }
391}
tests/incremental.rs +67
@@ -294,3 +294,70 @@ fn corrupt_cache_falls_back_without_crashing() {
294294 let r = build_site(&src, &out_dir, &BuildOptions::default()).unwrap();
295295 assert_eq!(r.rendered.len(), 2, "a corrupt cache is never a correctness dependency");
296296}
297
298/// PARSE, RESOLVE and RENDER/EMIT all run in parallel (rayon). Parallelism must not be
299/// observable in the result: the emitted bytes and the *ordering* of the build report
300/// have to be identical run to run, or a build stops being reproducible.
301///
302/// The report ordering is the fragile half. Pushing to `rendered`/`skipped` from inside
303/// the parallel pass would order them by thread scheduling, giving a non-deterministic
304/// report over a deterministic site — so the report is assembled sequentially afterwards,
305/// and this test is what holds that line. Enough pages to make a race likely if one exists.
306#[test]
307fn parallel_builds_are_deterministic_in_output_and_report_order() {
308 let root = tmpdir("parallel");
309 let src = root.join("src");
310 std::fs::create_dir_all(src.join("deep")).unwrap();
311
312 for i in 0..40 {
313 // Cross-link every page to its neighbour so RESOLVE has real work, and give each
314 // a source block so RENDER does too.
315 let body = format!(
316 "#+TITLE: Page {i}\n#+SLUG: page-{i}\n\n\
317 See [[#anchor-{next}][the next page]].\n\n\
318 * Heading {i}\n:PROPERTIES:\n:CUSTOM_ID: anchor-{i}\n:END:\n\n\
319 #+BEGIN_SRC rust\nfn page_{i}() -> u32 {{ {i} }}\n#+END_SRC\n",
320 next = (i + 1) % 40
321 );
322 let dir = if i % 3 == 0 { src.join("deep") } else { src.clone() };
323 std::fs::write(dir.join(format!("p{i}.org")), body).unwrap();
324 }
325
326 let build = |out: &Utf8PathBuf| {
327 build_site(
328 &src,
329 out,
330 &BuildOptions {
331 no_cache: true,
332 strict: false,
333 },
334 )
335 .unwrap()
336 };
337
338 let first_out = root.join("first");
339 let first = build(&first_out);
340 assert_eq!(first.rendered.len(), 40, "every page renders");
341
342 for _ in 0..3 {
343 let out = tmpdir("parallel-again").join("out");
344 let again = build(&out);
345 assert_eq!(
346 first.pages, again.pages,
347 "page ordering in the report must be deterministic"
348 );
349 assert_eq!(
350 first.rendered, again.rendered,
351 "rendered ordering in the report must be deterministic"
352 );
353 assert_eq!(
354 first.skipped, again.skipped,
355 "skipped ordering in the report must be deterministic"
356 );
357 assert_eq!(
358 output_files(&first_out),
359 output_files(&out),
360 "emitted bytes must be identical across runs"
361 );
362 }
363}