krz/orgo
Lightning fast org-mode static site generator.
clone: git clone https://gitbay.org/krz/orgo.git
1//! Corpus audit (spec §5, Phase 0): measure which org constructs a real corpus actually
2//! uses, and classify each against the v1 IN/OUT line.
3//!
4//! This exists because the v1 scope was, on the README's own admission, *recommended*
5//! rather than measured — a guess about which slice of org matters. A guess about a
6//! corpus is a hypothesis, and this is the experiment. It answers two questions:
7//!
8//! 1. **Coverage** — of the constructs this corpus uses, which do we handle? A construct
9//! that is common here and out of scope is a scope bug, not a corpus quirk.
10//! 2. **Blind spots** — which constructs are here that the implementation has no opinion
11//! about at all? These are the dangerous ones: not "known unsupported" but unknown.
12//!
13//! The audit is deliberately a *separate, line-oriented scanner* rather than a reuse of
14//! [`crate::parser`]. Auditing with the parser could only ever find constructs the parser
15//! already knows about, which is precisely the wrong instrument for question 2 — it would
16//! report a blind spot as clean.
17//!
18//! Nothing here reports document *text*. Counts, construct names, and `file:line`
19//! locations only, so an audit of private notes stays publishable.
20
21use std::collections::BTreeMap;
22
23use anyhow::{Context, Result};
24use camino::{Utf8Path, Utf8PathBuf};
25use walkdir::WalkDir;
26
27/// Where a construct sits relative to the v1 scope line (README §"v1 scope").
28#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
29pub enum Scope {
30 /// v1 handles this.
31 In,
32 /// v1 deliberately excludes this; it degrades predictably.
33 Out,
34}
35
36impl Scope {
37 fn label(self) -> &'static str {
38 match self {
39 Scope::In => "IN ",
40 Scope::Out => "OUT",
41 }
42 }
43}
44
45/// One construct's tally across the corpus.
46#[derive(Debug, Default, Clone)]
47pub struct Tally {
48 pub occurrences: usize,
49 pub files: usize,
50 /// First `file:line` the construct was seen at, to make a finding actionable.
51 pub first_seen: Option<String>,
52 /// Set while scanning one file, to count each file once.
53 seen_in_current_file: bool,
54}
55
56/// The audit result: the fixed construct catalog plus the dynamic name censuses.
57#[derive(Debug, Default)]
58pub struct Audit {
59 pub files: usize,
60 pub lines: usize,
61 /// Catalogued constructs → tally.
62 pub constructs: BTreeMap<(Scope, &'static str), Tally>,
63 /// Every distinct `#+KEYWORD:` seen, by name.
64 pub keywords: BTreeMap<String, Tally>,
65 /// Every distinct `#+BEGIN_<TYPE>` seen, by type.
66 pub blocks: BTreeMap<String, Tally>,
67 /// Every distinct `:DRAWER:` seen, by name.
68 pub drawers: BTreeMap<String, Tally>,
69 /// Every distinct link scheme seen (`https`, `file`, `id`, `denote`, ...).
70 pub link_schemes: BTreeMap<String, Tally>,
71}
72
73/// Names the implementation understands, so the census can flag everything else. These
74/// are the *recognized* sets, not the supported ones: `INCLUDE` is recognized (it is
75/// deliberately inert) while an unlisted keyword is a genuine blind spot.
76const KNOWN_KEYWORDS: &[&str] = &[
77 "TITLE", "AUTHOR", "DATE", "EMAIL", "LANGUAGE", "OPTIONS", "FILETAGS", "DESCRIPTION",
78 "KEYWORDS", "CAPTION", "NAME", "ATTR_HTML", "RESULTS", "TBLFM", "INCLUDE", "TODO",
79 "STARTUP", "SUBTITLE", "SETUPFILE", "MACRO", "PROPERTY", "HTML_HEAD", "EXCLUDE_TAGS",
80 // orgo's own keywords, each read by name: `SLUG` names the output file
81 // (`util::output_path`), `DRAFT` decides whether the page publishes at all
82 // (`util::is_draft`), and `TEMPLATE` picks the template (`config::page_template`).
83 // Leaving them out reported the corpus's most-used keyword as unrecognized.
84 "SLUG", "DRAFT", "TEMPLATE",
85];
86/// Blocks with dedicated handling. Any *other* name renders as a special block — a div
87/// with that name holding parsed org — so an unlisted block is a note about what a corpus
88/// contains rather than a construct that will be lost.
89const KNOWN_BLOCKS: &[&str] = &[
90 "SRC", "QUOTE", "EXAMPLE", "CENTER", "EXPORT", "VERSE", "COMMENT",
91];
92const KNOWN_DRAWERS: &[&str] = &["PROPERTIES", "LOGBOOK", "END"];
93/// Keyword names conventional enough to be worth flagging when they lead a heading.
94/// A custom sequence is only *real* if some `#+TODO:` declares it, which the census
95/// reports separately — this list keeps the heading-level signal honest.
96const CONVENTIONAL_TODO_KEYWORDS: &[&str] = &[
97 "NEXT", "WAITING", "HOLD", "CANCELLED", "CANCELED", "STARTED", "SOMEDAY", "PROJ",
98 "IN-PROGRESS", "BLOCKED", "REVIEW",
99];
100const KNOWN_SCHEMES: &[&str] = &[
101 "http", "https", "mailto", "ftp", "news", "tel", "file", "id", "custom-id", "heading",
102 "relative",
103];
104
105impl Audit {
106 /// Is this name one the implementation recognizes?
107 pub fn is_known(kind: Census, name: &str) -> bool {
108 let known = match kind {
109 Census::Keyword => KNOWN_KEYWORDS,
110 Census::Block => KNOWN_BLOCKS,
111 Census::Drawer => KNOWN_DRAWERS,
112 Census::Scheme => KNOWN_SCHEMES,
113 };
114 known.iter().any(|k| k.eq_ignore_ascii_case(name))
115 }
116}
117
118/// Which dynamic census a name belongs to.
119#[derive(Debug, Clone, Copy)]
120pub enum Census {
121 Keyword,
122 Block,
123 Drawer,
124 Scheme,
125}
126
127/// Walk `root`, auditing every `.org` file.
128pub fn audit(root: &Utf8Path) -> Result<Audit> {
129 let mut audit = Audit::default();
130 let mut paths: Vec<Utf8PathBuf> = Vec::new();
131
132 if root.is_file() {
133 paths.push(root.to_owned());
134 } else {
135 for entry in WalkDir::new(root).sort_by_file_name() {
136 let entry = entry.with_context(|| format!("walking {root}"))?;
137 if !entry.file_type().is_file() {
138 continue;
139 }
140 let path = Utf8PathBuf::from_path_buf(entry.into_path())
141 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
142 if path.extension() == Some("org") {
143 paths.push(path);
144 }
145 }
146 }
147
148 for path in &paths {
149 // A file that cannot be read is reported and skipped: an audit of 179 files
150 // should not be lost to one unreadable one.
151 let source = match std::fs::read_to_string(path) {
152 Ok(s) => s,
153 Err(e) => {
154 eprintln!("warning: skipping {path}: {e}");
155 continue;
156 }
157 };
158 let rel = path.strip_prefix(root).unwrap_or(path).to_owned();
159 audit.scan_file(&rel, &source);
160 audit.files += 1;
161 }
162 Ok(audit)
163}
164
165impl Audit {
166 fn scan_file(&mut self, path: &Utf8Path, source: &str) {
167 // Reset the per-file flags so each construct counts this file at most once.
168 for tally in self.constructs.values_mut() {
169 tally.seen_in_current_file = false;
170 }
171 for map in [
172 &mut self.keywords,
173 &mut self.blocks,
174 &mut self.drawers,
175 &mut self.link_schemes,
176 ] {
177 for tally in map.values_mut() {
178 tally.seen_in_current_file = false;
179 }
180 }
181
182 let mut in_block: Option<String> = None;
183 for (idx, line) in source.lines().enumerate() {
184 self.lines += 1;
185 let at = format!("{path}:{}", idx + 1);
186 let trimmed = line.trim_start();
187
188 // Inside a verbatim block only the terminator matters — a `*` in a source
189 // block is not a heading, and counting it as one would corrupt the audit.
190 if let Some(kind) = &in_block {
191 if trimmed.to_ascii_uppercase().starts_with("#+END_") {
192 in_block = None;
193 } else if kind.eq_ignore_ascii_case("SRC") || kind.eq_ignore_ascii_case("EXAMPLE") {
194 continue;
195 }
196 continue;
197 }
198 if let Some(rest) = trimmed.to_ascii_uppercase().strip_prefix("#+BEGIN_") {
199 let kind = rest.split_whitespace().next().unwrap_or("").to_string();
200 self.count_census(Census::Block, &kind, &at);
201 self.count(scope_of_block(&kind), block_construct(&kind), &at);
202 if trimmed.to_ascii_uppercase().contains(":RESULTS") {
203 self.count(Scope::Out, "babel header args (:results)", &at);
204 }
205 in_block = Some(kind);
206 continue;
207 }
208
209 self.scan_line(line, trimmed, &at);
210 }
211 }
212
213 fn scan_line(&mut self, line: &str, trimmed: &str, at: &str) {
214 // --- headings and their metadata ---
215 if let Some(stars) = heading_stars(line) {
216 self.count(Scope::In, "heading", at);
217 let rest = line[stars..].trim();
218 let word = rest.split_whitespace().next().unwrap_or("");
219 if word == "TODO" || word == "DONE" {
220 self.count(Scope::In, "TODO keyword (default set)", at);
221 } else if CONVENTIONAL_TODO_KEYWORDS.contains(&word) {
222 // Only conventional keyword names count. "Any all-caps first word" is
223 // the tempting rule and it is wrong: it reads `* CSS Variables` as the
224 // keyword `CSS`, which on this corpus produced 23 false positives and
225 // zero true ones. An audit that overstates a gap is worse than no audit,
226 // because it argues for work nobody needs.
227 self.count(Scope::Out, "TODO keyword (custom sequence)", at);
228 }
229 if rest.contains("[#") {
230 self.count(Scope::In, "priority cookie", at);
231 }
232 if rest.trim_end().ends_with(':') && rest.trim_end().matches(':').count() >= 2 {
233 self.count(Scope::In, "heading tags", at);
234 }
235 if rest.contains("[/") || rest.contains("[%") {
236 self.count(Scope::Out, "statistics cookie", at);
237 }
238 return;
239 }
240
241 // --- planning and clocking ---
242 for marker in ["SCHEDULED:", "DEADLINE:", "CLOSED:"] {
243 if trimmed.starts_with(marker) {
244 self.count(Scope::Out, "planning line", at);
245 }
246 }
247 if trimmed.starts_with("CLOCK:") {
248 self.count(Scope::Out, "clock entry", at);
249 }
250
251 // --- keywords and drawers ---
252 if let Some(rest) = trimmed.strip_prefix("#+") {
253 if let Some(colon) = rest.find(':') {
254 let key = rest[..colon].trim().to_ascii_uppercase();
255 if !key.is_empty() && !key.contains(char::is_whitespace) {
256 self.count_census(Census::Keyword, &key, at);
257 match key.as_str() {
258 "CAPTION" | "NAME" | "ATTR_HTML" => {
259 self.count(Scope::In, "affiliated keyword", at)
260 }
261 "TBLFM" => self.count(Scope::Out, "table formula (#+TBLFM:)", at),
262 "INCLUDE" => self.count(Scope::Out, "#+INCLUDE:", at),
263 "RESULTS" => self.count(Scope::Out, "babel results block", at),
264 "TODO" => self.count(Scope::Out, "#+TODO: keyword sequence", at),
265 "MACRO" => self.count(Scope::Out, "macro definition", at),
266 _ => self.count(Scope::In, "#+ keyword", at),
267 }
268 }
269 }
270 } else if is_drawer(trimmed) {
271 let name = trimmed[1..trimmed.len() - 1].to_ascii_uppercase();
272 if name != "END" {
273 self.count_census(Census::Drawer, &name, at);
274 match name.as_str() {
275 "PROPERTIES" => self.count(Scope::In, "property drawer", at),
276 _ => self.count(Scope::Out, "non-PROPERTIES drawer", at),
277 }
278 }
279 }
280
281 // --- lists, tables, rules ---
282 if let Some(bullet) = list_bullet(trimmed) {
283 self.count(Scope::In, "list item", at);
284 if bullet == Bullet::Ordered {
285 self.count(Scope::In, "ordered list", at);
286 }
287 let indent = line.len() - trimmed.len();
288 if indent > 0 {
289 self.count(Scope::In, "nested list item", at);
290 }
291 if trimmed.contains(" :: ") {
292 self.count(Scope::In, "description list", at);
293 }
294 let after = trimmed.trim_start_matches(['-', '+', '*', ' ']);
295 if after.starts_with("[ ]") || after.starts_with("[X]") || after.starts_with("[-]") {
296 self.count(Scope::In, "checkbox", at);
297 }
298 }
299 if trimmed.starts_with('|') {
300 self.count(Scope::In, "table row", at);
301 }
302 if trimmed.starts_with(':') && !is_drawer(trimmed) && trimmed.starts_with(": ") {
303 self.count(Scope::Out, "fixed-width line", at);
304 }
305
306 // --- footnotes ---
307 if trimmed.starts_with("[fn:") {
308 self.count(Scope::In, "footnote definition", at);
309 } else if line.contains("[fn:") {
310 self.count(Scope::In, "footnote reference", at);
311 }
312
313 // --- inline objects ---
314 self.scan_inline(line, at);
315 }
316
317 fn scan_inline(&mut self, line: &str, at: &str) {
318 // Links: count each `[[target]]`, censusing its scheme.
319 let mut rest = line;
320 while let Some(start) = rest.find("[[") {
321 let after = &rest[start + 2..];
322 let Some(end) = after.find("]]") else { break };
323 let inner = &after[..end];
324 let target = inner.split("][").next().unwrap_or(inner);
325 self.count(Scope::In, "link", at);
326 self.count_census(Census::Scheme, &link_scheme(target), at);
327 rest = &after[end..];
328 }
329
330 if has_timestamp(line) {
331 self.count(Scope::In, "timestamp", at);
332 }
333 if line.contains("{{{") {
334 self.count(Scope::Out, "macro call", at);
335 }
336 if line.contains("<<<") {
337 self.count(Scope::Out, "radio target", at);
338 } else if line.contains("<<") && line.contains(">>") {
339 self.count(Scope::Out, "internal target", at);
340 }
341 if line.contains("\\begin{") || latex_inline(line) {
342 self.count(Scope::Out, "LaTeX fragment", at);
343 }
344 if entity_ref(line) {
345 self.count(Scope::Out, "entity (\\name)", at);
346 }
347 for (marker, name) in [
348 ('*', "bold"),
349 ('/', "italic"),
350 ('_', "underline"),
351 ('+', "strike-through"),
352 ('=', "verbatim"),
353 ('~', "code"),
354 ] {
355 if emphasis_pair(line, marker) {
356 self.count(Scope::In, name, at);
357 }
358 }
359 }
360
361 fn count(&mut self, scope: Scope, name: &'static str, at: &str) {
362 let tally = self.constructs.entry((scope, name)).or_default();
363 bump(tally, at);
364 }
365
366 fn count_census(&mut self, kind: Census, name: &str, at: &str) {
367 let map = match kind {
368 Census::Keyword => &mut self.keywords,
369 Census::Block => &mut self.blocks,
370 Census::Drawer => &mut self.drawers,
371 Census::Scheme => &mut self.link_schemes,
372 };
373 let tally = map.entry(name.to_string()).or_default();
374 bump(tally, at);
375 }
376}
377
378fn bump(tally: &mut Tally, at: &str) {
379 tally.occurrences += 1;
380 if !tally.seen_in_current_file {
381 tally.seen_in_current_file = true;
382 tally.files += 1;
383 }
384 if tally.first_seen.is_none() {
385 tally.first_seen = Some(at.to_string());
386 }
387}
388
389// ---------------------------------------------------------------------------
390// Line-level detectors. Deliberately independent of the parser (see module docs).
391// ---------------------------------------------------------------------------
392
393fn heading_stars(line: &str) -> Option<usize> {
394 if !line.starts_with('*') {
395 return None;
396 }
397 let stars = line.chars().take_while(|c| *c == '*').count();
398 let after = &line[stars..];
399 (after.starts_with(' ') || after.is_empty()).then_some(stars)
400}
401
402#[derive(PartialEq)]
403enum Bullet {
404 Unordered,
405 Ordered,
406}
407
408fn list_bullet(trimmed: &str) -> Option<Bullet> {
409 let bytes = trimmed.as_bytes();
410 if bytes.is_empty() {
411 return None;
412 }
413 if (bytes[0] == b'-' || bytes[0] == b'+') && (bytes.len() == 1 || bytes[1] == b' ') {
414 return Some(Bullet::Unordered);
415 }
416 let digits = trimmed.chars().take_while(|c| c.is_ascii_digit()).count();
417 if digits > 0 {
418 let after = &trimmed[digits..];
419 if (after.starts_with('.') || after.starts_with(')'))
420 && (after.len() == 1 || after.as_bytes()[1] == b' ')
421 {
422 return Some(Bullet::Ordered);
423 }
424 }
425 None
426}
427
428fn is_drawer(trimmed: &str) -> bool {
429 let t = trimmed.trim_end();
430 t.len() >= 3
431 && t.starts_with(':')
432 && t.ends_with(':')
433 && t[1..t.len() - 1]
434 .chars()
435 .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
436 && t.len() > 2
437}
438
439fn scope_of_block(kind: &str) -> Scope {
440 if KNOWN_BLOCKS.iter().any(|k| k.eq_ignore_ascii_case(kind)) {
441 Scope::In
442 } else {
443 Scope::Out
444 }
445}
446
447fn block_construct(kind: &str) -> &'static str {
448 match kind.to_ascii_uppercase().as_str() {
449 "SRC" => "source block",
450 "QUOTE" => "quote block",
451 "EXAMPLE" => "example block",
452 "CENTER" => "center block",
453 "EXPORT" => "export block",
454 _ => "unmodelled block type",
455 }
456}
457
458/// The scheme of a link target, normalized into the census's vocabulary.
459fn link_scheme(target: &str) -> String {
460 if let Some(rest) = target.split_once(':') {
461 let scheme = rest.0;
462 if !scheme.is_empty()
463 && scheme
464 .chars()
465 .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '+')
466 {
467 return scheme.to_ascii_lowercase();
468 }
469 }
470 if target.starts_with('#') {
471 return "custom-id".to_string();
472 }
473 if target.starts_with('*') {
474 return "heading".to_string();
475 }
476 "relative".to_string()
477}
478
479/// A `<...>`/`[...]` span opening with an ISO date is a timestamp.
480fn has_timestamp(line: &str) -> bool {
481 let bytes = line.as_bytes();
482 for (i, c) in line.char_indices() {
483 if c != '<' && c != '[' {
484 continue;
485 }
486 let rest = &bytes[i + 1..];
487 if rest.len() >= 10
488 && rest[..4].iter().all(u8::is_ascii_digit)
489 && rest[4] == b'-'
490 && rest[5..7].iter().all(u8::is_ascii_digit)
491 && rest[7] == b'-'
492 && rest[8..10].iter().all(u8::is_ascii_digit)
493 {
494 return true;
495 }
496 }
497 false
498}
499
500/// `$x$` or `\(x\)` inline math. `$` alone (a price, a shell prompt) is not math.
501fn latex_inline(line: &str) -> bool {
502 if line.contains("\\(") && line.contains("\\)") {
503 return true;
504 }
505 let dollars = line.matches('$').count();
506 dollars >= 2 && line.contains("$\\")
507}
508
509/// A `\name` entity reference such as `\alpha`, excluding LaTeX environment commands.
510fn entity_ref(line: &str) -> bool {
511 for (i, c) in line.char_indices() {
512 if c != '\\' {
513 continue;
514 }
515 let rest = &line[i + 1..];
516 let name: String = rest.chars().take_while(|c| c.is_ascii_alphabetic()).collect();
517 if name.len() >= 3 && !matches!(name.as_str(), "begin" | "end") {
518 return true;
519 }
520 }
521 false
522}
523
524/// A plausible `*bold*`-style emphasis pair: two markers on one line with non-space
525/// content between them. Approximate by design — the audit measures prevalence, and the
526/// parser owns the exact pre/post-character rules.
527fn emphasis_pair(line: &str, marker: char) -> bool {
528 let positions: Vec<usize> = line
529 .char_indices()
530 .filter(|(_, c)| *c == marker)
531 .map(|(i, _)| i)
532 .collect();
533 if positions.len() < 2 {
534 return false;
535 }
536 // A leading `*` is a heading, and `-`/`+` at line start is a bullet.
537 let trimmed = line.trim_start();
538 if trimmed.starts_with(marker) {
539 return false;
540 }
541 positions.windows(2).any(|w| w[1] > w[0] + 1)
542}
543
544// ---------------------------------------------------------------------------
545// Report
546// ---------------------------------------------------------------------------
547
548/// Render the audit as a readable report. Names, counts and locations only — never
549/// document text, so an audit of private notes is safe to paste into an issue.
550pub fn report(audit: &Audit) -> String {
551 let mut out = String::new();
552 out.push_str(&format!(
553 "corpus: {} file(s), {} line(s)\n",
554 audit.files, audit.lines
555 ));
556
557 let mut rows: Vec<(&(Scope, &str), &Tally)> = audit.constructs.iter().collect();
558 rows.sort_by(|a, b| {
559 b.1.occurrences
560 .cmp(&a.1.occurrences)
561 .then_with(|| a.0 .1.cmp(b.0 .1))
562 });
563
564 out.push_str("\nCONSTRUCTS (by frequency)\n");
565 out.push_str(&format!(
566 "{:<4} {:<32} {:>8} {:>7} {}\n",
567 "", "construct", "uses", "files", "first seen"
568 ));
569 for ((scope, name), tally) in &rows {
570 out.push_str(&format!(
571 "{:<4} {:<32} {:>8} {:>7} {}\n",
572 scope.label(),
573 name,
574 tally.occurrences,
575 tally.files,
576 tally.first_seen.as_deref().unwrap_or("")
577 ));
578 }
579
580 let in_uses: usize = rows
581 .iter()
582 .filter(|((s, _), _)| *s == Scope::In)
583 .map(|(_, t)| t.occurrences)
584 .sum();
585 let out_uses: usize = rows
586 .iter()
587 .filter(|((s, _), _)| *s == Scope::Out)
588 .map(|(_, t)| t.occurrences)
589 .sum();
590 let total = in_uses + out_uses;
591 let pct = |n: usize| {
592 if total == 0 {
593 0.0
594 } else {
595 100.0 * n as f64 / total as f64
596 }
597 };
598 out.push_str(&format!(
599 "\ncoverage: {in_uses} in-scope use(s) ({:.1}%), {out_uses} out-of-scope ({:.1}%)\n",
600 pct(in_uses),
601 pct(out_uses)
602 ));
603
604 for (title, kind, map) in [
605 ("KEYWORDS", Census::Keyword, &audit.keywords),
606 ("BLOCK TYPES", Census::Block, &audit.blocks),
607 ("DRAWERS", Census::Drawer, &audit.drawers),
608 ("LINK SCHEMES", Census::Scheme, &audit.link_schemes),
609 ] {
610 let mut names: Vec<(&String, &Tally)> = map.iter().collect();
611 names.sort_by(|a, b| b.1.occurrences.cmp(&a.1.occurrences).then(a.0.cmp(b.0)));
612 out.push_str(&format!("\n{title}\n"));
613 for (name, tally) in names {
614 let flag = if Audit::is_known(kind, name) {
615 " "
616 } else {
617 "??? "
618 };
619 out.push_str(&format!(
620 "{flag}{:<32} {:>8} {:>7} {}\n",
621 name,
622 tally.occurrences,
623 tally.files,
624 tally.first_seen.as_deref().unwrap_or("")
625 ));
626 }
627 }
628 out.push_str("\n`???` marks a name the implementation does not recognize at all.\n");
629 out
630}