krz/orgo
Lightning fast org-mode static site generator.
clone: git clone https://gitbay.org/krz/orgo.git
1//! Corpus audit (spec §5, Phase 0): measure which org constructs a real corpus actually
2//! uses, and classify each against the v1 IN/OUT line.
3//!
4//! This exists because the v1 scope was, on the README's own admission, *recommended*
5//! rather than measured — a guess about which slice of org matters. A guess about a
6//! corpus is a hypothesis, and this is the experiment. It answers two questions:
7//!
8//! 1. **Coverage** — of the constructs this corpus uses, which do we handle? A construct
9//! that is common here and out of scope is a scope bug, not a corpus quirk.
10//! 2. **Blind spots** — which constructs are here that the implementation has no opinion
11//! about at all? These are the dangerous ones: not "known unsupported" but unknown.
12//!
13//! The audit is deliberately a *separate, line-oriented scanner* rather than a reuse of
14//! [`crate::parser`]. Auditing with the parser could only ever find constructs the parser
15//! already knows about, which is precisely the wrong instrument for question 2 — it would
16//! report a blind spot as clean.
17//!
18//! Nothing here reports document *text*. Counts, construct names, and `file:line`
19//! locations only, so an audit of private notes stays publishable.
20
21use std::collections::BTreeMap;
22
23use anyhow::{Context, Result};
24use camino::{Utf8Path, Utf8PathBuf};
25use walkdir::WalkDir;
26
27/// Where a construct sits relative to the v1 scope line (README §"v1 scope").
28#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
29pub enum Scope {
30 /// v1 handles this.
31 In,
32 /// v1 deliberately excludes this; it degrades predictably.
33 Out,
34}
35
36impl Scope {
37 fn label(self) -> &'static str {
38 match self {
39 Scope::In => "IN ",
40 Scope::Out => "OUT",
41 }
42 }
43}
44
45/// One construct's tally across the corpus.
46#[derive(Debug, Default, Clone)]
47pub struct Tally {
48 pub occurrences: usize,
49 pub files: usize,
50 /// First `file:line` the construct was seen at, to make a finding actionable.
51 pub first_seen: Option<String>,
52 /// Set while scanning one file, to count each file once.
53 seen_in_current_file: bool,
54}
55
56/// The audit result: the fixed construct catalog plus the dynamic name censuses.
57#[derive(Debug, Default)]
58pub struct Audit {
59 pub files: usize,
60 pub lines: usize,
61 /// Catalogued constructs → tally.
62 pub constructs: BTreeMap<(Scope, &'static str), Tally>,
63 /// Every distinct `#+KEYWORD:` seen, by name.
64 pub keywords: BTreeMap<String, Tally>,
65 /// Every distinct `#+BEGIN_<TYPE>` seen, by type.
66 pub blocks: BTreeMap<String, Tally>,
67 /// Every distinct `:DRAWER:` seen, by name.
68 pub drawers: BTreeMap<String, Tally>,
69 /// Every distinct link scheme seen (`https`, `file`, `id`, `denote`, ...).
70 pub link_schemes: BTreeMap<String, Tally>,
71}
72
73/// Names the implementation understands, so the census can flag everything else. These
74/// are the *recognized* sets, not the supported ones: `INCLUDE` is recognized (it is
75/// deliberately inert) while an unlisted keyword is a genuine blind spot.
76const KNOWN_KEYWORDS: &[&str] = &[
77 "TITLE", "AUTHOR", "DATE", "EMAIL", "LANGUAGE", "OPTIONS", "FILETAGS", "DESCRIPTION",
78 "KEYWORDS", "CAPTION", "NAME", "ATTR_HTML", "RESULTS", "TBLFM", "INCLUDE", "TODO",
79 "STARTUP", "SUBTITLE", "SETUPFILE", "MACRO", "PROPERTY", "HTML_HEAD", "EXCLUDE_TAGS",
80];
81/// Blocks with dedicated handling. Any *other* name renders as a special block — a div
82/// with that name holding parsed org — so an unlisted block is a note about what a corpus
83/// contains rather than a construct that will be lost.
84const KNOWN_BLOCKS: &[&str] = &[
85 "SRC", "QUOTE", "EXAMPLE", "CENTER", "EXPORT", "VERSE", "COMMENT",
86];
87const KNOWN_DRAWERS: &[&str] = &["PROPERTIES", "LOGBOOK", "END"];
88/// Keyword names conventional enough to be worth flagging when they lead a heading.
89/// A custom sequence is only *real* if some `#+TODO:` declares it, which the census
90/// reports separately — this list keeps the heading-level signal honest.
91const CONVENTIONAL_TODO_KEYWORDS: &[&str] = &[
92 "NEXT", "WAITING", "HOLD", "CANCELLED", "CANCELED", "STARTED", "SOMEDAY", "PROJ",
93 "IN-PROGRESS", "BLOCKED", "REVIEW",
94];
95const KNOWN_SCHEMES: &[&str] = &[
96 "http", "https", "mailto", "ftp", "news", "tel", "file", "id", "custom-id", "heading",
97 "relative",
98];
99
100impl Audit {
101 /// Is this name one the implementation recognizes?
102 pub fn is_known(kind: Census, name: &str) -> bool {
103 let known = match kind {
104 Census::Keyword => KNOWN_KEYWORDS,
105 Census::Block => KNOWN_BLOCKS,
106 Census::Drawer => KNOWN_DRAWERS,
107 Census::Scheme => KNOWN_SCHEMES,
108 };
109 known.iter().any(|k| k.eq_ignore_ascii_case(name))
110 }
111}
112
113/// Which dynamic census a name belongs to.
114#[derive(Debug, Clone, Copy)]
115pub enum Census {
116 Keyword,
117 Block,
118 Drawer,
119 Scheme,
120}
121
122/// Walk `root`, auditing every `.org` file.
123pub fn audit(root: &Utf8Path) -> Result<Audit> {
124 let mut audit = Audit::default();
125 let mut paths: Vec<Utf8PathBuf> = Vec::new();
126
127 if root.is_file() {
128 paths.push(root.to_owned());
129 } else {
130 for entry in WalkDir::new(root).sort_by_file_name() {
131 let entry = entry.with_context(|| format!("walking {root}"))?;
132 if !entry.file_type().is_file() {
133 continue;
134 }
135 let path = Utf8PathBuf::from_path_buf(entry.into_path())
136 .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?;
137 if path.extension() == Some("org") {
138 paths.push(path);
139 }
140 }
141 }
142
143 for path in &paths {
144 // A file that cannot be read is reported and skipped: an audit of 179 files
145 // should not be lost to one unreadable one.
146 let source = match std::fs::read_to_string(path) {
147 Ok(s) => s,
148 Err(e) => {
149 eprintln!("warning: skipping {path}: {e}");
150 continue;
151 }
152 };
153 let rel = path.strip_prefix(root).unwrap_or(path).to_owned();
154 audit.scan_file(&rel, &source);
155 audit.files += 1;
156 }
157 Ok(audit)
158}
159
160impl Audit {
161 fn scan_file(&mut self, path: &Utf8Path, source: &str) {
162 // Reset the per-file flags so each construct counts this file at most once.
163 for tally in self.constructs.values_mut() {
164 tally.seen_in_current_file = false;
165 }
166 for map in [
167 &mut self.keywords,
168 &mut self.blocks,
169 &mut self.drawers,
170 &mut self.link_schemes,
171 ] {
172 for tally in map.values_mut() {
173 tally.seen_in_current_file = false;
174 }
175 }
176
177 let mut in_block: Option<String> = None;
178 for (idx, line) in source.lines().enumerate() {
179 self.lines += 1;
180 let at = format!("{path}:{}", idx + 1);
181 let trimmed = line.trim_start();
182
183 // Inside a verbatim block only the terminator matters — a `*` in a source
184 // block is not a heading, and counting it as one would corrupt the audit.
185 if let Some(kind) = &in_block {
186 if trimmed.to_ascii_uppercase().starts_with("#+END_") {
187 in_block = None;
188 } else if kind.eq_ignore_ascii_case("SRC") || kind.eq_ignore_ascii_case("EXAMPLE") {
189 continue;
190 }
191 continue;
192 }
193 if let Some(rest) = trimmed.to_ascii_uppercase().strip_prefix("#+BEGIN_") {
194 let kind = rest.split_whitespace().next().unwrap_or("").to_string();
195 self.count_census(Census::Block, &kind, &at);
196 self.count(scope_of_block(&kind), block_construct(&kind), &at);
197 if trimmed.to_ascii_uppercase().contains(":RESULTS") {
198 self.count(Scope::Out, "babel header args (:results)", &at);
199 }
200 in_block = Some(kind);
201 continue;
202 }
203
204 self.scan_line(line, trimmed, &at);
205 }
206 }
207
208 fn scan_line(&mut self, line: &str, trimmed: &str, at: &str) {
209 // --- headings and their metadata ---
210 if let Some(stars) = heading_stars(line) {
211 self.count(Scope::In, "heading", at);
212 let rest = line[stars..].trim();
213 let word = rest.split_whitespace().next().unwrap_or("");
214 if word == "TODO" || word == "DONE" {
215 self.count(Scope::In, "TODO keyword (default set)", at);
216 } else if CONVENTIONAL_TODO_KEYWORDS.contains(&word) {
217 // Only conventional keyword names count. "Any all-caps first word" is
218 // the tempting rule and it is wrong: it reads `* CSS Variables` as the
219 // keyword `CSS`, which on this corpus produced 23 false positives and
220 // zero true ones. An audit that overstates a gap is worse than no audit,
221 // because it argues for work nobody needs.
222 self.count(Scope::Out, "TODO keyword (custom sequence)", at);
223 }
224 if rest.contains("[#") {
225 self.count(Scope::In, "priority cookie", at);
226 }
227 if rest.trim_end().ends_with(':') && rest.trim_end().matches(':').count() >= 2 {
228 self.count(Scope::In, "heading tags", at);
229 }
230 if rest.contains("[/") || rest.contains("[%") {
231 self.count(Scope::Out, "statistics cookie", at);
232 }
233 return;
234 }
235
236 // --- planning and clocking ---
237 for marker in ["SCHEDULED:", "DEADLINE:", "CLOSED:"] {
238 if trimmed.starts_with(marker) {
239 self.count(Scope::Out, "planning line", at);
240 }
241 }
242 if trimmed.starts_with("CLOCK:") {
243 self.count(Scope::Out, "clock entry", at);
244 }
245
246 // --- keywords and drawers ---
247 if let Some(rest) = trimmed.strip_prefix("#+") {
248 if let Some(colon) = rest.find(':') {
249 let key = rest[..colon].trim().to_ascii_uppercase();
250 if !key.is_empty() && !key.contains(char::is_whitespace) {
251 self.count_census(Census::Keyword, &key, at);
252 match key.as_str() {
253 "CAPTION" | "NAME" | "ATTR_HTML" => {
254 self.count(Scope::In, "affiliated keyword", at)
255 }
256 "TBLFM" => self.count(Scope::Out, "table formula (#+TBLFM:)", at),
257 "INCLUDE" => self.count(Scope::Out, "#+INCLUDE:", at),
258 "RESULTS" => self.count(Scope::Out, "babel results block", at),
259 "TODO" => self.count(Scope::Out, "#+TODO: keyword sequence", at),
260 "MACRO" => self.count(Scope::Out, "macro definition", at),
261 _ => self.count(Scope::In, "#+ keyword", at),
262 }
263 }
264 }
265 } else if is_drawer(trimmed) {
266 let name = trimmed[1..trimmed.len() - 1].to_ascii_uppercase();
267 if name != "END" {
268 self.count_census(Census::Drawer, &name, at);
269 match name.as_str() {
270 "PROPERTIES" => self.count(Scope::In, "property drawer", at),
271 _ => self.count(Scope::Out, "non-PROPERTIES drawer", at),
272 }
273 }
274 }
275
276 // --- lists, tables, rules ---
277 if let Some(bullet) = list_bullet(trimmed) {
278 self.count(Scope::In, "list item", at);
279 if bullet == Bullet::Ordered {
280 self.count(Scope::In, "ordered list", at);
281 }
282 let indent = line.len() - trimmed.len();
283 if indent > 0 {
284 self.count(Scope::In, "nested list item", at);
285 }
286 if trimmed.contains(" :: ") {
287 self.count(Scope::In, "description list", at);
288 }
289 let after = trimmed.trim_start_matches(['-', '+', '*', ' ']);
290 if after.starts_with("[ ]") || after.starts_with("[X]") || after.starts_with("[-]") {
291 self.count(Scope::In, "checkbox", at);
292 }
293 }
294 if trimmed.starts_with('|') {
295 self.count(Scope::In, "table row", at);
296 }
297 if trimmed.starts_with(':') && !is_drawer(trimmed) && trimmed.starts_with(": ") {
298 self.count(Scope::Out, "fixed-width line", at);
299 }
300
301 // --- footnotes ---
302 if trimmed.starts_with("[fn:") {
303 self.count(Scope::In, "footnote definition", at);
304 } else if line.contains("[fn:") {
305 self.count(Scope::In, "footnote reference", at);
306 }
307
308 // --- inline objects ---
309 self.scan_inline(line, at);
310 }
311
312 fn scan_inline(&mut self, line: &str, at: &str) {
313 // Links: count each `[[target]]`, censusing its scheme.
314 let mut rest = line;
315 while let Some(start) = rest.find("[[") {
316 let after = &rest[start + 2..];
317 let Some(end) = after.find("]]") else { break };
318 let inner = &after[..end];
319 let target = inner.split("][").next().unwrap_or(inner);
320 self.count(Scope::In, "link", at);
321 self.count_census(Census::Scheme, &link_scheme(target), at);
322 rest = &after[end..];
323 }
324
325 if has_timestamp(line) {
326 self.count(Scope::In, "timestamp", at);
327 }
328 if line.contains("{{{") {
329 self.count(Scope::Out, "macro call", at);
330 }
331 if line.contains("<<<") {
332 self.count(Scope::Out, "radio target", at);
333 } else if line.contains("<<") && line.contains(">>") {
334 self.count(Scope::Out, "internal target", at);
335 }
336 if line.contains("\\begin{") || latex_inline(line) {
337 self.count(Scope::Out, "LaTeX fragment", at);
338 }
339 if entity_ref(line) {
340 self.count(Scope::Out, "entity (\\name)", at);
341 }
342 for (marker, name) in [
343 ('*', "bold"),
344 ('/', "italic"),
345 ('_', "underline"),
346 ('+', "strike-through"),
347 ('=', "verbatim"),
348 ('~', "code"),
349 ] {
350 if emphasis_pair(line, marker) {
351 self.count(Scope::In, name, at);
352 }
353 }
354 }
355
356 fn count(&mut self, scope: Scope, name: &'static str, at: &str) {
357 let tally = self.constructs.entry((scope, name)).or_default();
358 bump(tally, at);
359 }
360
361 fn count_census(&mut self, kind: Census, name: &str, at: &str) {
362 let map = match kind {
363 Census::Keyword => &mut self.keywords,
364 Census::Block => &mut self.blocks,
365 Census::Drawer => &mut self.drawers,
366 Census::Scheme => &mut self.link_schemes,
367 };
368 let tally = map.entry(name.to_string()).or_default();
369 bump(tally, at);
370 }
371}
372
373fn bump(tally: &mut Tally, at: &str) {
374 tally.occurrences += 1;
375 if !tally.seen_in_current_file {
376 tally.seen_in_current_file = true;
377 tally.files += 1;
378 }
379 if tally.first_seen.is_none() {
380 tally.first_seen = Some(at.to_string());
381 }
382}
383
384// ---------------------------------------------------------------------------
385// Line-level detectors. Deliberately independent of the parser (see module docs).
386// ---------------------------------------------------------------------------
387
388fn heading_stars(line: &str) -> Option<usize> {
389 if !line.starts_with('*') {
390 return None;
391 }
392 let stars = line.chars().take_while(|c| *c == '*').count();
393 let after = &line[stars..];
394 (after.starts_with(' ') || after.is_empty()).then_some(stars)
395}
396
397#[derive(PartialEq)]
398enum Bullet {
399 Unordered,
400 Ordered,
401}
402
403fn list_bullet(trimmed: &str) -> Option<Bullet> {
404 let bytes = trimmed.as_bytes();
405 if bytes.is_empty() {
406 return None;
407 }
408 if (bytes[0] == b'-' || bytes[0] == b'+') && (bytes.len() == 1 || bytes[1] == b' ') {
409 return Some(Bullet::Unordered);
410 }
411 let digits = trimmed.chars().take_while(|c| c.is_ascii_digit()).count();
412 if digits > 0 {
413 let after = &trimmed[digits..];
414 if (after.starts_with('.') || after.starts_with(')'))
415 && (after.len() == 1 || after.as_bytes()[1] == b' ')
416 {
417 return Some(Bullet::Ordered);
418 }
419 }
420 None
421}
422
423fn is_drawer(trimmed: &str) -> bool {
424 let t = trimmed.trim_end();
425 t.len() >= 3
426 && t.starts_with(':')
427 && t.ends_with(':')
428 && t[1..t.len() - 1]
429 .chars()
430 .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
431 && t.len() > 2
432}
433
434fn scope_of_block(kind: &str) -> Scope {
435 if KNOWN_BLOCKS.iter().any(|k| k.eq_ignore_ascii_case(kind)) {
436 Scope::In
437 } else {
438 Scope::Out
439 }
440}
441
442fn block_construct(kind: &str) -> &'static str {
443 match kind.to_ascii_uppercase().as_str() {
444 "SRC" => "source block",
445 "QUOTE" => "quote block",
446 "EXAMPLE" => "example block",
447 "CENTER" => "center block",
448 "EXPORT" => "export block",
449 _ => "unmodelled block type",
450 }
451}
452
453/// The scheme of a link target, normalized into the census's vocabulary.
454fn link_scheme(target: &str) -> String {
455 if let Some(rest) = target.split_once(':') {
456 let scheme = rest.0;
457 if !scheme.is_empty()
458 && scheme
459 .chars()
460 .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '+')
461 {
462 return scheme.to_ascii_lowercase();
463 }
464 }
465 if target.starts_with('#') {
466 return "custom-id".to_string();
467 }
468 if target.starts_with('*') {
469 return "heading".to_string();
470 }
471 "relative".to_string()
472}
473
474/// A `<...>`/`[...]` span opening with an ISO date is a timestamp.
475fn has_timestamp(line: &str) -> bool {
476 let bytes = line.as_bytes();
477 for (i, c) in line.char_indices() {
478 if c != '<' && c != '[' {
479 continue;
480 }
481 let rest = &bytes[i + 1..];
482 if rest.len() >= 10
483 && rest[..4].iter().all(u8::is_ascii_digit)
484 && rest[4] == b'-'
485 && rest[5..7].iter().all(u8::is_ascii_digit)
486 && rest[7] == b'-'
487 && rest[8..10].iter().all(u8::is_ascii_digit)
488 {
489 return true;
490 }
491 }
492 false
493}
494
495/// `$x$` or `\(x\)` inline math. `$` alone (a price, a shell prompt) is not math.
496fn latex_inline(line: &str) -> bool {
497 if line.contains("\\(") && line.contains("\\)") {
498 return true;
499 }
500 let dollars = line.matches('$').count();
501 dollars >= 2 && line.contains("$\\")
502}
503
504/// A `\name` entity reference such as `\alpha`, excluding LaTeX environment commands.
505fn entity_ref(line: &str) -> bool {
506 for (i, c) in line.char_indices() {
507 if c != '\\' {
508 continue;
509 }
510 let rest = &line[i + 1..];
511 let name: String = rest.chars().take_while(|c| c.is_ascii_alphabetic()).collect();
512 if name.len() >= 3 && !matches!(name.as_str(), "begin" | "end") {
513 return true;
514 }
515 }
516 false
517}
518
519/// A plausible `*bold*`-style emphasis pair: two markers on one line with non-space
520/// content between them. Approximate by design — the audit measures prevalence, and the
521/// parser owns the exact pre/post-character rules.
522fn emphasis_pair(line: &str, marker: char) -> bool {
523 let positions: Vec<usize> = line
524 .char_indices()
525 .filter(|(_, c)| *c == marker)
526 .map(|(i, _)| i)
527 .collect();
528 if positions.len() < 2 {
529 return false;
530 }
531 // A leading `*` is a heading, and `-`/`+` at line start is a bullet.
532 let trimmed = line.trim_start();
533 if trimmed.starts_with(marker) {
534 return false;
535 }
536 positions.windows(2).any(|w| w[1] > w[0] + 1)
537}
538
539// ---------------------------------------------------------------------------
540// Report
541// ---------------------------------------------------------------------------
542
543/// Render the audit as a readable report. Names, counts and locations only — never
544/// document text, so an audit of private notes is safe to paste into an issue.
545pub fn report(audit: &Audit) -> String {
546 let mut out = String::new();
547 out.push_str(&format!(
548 "corpus: {} file(s), {} line(s)\n",
549 audit.files, audit.lines
550 ));
551
552 let mut rows: Vec<(&(Scope, &str), &Tally)> = audit.constructs.iter().collect();
553 rows.sort_by(|a, b| {
554 b.1.occurrences
555 .cmp(&a.1.occurrences)
556 .then_with(|| a.0 .1.cmp(b.0 .1))
557 });
558
559 out.push_str("\nCONSTRUCTS (by frequency)\n");
560 out.push_str(&format!(
561 "{:<4} {:<32} {:>8} {:>7} {}\n",
562 "", "construct", "uses", "files", "first seen"
563 ));
564 for ((scope, name), tally) in &rows {
565 out.push_str(&format!(
566 "{:<4} {:<32} {:>8} {:>7} {}\n",
567 scope.label(),
568 name,
569 tally.occurrences,
570 tally.files,
571 tally.first_seen.as_deref().unwrap_or("")
572 ));
573 }
574
575 let in_uses: usize = rows
576 .iter()
577 .filter(|((s, _), _)| *s == Scope::In)
578 .map(|(_, t)| t.occurrences)
579 .sum();
580 let out_uses: usize = rows
581 .iter()
582 .filter(|((s, _), _)| *s == Scope::Out)
583 .map(|(_, t)| t.occurrences)
584 .sum();
585 let total = in_uses + out_uses;
586 let pct = |n: usize| {
587 if total == 0 {
588 0.0
589 } else {
590 100.0 * n as f64 / total as f64
591 }
592 };
593 out.push_str(&format!(
594 "\ncoverage: {in_uses} in-scope use(s) ({:.1}%), {out_uses} out-of-scope ({:.1}%)\n",
595 pct(in_uses),
596 pct(out_uses)
597 ));
598
599 for (title, kind, map) in [
600 ("KEYWORDS", Census::Keyword, &audit.keywords),
601 ("BLOCK TYPES", Census::Block, &audit.blocks),
602 ("DRAWERS", Census::Drawer, &audit.drawers),
603 ("LINK SCHEMES", Census::Scheme, &audit.link_schemes),
604 ] {
605 let mut names: Vec<(&String, &Tally)> = map.iter().collect();
606 names.sort_by(|a, b| b.1.occurrences.cmp(&a.1.occurrences).then(a.0.cmp(b.0)));
607 out.push_str(&format!("\n{title}\n"));
608 for (name, tally) in names {
609 let flag = if Audit::is_known(kind, name) {
610 " "
611 } else {
612 "??? "
613 };
614 out.push_str(&format!(
615 "{flag}{:<32} {:>8} {:>7} {}\n",
616 name,
617 tally.occurrences,
618 tally.files,
619 tally.first_seen.as_deref().unwrap_or("")
620 ));
621 }
622 }
623 out.push_str("\n`???` marks a name the implementation does not recognize at all.\n");
624 out
625}