Commit cbd8cc8c4b
Verified · cmc
Layout: unified · split
Cargo.lock +1 −1
| @@ -675,7 +675,7 @@ dependencies = [ | |||
| 675 | 675 | ||
| 676 | [[package]] | 676 | [[package]] |
| 677 | name = "org-ssg" | 677 | name = "org-ssg" |
| 678 | version = "0.16.0" | 678 | version = "0.17.0" |
| 679 | dependencies = [ | 679 | dependencies = [ |
| 680 | "anyhow", | 680 | "anyhow", |
| 681 | "blake3", | 681 | "blake3", |
Cargo.toml +1 −1
| @@ -1,6 +1,6 @@ | |||
| 1 | [package] | 1 | [package] |
| 2 | name = "org-ssg" | 2 | name = "org-ssg" |
| 3 | version = "0.16.0" | 3 | version = "0.17.0" |
| 4 | edition = "2021" | 4 | edition = "2021" |
| 5 | description = "Org-mode static site generator that renders the org element tree straight to HTML" | 5 | description = "Org-mode static site generator that renders the org element tree straight to HTML" |
| 6 | license = "MIT" | 6 | license = "MIT" |
README.md +5 −1
| @@ -69,6 +69,10 @@ expose_page_list = false | |||
| 69 | [highlight] | 69 | [highlight] |
| 70 | theme = "InspiredGitHub" | 70 | theme = "InspiredGitHub" |
| 71 | 71 | ||
| 72 | [build] | ||
| 73 | drafts = false | ||
| 74 | assets = [] # extra directories copied to the site root, e.g. ["../theme/static"] | ||
| 75 | |||
| 72 | [html] | 76 | [html] |
| 73 | heading_offset = 1 # a level-1 org heading becomes <h2>, beneath the layout's <h1> | 77 | heading_offset = 1 # a level-1 org heading becomes <h2>, beneath the layout's <h1> |
| 74 | ``` | 78 | ``` |
| @@ -382,7 +386,7 @@ all-of-org. Phase 0 checked this line against a real 179-file corpus and found i | |||
| 382 | | **18** | **Per-page layouts: `[[pages]]` rules and `#+TEMPLATE:`** | **done** | | 386 | | **18** | **Per-page layouts: `[[pages]]` rules and `#+TEMPLATE:`** | **done** | |
| 383 | | **19** | **Export parity: relative heading levels, special strings, sub/superscript, caption numbering, checkbox and counter markup, table marker columns, special blocks** | **done** | | 387 | | **19** | **Export parity: relative heading levels, special strings, sub/superscript, caption numbering, checkbox and counter markup, table marker columns, special blocks** | **done** | |
| 384 | | **20** | **Correctness debt: org's entity table, table captions, a reported `#+INCLUDE:`, and an oracle that separates deliberate divergence from defects** | **done** | | 388 | | **20** | **Correctness debt: org's entity table, table captions, a reported `#+INCLUDE:`, and an oracle that separates deliberate divergence from defects** | **done** | |
| 385 | | 21 | Extra asset roots; per-template hashing so one layout edit does not re-render the site | next | | 389 | | **21** | **Extra asset roots; per-template hashing so one layout edit does not re-render the site** | **done** | |
| 386 | | 22 | Release engineering: CI, MSRV, published binaries, changelog, a written compatibility promise | 1.0 | | 390 | | 22 | Release engineering: CI, MSRV, published binaries, changelog, a written compatibility promise | 1.0 | |
| 387 | 391 | ||
| 388 | ### v0.2 in / out | 392 | ### v0.2 in / out |
docs/guide/02-configuration.org +23
| @@ -33,6 +33,7 @@ syntaxes_dir = "syntaxes" | |||
| 33 | 33 | ||
| 34 | [build] | 34 | [build] |
| 35 | drafts = false | 35 | drafts = false |
| 36 | assets = [] | ||
| 36 | 37 | ||
| 37 | [html] | 38 | [html] |
| 38 | heading_offset = 1 | 39 | heading_offset = 1 |
| @@ -190,10 +191,32 @@ should not stop a site from building. | |||
| 190 | | Key | Default | Meaning | | 191 | | Key | Default | Meaning | |
| 191 | |-----+---------+---------| | 192 | |-----+---------+---------| |
| 192 | | =drafts= | =false= | Include pages marked =#+DRAFT:=. | | 193 | | =drafts= | =false= | Include pages marked =#+DRAFT:=. | |
| 194 | | =assets= | =[]= | Extra directories copied to the *site root*. | | ||
| 193 | 195 | ||
| 194 | =--drafts= on the command line turns this on for one run. The flag can only turn drafts | 196 | =--drafts= on the command line turns this on for one run. The flag can only turn drafts |
| 195 | on; it never turns off a config that asked for them. | 197 | on; it never turns off a config that asked for them. |
| 196 | 198 | ||
| 199 | ** Static files that live elsewhere | ||
| 200 | |||
| 201 | A site's static files do not always sit where its writing does. weblorg publishes | ||
| 202 | =theme/static/= at =/=, and a repository migrating from it should not have to move | ||
| 203 | =robots.txt= next to its blog posts to keep the URL: | ||
| 204 | |||
| 205 | #+BEGIN_SRC toml | ||
| 206 | [build] | ||
| 207 | assets = ["../theme/static"] | ||
| 208 | #+END_SRC | ||
| 209 | |||
| 210 | Paths are relative to the source root and may point outside it. Each directory's | ||
| 211 | *contents* land at the site root — =theme/static/img/logo.svg= publishes at =/img/logo.svg=, | ||
| 212 | not =/static/img/logo.svg=. | ||
| 213 | |||
| 214 | Two files claiming one URL is a build error naming both, rather than a coin flip decided | ||
| 215 | by directory order. A path that is not a directory is an error too, since it is a typo. | ||
| 216 | |||
| 217 | Under =watch= and =serve= these directories are watched as well, so editing a stylesheet | ||
| 218 | outside the source tree still reloads the page. | ||
| 219 | |||
| 197 | * [html] | 220 | * [html] |
| 198 | 221 | ||
| 199 | | Key | Default | Meaning | | 222 | | Key | Default | Meaning | |
docs/guide/07-incremental.org +14 −1
| @@ -36,11 +36,24 @@ Every page has a key composed from four hashes: | |||
| 36 | | content | The source file's bytes change. | | 36 | | content | The source file's bytes change. | |
| 37 | | resolved links | A link's target moves, is renamed, or disappears. | | 37 | | resolved links | A link's target moves, is renamed, or disappears. | |
| 38 | | config | =org-ssg.toml= changes, or the shared chrome does. | | 38 | | config | =org-ssg.toml= changes, or the shared chrome does. | |
| 39 | | templates | Any template's source changes. | | 39 | | templates | *This page's* layout changes, or something that layout extends or includes. | |
| 40 | 40 | ||
| 41 | If a page's key matches the cached one and its output file still exists, the file on disk | 41 | If a page's key matches the cached one and its output file still exists, the file on disk |
| 42 | is already correct and is left untouched. | 42 | is already correct and is left untouched. |
| 43 | 43 | ||
| 44 | ** Template scope | ||
| 45 | |||
| 46 | The template component covers the layout a page actually renders through, plus everything | ||
| 47 | that layout pulls in — followed through ={% extends %}=, ={% include %}=, ={% import %}= | ||
| 48 | and ={% from %}=. Editing =feed.xml= on a 196-page site re-renders one page; editing a | ||
| 49 | =post.html= that only blog posts use re-renders the posts. =base.html= is extended by | ||
| 50 | almost everything, so editing it still re-renders almost everything — which is correct, | ||
| 51 | and is why the win shows up on the *other* edits. | ||
| 52 | |||
| 53 | A template whose include is computed at render time — ={% include chooser %}= — cannot be | ||
| 54 | followed, so it is treated as depending on every template. Over-invalidating costs time; | ||
| 55 | under-invalidating publishes a stale page. | ||
| 56 | |||
| 44 | The cache lives in =<output>/.org-ssg-cache.json= and is tagged with a format version. A | 57 | The cache lives in =<output>/.org-ssg-cache.json= and is tagged with a format version. A |
| 45 | version mismatch, a missing file or a corrupt file all fall back to a full rebuild — the | 58 | version mismatch, a missing file or a corrupt file all fall back to a full rebuild — the |
| 46 | cache is an optimisation, never a correctness dependency. There is a test for each of | 59 | cache is an optimisation, never a correctness dependency. There is a test for each of |
src/config.rs +12
| @@ -201,6 +201,14 @@ pub struct Build { | |||
| 201 | /// ready to be read. `--drafts` turns it on for a session, which is what you want | 201 | /// ready to be read. `--drafts` turns it on for a session, which is what you want |
| 202 | /// under `watch` while writing one. | 202 | /// under `watch` while writing one. |
| 203 | pub drafts: bool, | 203 | pub drafts: bool, |
| 204 | /// Extra directories whose contents are copied to the *site root*, on top of the | ||
| 205 | /// non-`.org` files found in the source directory. Relative to the source root, and | ||
| 206 | /// allowed to point outside it. | ||
| 207 | /// | ||
| 208 | /// This exists because a site's static files do not always live where its writing | ||
| 209 | /// does: weblorg publishes `theme/static/` to `/`, and a repository migrating from it | ||
| 210 | /// should not have to move `robots.txt` next to its blog posts to keep the URL. | ||
| 211 | pub assets: Vec<Utf8PathBuf>, | ||
| 204 | } | 212 | } |
| 205 | 213 | ||
| 206 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] | 214 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] |
| @@ -565,6 +573,10 @@ syntaxes_dir = "syntaxes" | |||
| 565 | # Include pages marked `#+DRAFT:`. Off by default — the point of marking a draft is that | 573 | # Include pages marked `#+DRAFT:`. Off by default — the point of marking a draft is that |
| 566 | # it is not ready to be read. `--drafts` turns it on for one run, handy under `watch`. | 574 | # it is not ready to be read. `--drafts` turns it on for one run, handy under `watch`. |
| 567 | drafts = false | 575 | drafts = false |
| 576 | # Extra directories copied to the site root, for static files that live outside the | ||
| 577 | # source directory. `assets = ["../theme/static"]` publishes that directory's contents at | ||
| 578 | # `/`, not at `/static/`. | ||
| 579 | assets = [] | ||
| 568 | 580 | ||
| 569 | [html] | 581 | [html] |
| 570 | # How far to push heading levels down: a level-1 org heading becomes <h(1 + offset)>. | 582 | # How far to push heading levels down: a level-1 org heading becomes <h(1 + offset)>. |
src/incremental.rs +1 −2
| @@ -29,7 +29,7 @@ use crate::util::output_url; | |||
| 29 | /// Bump whenever the `Document` type, hashing scheme, or resolution rules change. | 29 | /// Bump whenever the `Document` type, hashing scheme, or resolution rules change. |
| 30 | /// On mismatch: discard cache, full rebuild (spec §4.5). The blake3 crate's major | 30 | /// On mismatch: discard cache, full rebuild (spec §4.5). The blake3 crate's major |
| 31 | /// version is folded in as the "hash-algo version" so a hash upgrade also busts. | 31 | /// version is folded in as the "hash-algo version" so a hash upgrade also busts. |
| 32 | pub const CACHE_FORMAT_VERSION: u32 = 6; | 32 | pub const CACHE_FORMAT_VERSION: u32 = 7; |
| 33 | 33 | ||
| 34 | /// blake3 hex identity for a content/config/template/render-key hash class (spec §4.1). | 34 | /// blake3 hex identity for a content/config/template/render-key hash class (spec §4.1). |
| 35 | pub type Hash = ContentHash; | 35 | pub type Hash = ContentHash; |
| @@ -192,7 +192,6 @@ pub struct PageRecord { | |||
| 192 | pub struct Manifest { | 192 | pub struct Manifest { |
| 193 | pub format_version: u32, | 193 | pub format_version: u32, |
| 194 | pub config_hash: Option<Hash>, | 194 | pub config_hash: Option<Hash>, |
| 195 | pub template_hash: Option<Hash>, | ||
| 196 | pub pages: HashMap<Utf8PathBuf, PageRecord>, | 195 | pub pages: HashMap<Utf8PathBuf, PageRecord>, |
| 197 | pub graph: DepGraph, | 196 | pub graph: DepGraph, |
| 198 | } | 197 | } |
src/site.rs +102 −14
| @@ -833,7 +833,8 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result | |||
| 833 | // Create the output directory up front so it can be recognised and excluded when it | 833 | // Create the output directory up front so it can be recognised and excluded when it |
| 834 | // lives inside the source tree. | 834 | // lives inside the source tree. |
| 835 | fs::create_dir_all(out).with_context(|| format!("creating {out}"))?; | 835 | fs::create_dir_all(out).with_context(|| format!("creating {out}"))?; |
| 836 | let (_org_rel, assets) = discover(src, &cfg, Some(out))?; | 836 | let (_org_rel, source_assets) = discover(src, &cfg, Some(out))?; |
| 837 | let assets = collect_assets(src, &cfg, Some(out), &source_assets)?; | ||
| 837 | let (preps, symbols) = prepare_pages(src, &cfg, Some(out))?; | 838 | let (preps, symbols) = prepare_pages(src, &cfg, Some(out))?; |
| 838 | 839 | ||
| 839 | let templater = Templater::load(Some(&src.join(&cfg.templates.dir)), &cfg.site.base_url)?; | 840 | let templater = Templater::load(Some(&src.join(&cfg.templates.dir)), &cfg.site.base_url)?; |
| @@ -876,7 +877,9 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result | |||
| 876 | site_structure_hash_ordered(&entries) | 877 | site_structure_hash_ordered(&entries) |
| 877 | }; | 878 | }; |
| 878 | let cfg_hash = combine(config_hash(&cfg), structure_hash); | 879 | let cfg_hash = combine(config_hash(&cfg), structure_hash); |
| 879 | let tmpl_hash = template_hash(templater.sources()); | 880 | // Per template rather than per site: a page's render key covers the layout it uses |
| 881 | // and that layout's own includes, so editing `feed.xml` re-renders the feed. | ||
| 882 | let tmpl_hash_for = |name: &str| template_hash(&templater.sources_for(name)); | ||
| 880 | 883 | ||
| 881 | // Compose each page's render key and record its dependency edges. | 884 | // Compose each page's render key and record its dependency edges. |
| 882 | let mut new_graph = DepGraph::default(); | 885 | let mut new_graph = DepGraph::default(); |
| @@ -884,7 +887,7 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result | |||
| 884 | let listings = build_listings(&cfg, &preps)?; | 887 | let listings = build_listings(&cfg, &preps)?; |
| 885 | for p in &preps { | 888 | for p in &preps { |
| 886 | let rlh = resolved_links_hash(&p.source, &p.used, &symbols); | 889 | let rlh = resolved_links_hash(&p.source, &p.used, &symbols); |
| 887 | let key = render_key(p.content_hash, rlh, cfg_hash, tmpl_hash); | 890 | let key = render_key(p.content_hash, rlh, cfg_hash, tmpl_hash_for(&p.template)); |
| 888 | new_graph.defines.insert(p.source.clone(), p.defines.clone()); | 891 | new_graph.defines.insert(p.source.clone(), p.defines.clone()); |
| 889 | new_graph.uses.insert(p.source.clone(), p.used.clone()); | 892 | new_graph.uses.insert(p.source.clone(), p.used.clone()); |
| 890 | new_records.push(( | 893 | new_records.push(( |
| @@ -911,7 +914,6 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result | |||
| 911 | &new_records, | 914 | &new_records, |
| 912 | &new_graph, | 915 | &new_graph, |
| 913 | cfg_hash, | 916 | cfg_hash, |
| 914 | tmpl_hash, | ||
| 915 | out, | 917 | out, |
| 916 | prior.as_ref(), | 918 | prior.as_ref(), |
| 917 | ); | 919 | ); |
| @@ -984,7 +986,10 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result | |||
| 984 | // therefore re-renders that section's index and nothing else — the same precision the | 986 | // therefore re-renders that section's index and nothing else — the same precision the |
| 985 | // rest of the build gets from content hashing. | 987 | // rest of the build gets from content hashing. |
| 986 | for listing in &listings { | 988 | for listing in &listings { |
| 987 | let key = combine(listing_entries_hash(listing), combine(cfg_hash, tmpl_hash)); | 989 | let key = combine( |
| 990 | listing_entries_hash(listing), | ||
| 991 | combine(cfg_hash, tmpl_hash_for(&listing.template)), | ||
| 992 | ); | ||
| 988 | let dest = out.join(&listing.output); | 993 | let dest = out.join(&listing.output); |
| 989 | let cached = prior | 994 | let cached = prior |
| 990 | .as_ref() | 995 | .as_ref() |
| @@ -1040,21 +1045,20 @@ pub fn build_site(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result | |||
| 1040 | 1045 | ||
| 1041 | // Assets are a dumb copy in v0.3 (spec §8 Q11): copy every run. Cheap, and keeps the | 1046 | // Assets are a dumb copy in v0.3 (spec §8 Q11): copy every run. Cheap, and keeps the |
| 1042 | // full-vs-incremental byte equivalence trivially true for non-`.org` files. | 1047 | // full-vs-incremental byte equivalence trivially true for non-`.org` files. |
| 1043 | for rel in &assets { | 1048 | for asset in &assets { |
| 1044 | let from = src.join(rel); | 1049 | let dest = out.join(&asset.rel); |
| 1045 | let dest = out.join(rel); | ||
| 1046 | if let Some(parent) = dest.parent() { | 1050 | if let Some(parent) = dest.parent() { |
| 1047 | fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?; | 1051 | fs::create_dir_all(parent).with_context(|| format!("creating {parent}"))?; |
| 1048 | } | 1052 | } |
| 1049 | fs::copy(&from, &dest).with_context(|| format!("copying {from} -> {dest}"))?; | 1053 | fs::copy(&asset.from, &dest) |
| 1050 | report.assets.push(rel.clone()); | 1054 | .with_context(|| format!("copying {} -> {dest}", asset.from))?; |
| 1055 | report.assets.push(asset.rel.clone()); | ||
| 1051 | } | 1056 | } |
| 1052 | 1057 | ||
| 1053 | // Persist the manifest for the next build. | 1058 | // Persist the manifest for the next build. |
| 1054 | let manifest = Manifest { | 1059 | let manifest = Manifest { |
| 1055 | format_version: CACHE_FORMAT_VERSION, | 1060 | format_version: CACHE_FORMAT_VERSION, |
| 1056 | config_hash: Some(cfg_hash), | 1061 | config_hash: Some(cfg_hash), |
| 1057 | template_hash: Some(tmpl_hash), | ||
| 1058 | pages: new_records | 1062 | pages: new_records |
| 1059 | .into_iter() | 1063 | .into_iter() |
| 1060 | .map(|(src_path, rec, _)| (src_path, rec)) | 1064 | .map(|(src_path, rec, _)| (src_path, rec)) |
| @@ -1098,7 +1102,6 @@ fn compute_rebuild_set( | |||
| 1098 | new_records: &[(Utf8PathBuf, PageRecord, Hash)], | 1102 | new_records: &[(Utf8PathBuf, PageRecord, Hash)], |
| 1099 | new_graph: &DepGraph, | 1103 | new_graph: &DepGraph, |
| 1100 | cfg_hash: Hash, | 1104 | cfg_hash: Hash, |
| 1101 | tmpl_hash: Hash, | ||
| 1102 | out: &Utf8Path, | 1105 | out: &Utf8Path, |
| 1103 | prior: Option<&Manifest>, | 1106 | prior: Option<&Manifest>, |
| 1104 | ) -> HashSet<Utf8PathBuf> { | 1107 | ) -> HashSet<Utf8PathBuf> { |
| @@ -1108,8 +1111,10 @@ fn compute_rebuild_set( | |||
| 1108 | return all; // No usable cache ⇒ full rebuild. | 1111 | return all; // No usable cache ⇒ full rebuild. |
| 1109 | }; | 1112 | }; |
| 1110 | 1113 | ||
| 1111 | // A global config/template change invalidates every page (spec §4.1). | 1114 | // A config change invalidates every page. Template changes do not come through here: |
| 1112 | if prior.config_hash != Some(cfg_hash) || prior.template_hash != Some(tmpl_hash) { | 1115 | // each page's render key carries the hash of the templates *it* uses, so the key |
| 1116 | // comparison below invalidates exactly the pages whose layout moved. | ||
| 1117 | if prior.config_hash != Some(cfg_hash) { | ||
| 1113 | return all; | 1118 | return all; |
| 1114 | } | 1119 | } |
| 1115 | 1120 | ||
| @@ -1196,6 +1201,89 @@ fn discover( | |||
| 1196 | Ok((org, assets)) | 1201 | Ok((org, assets)) |
| 1197 | } | 1202 | } |
| 1198 | 1203 | ||
| 1204 | /// One file to copy through to the output: where it is, and where it goes. | ||
| 1205 | #[derive(Debug, Clone, PartialEq)] | ||
| 1206 | pub struct Asset { | ||
| 1207 | /// Path to read from. | ||
| 1208 | pub from: Utf8PathBuf, | ||
| 1209 | /// Path to write, relative to the output root. | ||
| 1210 | pub rel: Utf8PathBuf, | ||
| 1211 | } | ||
| 1212 | |||
| 1213 | /// Every file to copy: the source directory's non-`.org` files, then each extra asset | ||
| 1214 | /// root's contents, flattened onto the site root. | ||
| 1215 | /// | ||
| 1216 | /// Two files claiming one output path is an error rather than a race — whichever won | ||
| 1217 | /// would depend on directory order, and a site whose favicon changes when a file is | ||
| 1218 | /// renamed elsewhere is worse than a build that stops. | ||
| 1219 | fn collect_assets( | ||
| 1220 | src: &Utf8Path, | ||
| 1221 | config: &Config, | ||
| 1222 | out: Option<&Utf8Path>, | ||
| 1223 | from_source: &[Utf8PathBuf], | ||
| 1224 | ) -> Result<Vec<Asset>> { | ||
| 1225 | let mut assets: Vec<Asset> = from_source | ||
| 1226 | .iter() | ||
| 1227 | .map(|rel| Asset { | ||
| 1228 | from: src.join(rel), | ||
| 1229 | rel: rel.clone(), | ||
| 1230 | }) | ||
| 1231 | .collect(); | ||
| 1232 | |||
| 1233 | for root in &config.build.assets { | ||
| 1234 | let base = src.join(root); | ||
| 1235 | if !base.is_dir() { | ||
| 1236 | anyhow::bail!( | ||
| 1237 | "build.assets lists {root}, which is not a directory (looked in {base})" | ||
| 1238 | ); | ||
| 1239 | } | ||
| 1240 | let base_canon = std::fs::canonicalize(&base) | ||
| 1241 | .ok() | ||
| 1242 | .and_then(|p| Utf8PathBuf::from_path_buf(p).ok()) | ||
| 1243 | .unwrap_or_else(|| base.clone()); | ||
| 1244 | // An asset root that contains the output directory would copy the site into | ||
| 1245 | // itself, one build at a time. | ||
| 1246 | let out_canon = out | ||
| 1247 | .and_then(|out| std::fs::canonicalize(out).ok()) | ||
| 1248 | .and_then(|p| Utf8PathBuf::from_path_buf(p).ok()); | ||
| 1249 | if out_canon.is_some_and(|o| o.starts_with(&base_canon)) { | ||
| 1250 | anyhow::bail!( | ||
| 1251 | "build.assets lists {root}, which contains the output directory {}", | ||
| 1252 | out.unwrap_or(Utf8Path::new("(none)")) | ||
| 1253 | ); | ||
| 1254 | } | ||
| 1255 | for entry in WalkDir::new(&base).sort_by_file_name() { | ||
| 1256 | let entry = entry.with_context(|| format!("walking {base}"))?; | ||
| 1257 | if !entry.file_type().is_file() { | ||
| 1258 | continue; | ||
| 1259 | } | ||
| 1260 | let abs = Utf8PathBuf::from_path_buf(entry.into_path()) | ||
| 1261 | .map_err(|p| anyhow::anyhow!("non-UTF-8 path: {}", p.display()))?; | ||
| 1262 | let rel = abs | ||
| 1263 | .strip_prefix(&base) | ||
| 1264 | .map(|p| p.to_owned()) | ||
| 1265 | .unwrap_or_else(|_| abs.clone()); | ||
| 1266 | if rel.components().any(|c| c.as_str().starts_with('.')) { | ||
| 1267 | continue; | ||
| 1268 | } | ||
| 1269 | assets.push(Asset { from: abs, rel }); | ||
| 1270 | } | ||
| 1271 | } | ||
| 1272 | |||
| 1273 | let mut seen: HashMap<&Utf8Path, &Utf8Path> = HashMap::new(); | ||
| 1274 | for asset in &assets { | ||
| 1275 | if let Some(first) = seen.insert(&asset.rel, &asset.from) { | ||
| 1276 | anyhow::bail!( | ||
| 1277 | "two files both publish to {}: {first} and {}", | ||
| 1278 | asset.rel, | ||
| 1279 | asset.from | ||
| 1280 | ); | ||
| 1281 | } | ||
| 1282 | } | ||
| 1283 | assets.sort_by(|a, b| a.rel.cmp(&b.rel)); | ||
| 1284 | Ok(assets) | ||
| 1285 | } | ||
| 1286 | |||
| 1199 | /// Source-relative directories that DISCOVER must not descend into: the template | 1287 | /// Source-relative directories that DISCOVER must not descend into: the template |
| 1200 | /// directory (build input, not content) and the output directory when it lives inside | 1288 | /// directory (build input, not content) and the output directory when it lives inside |
| 1201 | /// the source. | 1289 | /// the source. |
src/template.rs +83 −1
| @@ -11,7 +11,7 @@ | |||
| 11 | //! invalidates the pages that use it, and that has to hold for user templates too, or a | 11 | //! invalidates the pages that use it, and that has to hold for user templates too, or a |
| 12 | //! design change would leave a site half-updated. | 12 | //! design change would leave a site half-updated. |
| 13 | 13 | ||
| 14 | use std::collections::BTreeMap; | 14 | use std::collections::{BTreeMap, BTreeSet}; |
| 15 | 15 | ||
| 16 | use anyhow::{Context, Result}; | 16 | use anyhow::{Context, Result}; |
| 17 | use camino::Utf8Path; | 17 | use camino::Utf8Path; |
| @@ -206,6 +206,39 @@ impl Templater { | |||
| 206 | &self.sources | 206 | &self.sources |
| 207 | } | 207 | } |
| 208 | 208 | ||
| 209 | /// The sources a page rendered through `name` actually depends on: that template plus | ||
| 210 | /// everything it extends, includes or imports, transitively. | ||
| 211 | /// | ||
| 212 | /// This is what keeps a layout edit proportional. Hashing *all* templates into every | ||
| 213 | /// page means touching `feed.xml` re-renders a 200-page site, which is most of the | ||
| 214 | /// wait in a `serve` session spent on design. | ||
| 215 | /// | ||
| 216 | /// A template whose include is computed at render time — `{% include chooser %}` — | ||
| 217 | /// cannot be followed statically, so it depends on everything. Over-invalidating is | ||
| 218 | /// slow; under-invalidating publishes a stale page. | ||
| 219 | pub fn sources_for(&self, name: &str) -> Vec<(String, String)> { | ||
| 220 | let mut seen: BTreeSet<String> = BTreeSet::new(); | ||
| 221 | let mut queue = vec![name.to_string()]; | ||
| 222 | while let Some(current) = queue.pop() { | ||
| 223 | if !seen.insert(current.clone()) { | ||
| 224 | continue; | ||
| 225 | } | ||
| 226 | let Some((_, source)) = self.sources.iter().find(|(n, _)| *n == current) else { | ||
| 227 | continue; | ||
| 228 | }; | ||
| 229 | let (deps, dynamic) = referenced_templates(source); | ||
| 230 | if dynamic { | ||
| 231 | return self.sources.clone(); | ||
| 232 | } | ||
| 233 | queue.extend(deps); | ||
| 234 | } | ||
| 235 | self.sources | ||
| 236 | .iter() | ||
| 237 | .filter(|(n, _)| seen.contains(n)) | ||
| 238 | .cloned() | ||
| 239 | .collect() | ||
| 240 | } | ||
| 241 | |||
| 209 | /// Is a template with this name registered? | 242 | /// Is a template with this name registered? |
| 210 | pub fn has(&self, name: &str) -> bool { | 243 | pub fn has(&self, name: &str) -> bool { |
| 211 | self.env.get_template(name).is_ok() | 244 | self.env.get_template(name).is_ok() |
| @@ -532,6 +565,55 @@ pub const STARTER_FEED_TEMPLATE: &str = r#"<?xml version="1.0" encoding="utf-8"? | |||
| 532 | </rss> | 565 | </rss> |
| 533 | "#; | 566 | "#; |
| 534 | 567 | ||
| 568 | /// Template names a source refers to, and whether any reference is computed at render | ||
| 569 | /// time rather than written as a literal. | ||
| 570 | /// | ||
| 571 | /// A hand-rolled scan rather than a parse: minijinja does not expose the dependency | ||
| 572 | /// graph, and the three tags that pull in another template all name it as the first | ||
| 573 | /// string literal in the tag. | ||
| 574 | fn referenced_templates(source: &str) -> (Vec<String>, bool) { | ||
| 575 | const TAGS: &[&str] = &["extends", "include", "import", "from"]; | ||
| 576 | let mut names = Vec::new(); | ||
| 577 | let mut dynamic = false; | ||
| 578 | let mut rest = source; | ||
| 579 | while let Some(start) = rest.find("{%") { | ||
| 580 | let after = &rest[start + 2..]; | ||
| 581 | let Some(end) = after.find("%}") else { break }; | ||
| 582 | let tag = &after[..end]; | ||
| 583 | rest = &after[end + 2..]; | ||
| 584 | |||
| 585 | let keyword = tag | ||
| 586 | .trim_start() | ||
| 587 | .trim_start_matches('-') | ||
| 588 | .split_whitespace() | ||
| 589 | .next() | ||
| 590 | .unwrap_or(""); | ||
| 591 | if !TAGS.contains(&keyword) { | ||
| 592 | continue; | ||
| 593 | } | ||
| 594 | match string_literal(tag) { | ||
| 595 | Some(name) => names.push(name), | ||
| 596 | // `{% include some_variable %}` or `{% include ["a", "b"] %}` past the first | ||
| 597 | // entry: the set cannot be known here. | ||
| 598 | None => dynamic = true, | ||
| 599 | } | ||
| 600 | } | ||
| 601 | if source.contains("{% include [") || source.contains("{%- include [") { | ||
| 602 | dynamic = true; | ||
| 603 | } | ||
| 604 | (names, dynamic) | ||
| 605 | } | ||
| 606 | |||
| 607 | /// The first single- or double-quoted string in a tag body. | ||
| 608 | fn string_literal(tag: &str) -> Option<String> { | ||
| 609 | let bytes = tag.as_bytes(); | ||
| 610 | let quote = bytes.iter().position(|b| *b == b'"' || *b == b'\'')?; | ||
| 611 | let delim = bytes[quote]; | ||
| 612 | let after = &tag[quote + 1..]; | ||
| 613 | let end = after.find(delim as char)?; | ||
| 614 | Some(after[..end].to_string()) | ||
| 615 | } | ||
| 616 | |||
| 535 | /// HTML-escape template output, escaping the same characters Jinja2 does. | 617 | /// HTML-escape template output, escaping the same characters Jinja2 does. |
| 536 | /// | 618 | /// |
| 537 | /// minijinja additionally escapes `/` as `/`, which is a defence for values | 619 | /// minijinja additionally escapes `/` as `/`, which is a defence for values |
src/watch.rs +40 −4
| @@ -57,6 +57,12 @@ impl ChangeFilter { | |||
| 57 | /// Build a filter for a source and output directory. Paths are canonicalized so | 57 | /// Build a filter for a source and output directory. Paths are canonicalized so |
| 58 | /// `.`, `./src`, an absolute path and a symlinked one all compare equal. | 58 | /// `.`, `./src`, an absolute path and a symlinked one all compare equal. |
| 59 | pub fn new(src: &Utf8Path, out: &Utf8Path) -> Self { | 59 | pub fn new(src: &Utf8Path, out: &Utf8Path) -> Self { |
| 60 | ChangeFilter::with_asset_roots(src, out, &[]) | ||
| 61 | } | ||
| 62 | |||
| 63 | /// As [`ChangeFilter::new`], plus extra asset roots. Their paths are recognised too, | ||
| 64 | /// so editing a stylesheet that lives outside the source directory still rebuilds. | ||
| 65 | pub fn with_asset_roots(src: &Utf8Path, out: &Utf8Path, asset_roots: &[Utf8PathBuf]) -> Self { | ||
| 60 | let canon = |p: &Utf8Path| -> Option<Utf8PathBuf> { | 66 | let canon = |p: &Utf8Path| -> Option<Utf8PathBuf> { |
| 61 | std::fs::canonicalize(p) | 67 | std::fs::canonicalize(p) |
| 62 | .ok() | 68 | .ok() |
| @@ -77,6 +83,10 @@ impl ChangeFilter { | |||
| 77 | }; | 83 | }; |
| 78 | 84 | ||
| 79 | let mut roots: Vec<Utf8PathBuf> = src_canon.into_iter().chain([src.to_owned()]).collect(); | 85 | let mut roots: Vec<Utf8PathBuf> = src_canon.into_iter().chain([src.to_owned()]).collect(); |
| 86 | for root in asset_roots { | ||
| 87 | roots.extend(canon(root)); | ||
| 88 | roots.push(root.clone()); | ||
| 89 | } | ||
| 80 | roots.dedup(); | 90 | roots.dedup(); |
| 81 | // Longest first, so the most specific spelling wins. | 91 | // Longest first, so the most specific spelling wins. |
| 82 | roots.sort_by_key(|r| std::cmp::Reverse(r.as_str().len())); | 92 | roots.sort_by_key(|r| std::cmp::Reverse(r.as_str().len())); |
| @@ -148,6 +158,27 @@ fn is_editor_scratch(name: &str) -> bool { | |||
| 148 | || (name.starts_with('#') && name.ends_with('#')) | 158 | || (name.starts_with('#') && name.ends_with('#')) |
| 149 | } | 159 | } |
| 150 | 160 | ||
| 161 | /// The extra asset directories a build will read, as paths that can be watched. | ||
| 162 | /// | ||
| 163 | /// A config that fails to load is not this function's problem — the rebuild reports it | ||
| 164 | /// properly — so an unreadable config simply yields no extra roots. | ||
| 165 | fn asset_roots(src: &Utf8Path, opts: &BuildOptions) -> Vec<Utf8PathBuf> { | ||
| 166 | let config = match &opts.config_path { | ||
| 167 | Some(path) => crate::config::Config::load_file(path), | ||
| 168 | None => crate::config::Config::load(src), | ||
| 169 | }; | ||
| 170 | config | ||
| 171 | .map(|c| { | ||
| 172 | c.build | ||
| 173 | .assets | ||
| 174 | .iter() | ||
| 175 | .map(|root| src.join(root)) | ||
| 176 | .filter(|root| root.is_dir()) | ||
| 177 | .collect() | ||
| 178 | }) | ||
| 179 | .unwrap_or_default() | ||
| 180 | } | ||
| 181 | |||
| 151 | /// Build once, then rebuild whenever the source changes. Runs until interrupted. | 182 | /// Build once, then rebuild whenever the source changes. Runs until interrupted. |
| 152 | pub fn run(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result<()> { | 183 | pub fn run(src: &Utf8Path, out: &Utf8Path, opts: &BuildOptions) -> Result<()> { |
| 153 | run_with(src, out, opts, |_| {}) | 184 | run_with(src, out, opts, |_| {}) |
| @@ -172,12 +203,17 @@ pub fn run_with( | |||
| 172 | report.rendered.len() | 203 | report.rendered.len() |
| 173 | ); | 204 | ); |
| 174 | 205 | ||
| 175 | let filter = ChangeFilter::new(src, out); | 206 | // Asset roots can live outside the source directory, and a stylesheet that does not |
| 207 | // rebuild when saved is worse than no watching at all. | ||
| 208 | let asset_roots = asset_roots(src, opts); | ||
| 209 | let filter = ChangeFilter::with_asset_roots(src, out, &asset_roots); | ||
| 176 | let (tx, rx) = mpsc::channel(); | 210 | let (tx, rx) = mpsc::channel(); |
| 177 | let mut watcher = make_watcher(tx)?; | 211 | let mut watcher = make_watcher(tx)?; |
| 178 | watcher | 212 | for root in std::iter::once(src.to_owned()).chain(asset_roots) { |
| 179 | .watch(src.as_std_path(), RecursiveMode::Recursive) | 213 | watcher |
| 180 | .with_context(|| format!("watching {src}"))?; | 214 | .watch(root.as_std_path(), RecursiveMode::Recursive) |
| 215 | .with_context(|| format!("watching {root}"))?; | ||
| 216 | } | ||
| 181 | 217 | ||
| 182 | loop { | 218 | loop { |
| 183 | // Block until something happens, then keep draining while events keep arriving | 219 | // Block until something happens, then keep draining while events keep arriving |
tests/config.rs +79
| @@ -2210,3 +2210,82 @@ fn same_day_entries_sort_by_time_of_day() { | |||
| 2210 | "newest first, by the clock:\n{html}" | 2210 | "newest first, by the clock:\n{html}" |
| 2211 | ); | 2211 | ); |
| 2212 | } | 2212 | } |
| 2213 | |||
| 2214 | // --------------------------------------------------------------------------- | ||
| 2215 | // Extra asset roots | ||
| 2216 | // --------------------------------------------------------------------------- | ||
| 2217 | |||
| 2218 | /// A site's static files do not always live where its writing does. A repository | ||
| 2219 | /// migrating from a generator that published `theme/static/` to `/` should not have to | ||
| 2220 | /// move `robots.txt` next to its blog posts to keep the URL. | ||
| 2221 | #[test] | ||
| 2222 | fn an_asset_root_publishes_to_the_site_root() { | ||
| 2223 | let root = tmpdir("assetroot"); | ||
| 2224 | let src = root.join("src"); | ||
| 2225 | std::fs::create_dir_all(&src).unwrap(); | ||
| 2226 | write_site(&src); | ||
| 2227 | std::fs::create_dir_all(root.join("theme/static/img")).unwrap(); | ||
| 2228 | std::fs::write(root.join("theme/static/robots.txt"), "User-agent: *\n").unwrap(); | ||
| 2229 | std::fs::write(root.join("theme/static/img/logo.svg"), "<svg/>").unwrap(); | ||
| 2230 | std::fs::write( | ||
| 2231 | src.join("org-ssg.toml"), | ||
| 2232 | "[build]\nassets = [\"../theme/static\"]\n", | ||
| 2233 | ) | ||
| 2234 | .unwrap(); | ||
| 2235 | let out = root.join("out"); | ||
| 2236 | let report = build(&src, &out); | ||
| 2237 | |||
| 2238 | assert!(out.join("robots.txt").exists(), "flattened onto the root"); | ||
| 2239 | assert!( | ||
| 2240 | out.join("img/logo.svg").exists(), | ||
| 2241 | "and keeps its own structure below that" | ||
| 2242 | ); | ||
| 2243 | assert!( | ||
| 2244 | report.assets.contains(&Utf8PathBuf::from("robots.txt")), | ||
| 2245 | "the report counts it: {:?}", | ||
| 2246 | report.assets | ||
| 2247 | ); | ||
| 2248 | } | ||
| 2249 | |||
| 2250 | /// Two files claiming one URL is a coin flip decided by directory order. A build that | ||
| 2251 | /// stops is better than a favicon that changes when something elsewhere is renamed. | ||
| 2252 | #[test] | ||
| 2253 | fn two_assets_claiming_one_url_is_an_error() { | ||
| 2254 | let root = tmpdir("assetclash"); | ||
| 2255 | let src = root.join("src"); | ||
| 2256 | std::fs::create_dir_all(&src).unwrap(); | ||
| 2257 | write_site(&src); | ||
| 2258 | std::fs::write(src.join("style.css"), "body{}").unwrap(); | ||
| 2259 | std::fs::create_dir_all(root.join("static")).unwrap(); | ||
| 2260 | std::fs::write(root.join("static/style.css"), "body{color:red}").unwrap(); | ||
| 2261 | std::fs::write( | ||
| 2262 | src.join("org-ssg.toml"), | ||
| 2263 | "[build]\nassets = [\"../static\"]\n", | ||
| 2264 | ) | ||
| 2265 | .unwrap(); | ||
| 2266 | |||
| 2267 | let err = build_site(&src, &root.join("out"), &BuildOptions::default()) | ||
| 2268 | .expect_err("a collision must fail the build"); | ||
| 2269 | assert!( | ||
| 2270 | format!("{err:#}").contains("style.css"), | ||
| 2271 | "names the path: {err:#}" | ||
| 2272 | ); | ||
| 2273 | } | ||
| 2274 | |||
| 2275 | /// A typo in a path is a typo, not an empty directory to shrug at. | ||
| 2276 | #[test] | ||
| 2277 | fn a_missing_asset_root_is_an_error() { | ||
| 2278 | let root = tmpdir("assetmissing"); | ||
| 2279 | let src = root.join("src"); | ||
| 2280 | std::fs::create_dir_all(&src).unwrap(); | ||
| 2281 | write_site(&src); | ||
| 2282 | std::fs::write( | ||
| 2283 | src.join("org-ssg.toml"), | ||
| 2284 | "[build]\nassets = [\"../nope\"]\n", | ||
| 2285 | ) | ||
| 2286 | .unwrap(); | ||
| 2287 | |||
| 2288 | let err = build_site(&src, &root.join("out"), &BuildOptions::default()) | ||
| 2289 | .expect_err("a missing asset root must fail"); | ||
| 2290 | assert!(format!("{err:#}").contains("nope"), "names it: {err:#}"); | ||
| 2291 | } | ||
tests/incremental.rs +65
| @@ -448,3 +448,68 @@ fn retitling_a_top_level_page_still_rebuilds_the_site() { | |||
| 448 | let post = std::fs::read_to_string(out_dir.join("blog/first.html")).unwrap(); | 448 | let post = std::fs::read_to_string(out_dir.join("blog/first.html")).unwrap(); |
| 449 | assert!(post.contains("Colophon"), "nested pages show the updated nav title"); | 449 | assert!(post.contains("Colophon"), "nested pages show the updated nav title"); |
| 450 | } | 450 | } |
| 451 | |||
| 452 | /// Editing one layout must re-render the pages that use it, and only those. Hashing every | ||
| 453 | /// template into every page means a change to the feed template rewrites the whole site, | ||
| 454 | /// which is most of the wait in a `serve` session spent on design. | ||
| 455 | #[test] | ||
| 456 | fn editing_one_template_rebuilds_only_the_pages_that_use_it() { | ||
| 457 | let root = tmpdir("tmplscope"); | ||
| 458 | let src = root.join("src"); | ||
| 459 | std::fs::create_dir_all(src.join("blog")).unwrap(); | ||
| 460 | std::fs::create_dir_all(src.join("templates")).unwrap(); | ||
| 461 | std::fs::write(src.join("index.org"), "#+TITLE: Home\n\nWelcome.\n").unwrap(); | ||
| 462 | std::fs::write(src.join("about.org"), "#+TITLE: About\n\nAbout.\n").unwrap(); | ||
| 463 | std::fs::write( | ||
| 464 | src.join("blog/post.org"), | ||
| 465 | "#+TITLE: Post\n#+DATE: 2026-01-01\n\nBody.\n", | ||
| 466 | ) | ||
| 467 | .unwrap(); | ||
| 468 | std::fs::write( | ||
| 469 | src.join("templates/base.html"), | ||
| 470 | "<html><body>{% block content %}{{ body | safe }}{% endblock %}</body></html>", | ||
| 471 | ) | ||
| 472 | .unwrap(); | ||
| 473 | std::fs::write( | ||
| 474 | src.join("templates/post.html"), | ||
| 475 | "{% extends \"base.html\" %}{% block content %}{{ body | safe }}<p>reply</p>{% endblock %}", | ||
| 476 | ) | ||
| 477 | .unwrap(); | ||
| 478 | std::fs::write( | ||
| 479 | src.join("org-ssg.toml"), | ||
| 480 | "[[pages]]\nmatch = \"blog\"\ntemplate = \"post.html\"\n", | ||
| 481 | ) | ||
| 482 | .unwrap(); | ||
| 483 | let out_dir = root.join("out"); | ||
| 484 | build_site(&src, &out_dir, &BuildOptions::default()).unwrap(); | ||
| 485 | |||
| 486 | // post.html is used by one page. | ||
| 487 | std::fs::write( | ||
| 488 | src.join("templates/post.html"), | ||
| 489 | "{% extends \"base.html\" %}{% block content %}{{ body | safe }}<p>reply now</p>{% endblock %}", | ||
| 490 | ) | ||
| 491 | .unwrap(); | ||
| 492 | let r = build_site(&src, &out_dir, &BuildOptions::default()).unwrap(); | ||
| 493 | assert_eq!( | ||
| 494 | r.rendered, | ||
| 495 | vec![Utf8PathBuf::from("blog/post.html")], | ||
| 496 | "only the page whose layout changed" | ||
| 497 | ); | ||
| 498 | assert!(std::fs::read_to_string(out_dir.join("blog/post.html")) | ||
| 499 | .unwrap() | ||
| 500 | .contains("reply now")); | ||
| 501 | |||
| 502 | // base.html is extended by post.html, so editing it reaches both. | ||
| 503 | std::fs::write( | ||
| 504 | src.join("templates/base.html"), | ||
| 505 | "<html><body class=\"new\">{% block content %}{{ body | safe }}{% endblock %}</body></html>", | ||
| 506 | ) | ||
| 507 | .unwrap(); | ||
| 508 | let r = build_site(&src, &out_dir, &BuildOptions::default()).unwrap(); | ||
| 509 | assert_eq!( | ||
| 510 | r.rendered.len(), | ||
| 511 | 3, | ||
| 512 | "a layout everything inherits still re-renders everything: {:?}", | ||
| 513 | r.rendered | ||
| 514 | ); | ||
| 515 | } | ||