Commit `1a36bd71b5`

1a36bd71b546ece6081b58ff4693eeaca89aa5d3

parent: 3e4e84c12f

Verified · cmc

cmc <hello@cleberg.net> · 2026-10-04 23:54 UTC

Add incremental reparse

Layout: unified · split

Sources/OrgCore/Parser/Incremental.swift added +249

@@ -0,0 +1,249 @@
	1	import Foundation
	2
	3	public struct TextEdit: Sendable, Equatable {
	4	/// UTF-16 range in the old text, on unicode scalar boundaries.
	5	public let range: Range<Int>
	6	public let replacement: String
	7
	8	public init(range: Range<Int>, replacement: String) {
	9	self.range = range
	10	self.replacement = replacement
	11	}
	12
	13	/// Works on unicode scalars, so an edit between "\r" and "\n" stays exact.
	14	public func apply(to text: String) -> String {
	15	let scalars = text.unicodeScalars
	16	let start = String.Index(utf16Offset: range.lowerBound, in: text)
	17	let end = String.Index(utf16Offset: range.upperBound, in: text)
	18	return String(scalars[..<start]) + replacement + String(scalars[end...])
	19	}
	20	}
	21
	22	enum ReparseStrategy: Equatable {
	23	/// One element reparsed in place; everything else reused.
	24	case element
	25	/// A run of top-level sections reparsed; the sections before and after reused.
	26	case sections
	27	case full
	28	}
	29
	30	extension OrgParser {
	31	/// The tree for `edit.apply(to: oldText)`, reusing as much of `old` as stays valid. Always
	32	/// equal to a full parse of the new text.
	33	public static func reparse(_ old: OrgTree, oldText: String, edit: TextEdit, defaults: OrgSettings = .default) -> OrgTree {
	34	reparseWithStrategy(old, oldText: oldText, edit: edit, defaults: defaults).tree
	35	}
	36
	37	static func reparseWithStrategy(
	38	_ old: OrgTree, oldText: String, edit: TextEdit, defaults: OrgSettings = .default
	39	) -> (tree: OrgTree, strategy: ReparseStrategy) {
	40	let newText = edit.apply(to: oldText)
	41	let context = EditContext(oldText: oldText, newText: newText, edit: edit)
	42	if context.touchesSettings() { return (parse(newText, defaults: defaults), .full) }
	43	if let tree = context.reparseElement(old) { return (tree, .element) }
	44	if let tree = context.reparseSections(old) { return (tree, .sections) }
	45	return (parse(newText, defaults: defaults), .full)
	46	}
	47	}
	48
	49	private struct EditContext {
	50	let oldText: String
	51	let newText: String
	52	let edit: TextEdit
	53	/// Change in UTF-16 length.
	54	var delta: Int { edit.replacement.utf16.count - edit.range.count }
	55
	56	static let settingsKeys: Set<String> = ["TODO", "SEQ_TODO", "TYP_TODO", "PRIORITIES"]
	57
	58	/// Elements that parse the same in isolation as in place, as long as their lines keep
	59	/// their classes.
	60	static let leafKinds: Set<SyntaxKind> = [
	61	.paragraph, .heading, .tableRow, .planning, .clock, .keyword, .affiliatedKeyword, .horizontalRule,
	62	.nodeProperty, .block, .comment, .fixedWidth, .footnoteDefinition,
	63	]
	64
	65	/// File settings must be read again when an edited line is a settings keyword, or when it
	66	/// moves a block or heading boundary in a file that has settings keywords: that can hide or
	67	/// expose a keyword inside a block.
	68	func touchesSettings() -> Bool {
	69	let start = lineStart(oldText, edit.range.lowerBound)
	70	let oldLines = classes(slice(oldText, start, lineEnd(oldText, edit.range.upperBound)))
	71	let newLines = classes(slice(newText, start, lineEnd(newText, edit.range.lowerBound + edit.replacement.utf16.count)))
	72	var movesBoundary = false
	73	for line in oldLines + newLines {
	74	switch line.cls {
	75	case .keyword(let key) where Self.settingsKeys.contains(key):
	76	return true
	77	case .blockBegin, .blockEnd, .dynamicBegin, .dynamicEnd, .heading:
	78	movesBoundary = true
	79	default:
	80	break
	81	}
	82	}
	83	return movesBoundary && Self.settingsKeys.contains { newText.range(of: "#+\($0):", options: .caseInsensitive) != nil }
	84	}
	85
	86	// MARK: - Element
	87
	88	func reparseElement(_ old: OrgTree) -> OrgTree? {
	89	guard let leaf = leaf(in: old.root) else { return nil }
	90	let start = lineStart(oldText, leaf.range.lowerBound)
	91	// An element that starts mid-line (an item's paragraph) follows tokens that would absorb
	92	// whitespace typed at its start.
	93	guard start == leaf.range.lowerBound \|\| edit.range.lowerBound > leaf.range.lowerBound else { return nil }
	94	let oldEnd = leaf.range.upperBound
	95	let newEnd = oldEnd + delta
	96	let newLength = newText.utf16.count
	97	guard newEnd > leaf.range.lowerBound, newEnd <= newLength else { return nil }
	98	// The element must still end at a line end.
	99	if newEnd < newLength {
	100	let utf16 = newText.utf16
	101	guard utf16[utf16.index(utf16.startIndex, offsetBy: newEnd - 1)] == 0x0A else { return nil }
	102	}
	103	let oldClasses = classes(slice(oldText, start, oldEnd))
	104	let newClasses = classes(slice(newText, start, newEnd))
	105	guard oldClasses.count == newClasses.count,
	106	zip(oldClasses, newClasses).allSatisfy({ $0.cls == $1.cls && $0.indent == $1.indent }) else { return nil }
	107	let text = String(slice(newText, leaf.range.lowerBound, newEnd))
	108	guard let green = Parser.reparseElement(leaf.kind, text: text, settings: old.settings) else { return nil }
	109	return OrgTree(green: replacing(leaf, with: green), settings: old.settings)
	110	}
	111
	112	/// The outermost leaf element containing the edit.
	113	func leaf(in root: SyntaxNode) -> SyntaxNode? {
	114	let range = edit.range
	115	var current = root
	116	while let child = current.children.first(where: {
	117	$0.range.lowerBound <= range.lowerBound && range.lowerBound < $0.range.upperBound
	118	&& range.upperBound <= $0.range.upperBound
	119	}) {
	120	if Self.leafKinds.contains(child.kind) { return child }
	121	current = child
	122	}
	123	return nil
	124	}
	125
	126	/// Copies the path from the root down to `target`, swapping in `green`.
	127	func replacing(_ target: SyntaxNode, with green: GreenNode) -> GreenNode {
	128	var node = target
	129	var replacement = green
	130	while let parent = node.parent {
	131	var children = parent.green.children
	132	let index = children.firstIndex {
	133	if case .node(let child) = $0 { return child === node.green }
	134	return false
	135	}!
	136	children[index] = .node(replacement)
	137	replacement = GreenNode(kind: parent.kind, children: children)
	138	node = parent
	139	}
	140	return replacement
	141	}
	142
	143	// MARK: - Sections
	144
	145	/// Reparses from the top-level section before the edit up to the first later top-level
	146	/// heading that still ends the reparsed run, then reuses the old sections from there.
	147	/// Top-level sections parse independently: blocks and drawers never cross a heading.
	148	func reparseSections(_ old: OrgTree) -> OrgTree? {
	149	let units = old.root.children
	150	guard !units.isEmpty else { return nil }
	151	let anchor = max(0, lineStart(oldText, edit.range.lowerBound) - 1)
	152	guard let startIndex = units.firstIndex(where: { $0.range.contains(anchor) }) else { return nil }
	153	let start = units[startIndex].range.lowerBound
	154	var endIndex = units.firstIndex { $0.range.lowerBound > edit.range.upperBound } ?? units.count
	155	let newLength = newText.utf16.count
	156	while true {
	157	let newEnd = endIndex < units.count ? units[endIndex].range.lowerBound + delta : newLength
	158	var parser = Parser(text: String(slice(newText, start, newEnd)), settings: old.settings)
	159	let window = parser.run().green
	160	if endIndex == units.count \|\| endsRun(window, next: units[endIndex].green) {
	161	let children = units[..<startIndex].map { GreenElement.node($0.green) }
	162	+ window.children
	163	+ units[endIndex...].map { GreenElement.node($0.green) }
	164	return OrgTree(green: GreenNode(kind: .document, children: children), settings: old.settings)
	165	}
	166	endIndex += 1
	167	}
	168	}
	169
	170	/// Whether the heading of `next` ends the last section of the reparsed window.
	171	func endsRun(_ window: GreenNode, next: GreenNode) -> Bool {
	172	guard case .node(let last)? = window.children.last, last.kind == .section else { return true }
	173	return headingLevel(next) <= headingLevel(last)
	174	}
	175
	176	func headingLevel(_ section: GreenNode) -> Int {
	177	for case .node(let heading) in section.children where heading.kind == .heading {
	178	for case .token(let token) in heading.children where token.kind == .stars {
	179	return token.text.count
	180	}
	181	}
	182	return 0
	183	}
	184
	185	// MARK: - Text helpers
	186
	187	func classes(_ text: Substring) -> [ClassifiedLine] {
	188	splitRawLines(String(text)).map { classifyLine($0.content) }
	189	}
	190
	191	func slice(_ text: String, _ from: Int, _ to: Int) -> Substring {
	192	text[String.Index(utf16Offset: from, in: text)..<String.Index(utf16Offset: to, in: text)]
	193	}
	194
	195	/// UTF-16 offset of the start of the line containing `offset`.
	196	func lineStart(_ text: String, _ offset: Int) -> Int {
	197	let utf16 = text.utf16
	198	var index = utf16.index(utf16.startIndex, offsetBy: offset)
	199	while index > utf16.startIndex {
	200	let before = utf16.index(before: index)
	201	if utf16[before] == 0x0A { break }
	202	index = before
	203	}
	204	return utf16.distance(from: utf16.startIndex, to: index)
	205	}
	206
	207	/// UTF-16 offset just past the newline ending the line containing `offset`.
	208	func lineEnd(_ text: String, _ offset: Int) -> Int {
	209	let utf16 = text.utf16
	210	var index = utf16.index(utf16.startIndex, offsetBy: offset)
	211	while index < utf16.endIndex {
	212	let current = utf16[index]
	213	index = utf16.index(after: index)
	214	if current == 0x0A { break }
	215	}
	216	return utf16.distance(from: utf16.startIndex, to: index)
	217	}
	218	}
	219
	220	extension Parser {
	221	/// One element of `kind` parsed from exactly `text`, or nil if `text` doesn't parse as a
	222	/// single element of that kind.
	223	static func reparseElement(_ kind: SyntaxKind, text: String, settings: OrgSettings) -> GreenNode? {
	224	var parser = Parser(text: text, settings: settings)
	225	let count = parser.lines.count
	226	guard count > 0 else { return nil }
	227	switch kind {
	228	case .paragraph:
	229	parser.paragraph(limit: count, floor: nil)
	230	case .heading:
	231	guard count == 1 else { return nil }
	232	parser.headingLine(parser.lines[0])
	233	parser.i = 1
	234	case .tableRow:
	235	guard count == 1 else { return nil }
	236	parser.tableRow()
	237	case .planning, .clock, .keyword, .affiliatedKeyword, .horizontalRule, .nodeProperty:
	238	guard count == 1 else { return nil }
	239	parser.single(kind)
	240	case .block, .comment, .fixedWidth, .footnoteDefinition:
	241	parser.element(limit: count, floor: nil)
	242	default:
	243	return nil
	244	}
	245	guard parser.i == count else { return nil }
	246	let green = parser.builder.build()
	247	return green.kind == kind ? green : nil
	248	}
	249	}

Sources/OrgCore/Parser/Parser.swift +19 −4

	@@ -16,10 +16,25 @@ struct Parser {
16	var i = 0	16	var i = 0
17		17
18	init(text: String, defaults: OrgSettings) {	18	init(text: String, defaults: OrgSettings) {
19	lines = splitRawLines(text)	19	let lines = splitRawLines(text)
20	info = lines.map { classifyLine($0.content) }	20	let info = lines.map { classifyLine($0.content) }
21	blockEnds = Parser.matchEnds(info)	21	let ends = Parser.matchEnds(info)
22	settings = SettingsScanner.scan(lines: lines, info: info, blockEnds: blockEnds, defaults: defaults)	22	let settings = SettingsScanner.scan(lines: lines, info: info, blockEnds: ends, defaults: defaults)
		23	self.init(lines: lines, info: info, blockEnds: ends, settings: settings)
		24	}
		25
		26	/// Parses part of a document with the settings already read from the whole file.
		27	init(text: String, settings: OrgSettings) {
		28	let lines = splitRawLines(text)
		29	let info = lines.map { classifyLine($0.content) }
		30	self.init(lines: lines, info: info, blockEnds: Parser.matchEnds(info), settings: settings)
		31	}
		32
		33	private init(lines: [RawLine], info: [ClassifiedLine], blockEnds: [Int: Int], settings: OrgSettings) {
		34	self.lines = lines
		35	self.info = info
		36	self.blockEnds = blockEnds
		37	self.settings = settings
23	}	38	}
24		39
25	static func matchEnds(_ info: [ClassifiedLine]) -> [Int: Int] {	40	static func matchEnds(_ info: [ClassifiedLine]) -> [Int: Int] {

Tests/OrgCoreTests/IncrementalTests.swift added +102

@@ -0,0 +1,102 @@
	1	import Foundation
	2	import Testing
	3	@testable import OrgCore
	4
	5	/// Reparses `text` after an edit, checks the result against a full parse, and returns the
	6	/// strategy used.
	7	@discardableResult
	8	func checkReparse(_ text: String, _ range: Range<Int>, _ replacement: String) -> ReparseStrategy {
	9	let old = OrgParser.parse(text)
	10	let edit = TextEdit(range: range, replacement: replacement)
	11	let result = OrgParser.reparseWithStrategy(old, oldText: text, edit: edit)
	12	let expected = OrgParser.parse(edit.apply(to: text))
	13	#expect(result.tree.text == expected.text)
	14	#expect(result.tree.green == expected.green)
	15	return result.strategy
	16	}
	17
	18	/// UTF-16 offsets of every unicode scalar boundary.
	19	func scalarOffsets(_ text: String) -> [Int] {
	20	var offsets = [0]
	21	var offset = 0
	22	for scalar in text.unicodeScalars {
	23	offset += scalar.utf16.count
	24	offsets.append(offset)
	25	}
	26	return offsets
	27	}
	28
	29	struct IncrementalTests {
	30	@Test func applyKeepsCRLFHalves() {
	31	#expect(TextEdit(range: 1..<1, replacement: "x").apply(to: "\r\n") == "\rx\n")
	32	#expect(TextEdit(range: 1..<3, replacement: "").apply(to: "a😀b") == "ab")
	33	}
	34
	35	@Test func typingInAParagraphReparsesOneElement() {
	36	#expect(checkReparse("* a\nhello world\n* b\n", 10..<10, "big ") == .element)
	37	#expect(checkReparse("- item one\n - two\n", 8..<8, "x") == .element)
	38	}
	39
	40	@Test func typingAtTheStartOfAnItemParagraph() {
	41	#expect(checkReparse(" >\n1. one", 6..<6, " ") != .element)
	42	}
	43
	44	@Test func changingATodoKeywordReparsesOneElement() {
	45	#expect(checkReparse("* TODO a\nbody\n", 2..<6, "DONE") == .element)
	46	}
	47
	48	@Test func newHeadingReparsesSections() {
	49	#expect(checkReparse("* a\nx\ny\n* b\n", 6..<6, "* c\n") == .sections)
	50	}
	51
	52	@Test func joiningParagraphsReparsesSections() {
	53	#expect(checkReparse("a\n\nb\n", 1..<2, "") == .sections)
	54	}
	55
	56	@Test func runGrowsUntilAHeadingEndsIt() {
	57	#expect(checkReparse(" a\n b\n* c\n", 0..<0, "* x\n") == .sections)
	58	}
	59
	60	@Test func settingsLineForcesFullParse() {
	61	#expect(checkReparse("#+TODO: A \| B\n* A x\n", 8..<9, "C") == .full)
	62	}
	63
	64	@Test func exposingAKeywordInsideABlockForcesFullParse() {
	65	let text = "#+begin_example\n#+TODO: X\n#+end_example\n* X a\n"
	66	#expect(checkReparse(text, 26..<40, "") == .full)
	67	}
	68
	69	@Test func editsAtTheEdges() {
	70	checkReparse("", 0..<0, "* a\n")
	71	checkReparse("* a\n", 0..<4, "")
	72	checkReparse("* a", 3..<3, "\n")
	73	checkReparse("a\r\nb\r\n", 1..<1, "x")
	74	}
	75
	76	/// Set `ORGSTAR_FUZZ_EDITS` to run more (the phase 1 gate is 100,000 in a release build) and
	77	/// `ORGSTAR_FUZZ_SEED` to try another sequence.
	78	@Test func incrementalEqualsFull() {
	79	let environment = ProcessInfo.processInfo.environment
	80	var rng = SeededGenerator(state: UInt64(environment["ORGSTAR_FUZZ_SEED"] ?? "") ?? 20261005)
	81	let count = Int(environment["ORGSTAR_FUZZ_EDITS"] ?? "") ?? 3_000
	82	let small = ["a", " ", "*", "/", "=", "[", "]", "<", ">", "-", ":", "\|", "#", "+", "\n", "\r\n", "😀", "é"]
	83	for n in 0..<count {
	84	let text = randomDocument(&rng)
	85	let offsets = scalarOffsets(text)
	86	let a = offsets.randomElement(using: &rng)!
	87	let later = offsets.filter { $0 >= a && $0 <= a + 12 }
	88	let b = Bool.random(using: &rng) ? a : later.randomElement(using: &rng)!
	89	let replacement: String
	90	if Int.random(in: 0..<10, using: &rng) < 7 {
	91	replacement = (0..<Int.random(in: 0...2, using: &rng)).map { _ in small.randomElement(using: &rng)! }.joined()
	92	} else {
	93	replacement = fragments.randomElement(using: &rng)! + (Bool.random(using: &rng) ? "\n" : "")
	94	}
	95	let old = OrgParser.parse(text)
	96	let edit = TextEdit(range: a..<b, replacement: replacement)
	97	let tree = OrgParser.reparse(old, oldText: text, edit: edit)
	98	let expected = OrgParser.parse(edit.apply(to: text))
	99	#expect(tree.green == expected.green, "edit \(n): \(a)..<\(b) \(replacement.debugDescription) in \(text.debugDescription)")
	100	}
	101	}
	102	}

Commit 1a36bd71b5

Commit `1a36bd71b5`