Commit 17f903817b
17f903817bd22d0a767ba98d8caff718ddf7d7d7
parent: 845710a22b
Verified · cmc
cmc <hello@cleberg.net> · 2026-10-04 23:15 UTC
Add green and red syntax tree
Layout: unified · split
Sources/OrgCore/Syntax/GreenTree.swift
added
+87
| @@ -0,0 +1,87 @@ |
| 1 | public struct GreenToken: Sendable, Equatable { |
| 2 | public let kind: SyntaxKind |
| 3 | public let text: String |
| 4 | /// Length in UTF-16 code units. |
| 5 | public let length: Int |
| 6 | |
| 7 | public init(kind: SyntaxKind, text: String) { |
| 8 | self.kind = kind |
| 9 | self.text = text |
| 10 | self.length = text.utf16.count |
| 11 | } |
| 12 | } |
| 13 | |
| 14 | /// An immutable node. Stores only kind, children and length, so unchanged subtrees can be |
| 15 | /// shared between versions of a document. |
| 16 | public final class GreenNode: Sendable, Equatable { |
| 17 | public let kind: SyntaxKind |
| 18 | public let children: [GreenElement] |
| 19 | /// Length in UTF-16 code units. |
| 20 | public let length: Int |
| 21 | |
| 22 | public init(kind: SyntaxKind, children: [GreenElement]) { |
| 23 | self.kind = kind |
| 24 | self.children = children |
| 25 | self.length = children.reduce(0) { $0 + $1.length } |
| 26 | } |
| 27 | |
| 28 | public static func == (lhs: GreenNode, rhs: GreenNode) -> Bool { |
| 29 | lhs === rhs || (lhs.kind == rhs.kind && lhs.children == rhs.children) |
| 30 | } |
| 31 | |
| 32 | public var text: String { |
| 33 | var out = "" |
| 34 | write(to: &out) |
| 35 | return out |
| 36 | } |
| 37 | |
| 38 | func write(to out: inout String) { |
| 39 | for child in children { |
| 40 | switch child { |
| 41 | case .node(let node): node.write(to: &out) |
| 42 | case .token(let token): out += token.text |
| 43 | } |
| 44 | } |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | public enum GreenElement: Sendable, Equatable { |
| 49 | case node(GreenNode) |
| 50 | case token(GreenToken) |
| 51 | |
| 52 | public var length: Int { |
| 53 | switch self { |
| 54 | case .node(let node): node.length |
| 55 | case .token(let token): token.length |
| 56 | } |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | struct GreenBuilder { |
| 61 | private var stack: [(kind: SyntaxKind, children: [GreenElement])] = [] |
| 62 | private var root: GreenNode? |
| 63 | |
| 64 | mutating func start(_ kind: SyntaxKind) { |
| 65 | stack.append((kind, [])) |
| 66 | } |
| 67 | |
| 68 | mutating func token(_ kind: SyntaxKind, _ text: some StringProtocol) { |
| 69 | guard !text.isEmpty else { return } |
| 70 | stack[stack.count - 1].children.append(.token(GreenToken(kind: kind, text: String(text)))) |
| 71 | } |
| 72 | |
| 73 | mutating func finish() { |
| 74 | let (kind, children) = stack.removeLast() |
| 75 | let node = GreenNode(kind: kind, children: children) |
| 76 | if stack.isEmpty { |
| 77 | root = node |
| 78 | } else { |
| 79 | stack[stack.count - 1].children.append(.node(node)) |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | func build() -> GreenNode { |
| 84 | precondition(stack.isEmpty, "unfinished nodes") |
| 85 | return root! |
| 86 | } |
| 87 | } |
Sources/OrgCore/Syntax/SyntaxKind.swift
added
+12
| @@ -0,0 +1,12 @@ |
| 1 | public enum SyntaxKind: String, Sendable { |
| 2 | // Tokens |
| 3 | case text, newline, whitespace |
| 4 | case stars, todoKeyword, priority, title, tags |
| 5 | |
| 6 | // Nodes |
| 7 | case document, zerothSection, section, heading |
| 8 | case planning, propertyDrawer, nodeProperty, drawer, clock |
| 9 | case paragraph, plainList, item, table, tableRow, tableFormula |
| 10 | case block, dynamicBlock, keyword, affiliatedKeyword |
| 11 | case comment, fixedWidth, horizontalRule, footnoteDefinition |
| 12 | } |
Sources/OrgCore/Syntax/SyntaxNode.swift
added
+51
| @@ -0,0 +1,51 @@ |
| 1 | /// A view of a green node at an absolute offset, with a parent link. Created on demand. |
| 2 | public final class SyntaxNode: Sendable { |
| 3 | public let green: GreenNode |
| 4 | public let offset: Int |
| 5 | public let parent: SyntaxNode? |
| 6 | |
| 7 | init(green: GreenNode, offset: Int, parent: SyntaxNode?) { |
| 8 | self.green = green |
| 9 | self.offset = offset |
| 10 | self.parent = parent |
| 11 | } |
| 12 | |
| 13 | public var kind: SyntaxKind { green.kind } |
| 14 | public var range: Range<Int> { offset..<(offset + green.length) } |
| 15 | public var text: String { green.text } |
| 16 | |
| 17 | public var children: [SyntaxNode] { |
| 18 | var result: [SyntaxNode] = [] |
| 19 | var at = offset |
| 20 | for child in green.children { |
| 21 | if case .node(let node) = child { |
| 22 | result.append(SyntaxNode(green: node, offset: at, parent: self)) |
| 23 | } |
| 24 | at += child.length |
| 25 | } |
| 26 | return result |
| 27 | } |
| 28 | |
| 29 | public var tokens: [SyntaxToken] { |
| 30 | var result: [SyntaxToken] = [] |
| 31 | var at = offset |
| 32 | for child in green.children { |
| 33 | if case .token(let token) = child { |
| 34 | result.append(SyntaxToken(kind: token.kind, text: token.text, range: at..<(at + token.length))) |
| 35 | } |
| 36 | at += child.length |
| 37 | } |
| 38 | return result |
| 39 | } |
| 40 | |
| 41 | /// This node and every node below it, in document order. |
| 42 | public func descendants() -> [SyntaxNode] { |
| 43 | [self] + children.flatMap { $0.descendants() } |
| 44 | } |
| 45 | } |
| 46 | |
| 47 | public struct SyntaxToken: Sendable, Equatable { |
| 48 | public let kind: SyntaxKind |
| 49 | public let text: String |
| 50 | public let range: Range<Int> |
| 51 | } |
Tests/OrgCoreTests/SyntaxTreeTests.swift
added
+45
| @@ -0,0 +1,45 @@ |
| 1 | import Testing |
| 2 | @testable import OrgCore |
| 3 | |
| 4 | struct SyntaxTreeTests { |
| 5 | func sample() -> GreenNode { |
| 6 | var b = GreenBuilder() |
| 7 | b.start(.document) |
| 8 | b.start(.paragraph) |
| 9 | b.token(.text, "hé😀") |
| 10 | b.token(.newline, "\n") |
| 11 | b.finish() |
| 12 | b.token(.newline, "\r\n") |
| 13 | b.finish() |
| 14 | return b.build() |
| 15 | } |
| 16 | |
| 17 | @Test func lengthsAreUTF16() { |
| 18 | let green = sample() |
| 19 | #expect(green.length == 4 + 1 + 2) |
| 20 | #expect(green.text == "hé😀\n\r\n") |
| 21 | } |
| 22 | |
| 23 | @Test func redNodesCarryOffsets() { |
| 24 | let root = SyntaxNode(green: sample(), offset: 0, parent: nil) |
| 25 | let paragraph = root.children[0] |
| 26 | #expect(paragraph.kind == .paragraph) |
| 27 | #expect(paragraph.range == 0..<5) |
| 28 | #expect(paragraph.parent === root) |
| 29 | #expect(root.tokens.map(\.range) == [5..<7]) |
| 30 | #expect(paragraph.tokens.map(\.kind) == [.text, .newline]) |
| 31 | } |
| 32 | |
| 33 | @Test func builderSkipsEmptyTokens() { |
| 34 | var b = GreenBuilder() |
| 35 | b.start(.document) |
| 36 | b.token(.whitespace, "") |
| 37 | b.finish() |
| 38 | #expect(b.build().children.isEmpty) |
| 39 | } |
| 40 | |
| 41 | @Test func descendantsArePreorder() { |
| 42 | let root = SyntaxNode(green: sample(), offset: 0, parent: nil) |
| 43 | #expect(root.descendants().map(\.kind) == [.document, .paragraph]) |
| 44 | } |
| 45 | } |
docs/plans/2026-10-04-orgcore-parser.md
+3 −3
| @@ -209,7 +209,7 @@ struct SyntaxTreeTests { |
| 209 | 209 | |
| 210 | 210 | @Test func lengthsAreUTF16() { |
| 211 | 211 | let green = sample() |
| 212 | | #expect(green.length == 3 + 2 + 1 + 2) |
| 212 | #expect(green.length == 4 + 1 + 2) |
| 213 | 213 | #expect(green.text == "hé😀\n\r\n") |
| 214 | 214 | } |
| 215 | 215 | |
| @@ -217,9 +217,9 @@ struct SyntaxTreeTests { |
| 217 | 217 | let root = SyntaxNode(green: sample(), offset: 0, parent: nil) |
| 218 | 218 | let paragraph = root.children[0] |
| 219 | 219 | #expect(paragraph.kind == .paragraph) |
| 220 | | #expect(paragraph.range == 0..<6) |
| 220 | #expect(paragraph.range == 0..<5) |
| 221 | 221 | #expect(paragraph.parent === root) |
| 222 | | #expect(root.tokens.map(\.range) == [6..<8]) |
| 222 | #expect(root.tokens.map(\.range) == [5..<7]) |
| 223 | 223 | #expect(paragraph.tokens.map(\.kind) == [.text, .newline]) |
| 224 | 224 | } |
| 225 | 225 | |