krz/orgstar

Workspace and index !6

merged merged by cmc on 2026-10-05 00:18 UTC · krz/orgstar:phase1-workspace-index into main

14 files changed, +2545 −2

Layout: unified · split

Package.resolved added +15
@@ -0,0 +1,15 @@
1{
2 "originHash" : "c31b92d42ae4ae7fcf15c68db14883c5ded79cfac8c0d3c5f597eeac28e9899f",
3 "pins" : [
4 {
5 "identity" : "grdb.swift",
6 "kind" : "remoteSourceControl",
7 "location" : "https://github.com/groue/GRDB.swift",
8 "state" : {
9 "revision" : "b83108d10f42680d78f23fe4d4d80fc88dab3212",
10 "version" : "7.11.1"
11 }
12 }
13 ],
14 "version" : 3
15}
Package.swift +11 −2
@@ -6,12 +6,21 @@ let package = Package(
6 platforms: [.macOS(.v26), .iOS(.v26)], 6 platforms: [.macOS(.v26), .iOS(.v26)],
7 products: [ 7 products: [
8 .library(name: "OrgCore", targets: ["OrgCore"]), 8 .library(name: "OrgCore", targets: ["OrgCore"]),
9 .library(name: "OrgDocument", targets: ["OrgDocument"]) 9 .library(name: "OrgDocument", targets: ["OrgDocument"]),
10 .library(name: "OrgIndex", targets: ["OrgIndex"]),
11 .library(name: "OrgWorkspace", targets: ["OrgWorkspace"])
12 ],
13 dependencies: [
14 .package(url: "https://github.com/groue/GRDB.swift", from: "7.11.1")
10 ], 15 ],
11 targets: [ 16 targets: [
12 .target(name: "OrgCore"), 17 .target(name: "OrgCore"),
13 .target(name: "OrgDocument", dependencies: ["OrgCore"]), 18 .target(name: "OrgDocument", dependencies: ["OrgCore"]),
19 .target(name: "OrgIndex", dependencies: ["OrgCore", .product(name: "GRDB", package: "GRDB.swift")]),
20 .target(name: "OrgWorkspace", dependencies: ["OrgIndex"]),
14 .testTarget(name: "OrgCoreTests", dependencies: ["OrgCore"]), 21 .testTarget(name: "OrgCoreTests", dependencies: ["OrgCore"]),
15 .testTarget(name: "OrgDocumentTests", dependencies: ["OrgDocument"]) 22 .testTarget(name: "OrgDocumentTests", dependencies: ["OrgDocument"]),
23 .testTarget(name: "OrgIndexTests", dependencies: ["OrgIndex"]),
24 .testTarget(name: "OrgWorkspaceTests", dependencies: ["OrgWorkspace"])
16 ] 25 ]
17) 26)
Sources/OrgIndex/FileRecord.swift added +209
@@ -0,0 +1,209 @@
1import CryptoKit
2import Foundation
3import OrgCore
4
5public enum FileKind: String, Sendable {
6 case org
7 case archive
8 /// A Syncthing conflict copy: listed, never indexed for search, agenda or IDs.
9 case conflict
10}
11
12/// Everything that decides what an index row means. Bump `version` when a semantic setting
13/// changes, so files indexed under the old settings are indexed again.
14public struct IndexSettings: Sendable, Equatable {
15 public var version: Int
16 public var org: OrgSettings
17 public var semantic: SemanticSettings
18
19 public init(version: Int = 1, org: OrgSettings = .default, semantic: SemanticSettings = .default) {
20 self.version = version
21 self.org = org
22 self.semantic = semantic
23 }
24}
25
26public struct TagRow: Sendable, Equatable {
27 public let name: String
28 public let inherited: Bool
29}
30
31public struct PropertyRow: Sendable, Equatable {
32 /// Upper-cased.
33 public let key: String
34 public let value: String
35 public let inherited: Bool
36}
37
38public struct TimestampRow: Sendable, Equatable {
39 public enum Kind: String, Sendable { case scheduled, deadline, closed, active, inactive }
40 public let kind: Kind
41 /// `2026-10-04` or `2026-10-04T10:00`.
42 public let start: String
43 public let end: String?
44 /// As written: `+1w`, `.+2d/3d`.
45 public let repeater: String?
46 /// As written: `-2d`, `--1w`.
47 public let warning: String?
48}
49
50public struct ClockRow: Sendable, Equatable {
51 public let start: String
52 public let end: String?
53 public let minutes: Int?
54}
55
56public struct LinkRow: Sendable, Equatable {
57 /// The scheme before the first colon (`id`, `file`, `https`), or `fuzzy`.
58 public let type: String
59 public let target: String
60}
61
62public struct HeadingRecord: Sendable, Equatable {
63 /// Position in document order; parents refer to it.
64 public let ordinal: Int
65 public let parent: Int?
66 /// The section's UTF-16 range in the file's text.
67 public let start: Int
68 public let end: Int
69 public let level: Int
70 public let todo: String?
71 public let isDone: Bool
72 public let priority: String?
73 public let title: String
74 public let outlinePath: [String]
75 public let orgID: String?
76 /// Tagged `ARCHIVE`, directly or by inheritance.
77 public let archived: Bool
78 /// The heading's own text, without the heading line and child sections.
79 public let body: String
80 public let tags: [TagRow]
81 public let properties: [PropertyRow]
82 public let timestamps: [TimestampRow]
83 public let clocks: [ClockRow]
84 public let links: [LinkRow]
85}
86
87/// One file's index rows, computed without touching the database.
88public struct FileRecord: Sendable, Equatable {
89 public let path: String
90 public let root: String
91 public let kind: FileKind
92 public let size: Int
93 public let mtime: Double
94 /// SHA-256 of the bytes, hex.
95 public let hash: String
96 public let settingsVersion: Int
97 public let headings: [HeadingRecord]
98
99 public static func hash(_ bytes: [UInt8]) -> String {
100 SHA256.hash(data: Data(bytes)).map { String(format: "%02x", $0) }.joined()
101 }
102
103 public init(path: String, root: String, kind: FileKind, bytes: [UInt8], mtime: Double, settings: IndexSettings) {
104 self.path = path
105 self.root = root
106 self.kind = kind
107 self.size = bytes.count
108 self.mtime = mtime
109 self.hash = Self.hash(bytes)
110 self.settingsVersion = settings.version
111 let source = SourceText(bytes: bytes)
112 headings = kind == .conflict ? [] : Self.headings(source.text, settings: settings)
113 }
114
115 static func headings(_ text: String, settings: IndexSettings) -> [HeadingRecord] {
116 let model = DocumentModel(tree: OrgParser.parse(text, defaults: settings.org), settings: settings.semantic)
117 let utf16 = text.utf16
118 func slice(_ range: Range<Int>) -> String {
119 let start = utf16.index(utf16.startIndex, offsetBy: range.lowerBound)
120 let end = utf16.index(utf16.startIndex, offsetBy: range.upperBound)
121 return String(text[start..<end])
122 }
123 var firstChild: [Int: Int] = [:]
124 for (index, heading) in model.headings.enumerated() {
125 if let parent = heading.parent, firstChild[parent] == nil { firstChild[parent] = index }
126 }
127
128 return model.headings.enumerated().map { index, heading in
129 let bodyEnd = firstChild[index].map { model.headings[$0].sectionRange.lowerBound } ?? heading.sectionRange.upperBound
130 let tags = model.tags(of: index)
131 return HeadingRecord(
132 ordinal: index,
133 parent: heading.parent,
134 start: heading.sectionRange.lowerBound,
135 end: heading.sectionRange.upperBound,
136 level: heading.level,
137 todo: heading.todo,
138 isDone: heading.isDone,
139 priority: heading.priority,
140 title: heading.title,
141 outlinePath: model.outlinePath(of: index),
142 orgID: heading.id,
143 archived: tags.contains { $0.value == "ARCHIVE" },
144 body: slice(heading.headingRange.upperBound..<bodyEnd),
145 tags: tags.map { TagRow(name: $0.value, inherited: $0.source != .heading(index)) },
146 properties: properties(model, index),
147 timestamps: timestamps(heading),
148 clocks: heading.clocks.map {
149 ClockRow(start: format($0.start), end: $0.end.map(format), minutes: $0.minutes)
150 },
151 links: heading.links.map { target in
152 let scheme = target.prefix { $0 != ":" }
153 let isScheme = scheme.count < target.count && !scheme.isEmpty && scheme.allSatisfy { $0.isLetter || $0 == "-" }
154 return LinkRow(type: isScheme ? scheme.lowercased() : "fuzzy", target: target)
155 }
156 )
157 }
158 }
159
160 /// Own properties, plus inherited ones for keys that inherit.
161 static func properties(_ model: DocumentModel, _ index: Int) -> [PropertyRow] {
162 var keys: [String] = []
163 func add(_ entries: [Property], inheritedOnly: Bool) {
164 for entry in entries {
165 let key = entry.key.uppercased()
166 if !keys.contains(key), !inheritedOnly || model.inherits(key) { keys.append(key) }
167 }
168 }
169 add(model.headings[index].properties, inheritedOnly: false)
170 add(model.fileProperties, inheritedOnly: true)
171 for ancestor in model.ancestors(of: index) { add(model.headings[ancestor].properties, inheritedOnly: true) }
172 return keys.compactMap { key in
173 model.property(key, of: index).map {
174 PropertyRow(key: key, value: $0.value, inherited: $0.source != .heading(index))
175 }
176 }
177 }
178
179 static func timestamps(_ heading: HeadingInfo) -> [TimestampRow] {
180 var rows: [TimestampRow] = []
181 func add(_ stamp: Timestamp?, _ kind: TimestampRow.Kind) {
182 guard let stamp else { return }
183 rows.append(TimestampRow(
184 kind: kind,
185 start: format(stamp.start),
186 end: stamp.end.map(format),
187 repeater: stamp.repeater.map {
188 $0.kind.rawValue + format($0.interval) + ($0.habitDeadline.map { "/" + format($0) } ?? "")
189 },
190 warning: stamp.warning.map { ($0.firstOccurrenceOnly ? "--" : "-") + format($0.interval) }
191 ))
192 }
193 add(heading.scheduled, .scheduled)
194 add(heading.deadline, .deadline)
195 add(heading.closed, .closed)
196 for stamp in heading.timestamps { add(stamp, stamp.active ? .active : .inactive) }
197 return rows
198 }
199
200 static func format(_ point: Timestamp.Point) -> String {
201 let date = String(format: "%04d-%02d-%02d", point.year, point.month, point.day)
202 guard let hour = point.hour, let minute = point.minute else { return date }
203 return date + String(format: "T%02d:%02d", hour, minute)
204 }
205
206 static func format(_ interval: Timestamp.Interval) -> String {
207 "\(interval.value)\(interval.unit.rawValue)"
208 }
209}
Sources/OrgIndex/IndexStore.swift added +315
@@ -0,0 +1,315 @@
1import Foundation
2import GRDB
3
4/// What the index knows about a file without reading it.
5public struct FileState: Sendable, Equatable {
6 public let kind: FileKind
7 public let size: Int
8 public let mtime: Double
9 public let hash: String
10 public let settingsVersion: Int
11}
12
13/// A heading as found by a query. `contentHash` is the hash of the text the offsets refer to,
14/// so a caller can tell whether they still apply to an open buffer.
15public struct HeadingLocation: Sendable, Equatable {
16 public let path: String
17 public let ordinal: Int
18 public let title: String
19 public let start: Int
20 public let contentHash: String
21}
22
23/// One reconciliation's worth of changes, applied in a single transaction.
24public struct IndexChange: Sendable {
25 public var records: [FileRecord] = []
26 /// Unchanged content with a new modification time.
27 public var touches: [(path: String, mtime: Double)] = []
28 /// Renamed files whose content didn't change.
29 public var moves: [(from: String, to: String, mtime: Double)] = []
30 public var removals: [String] = []
31
32 public init() {}
33
34 public var isEmpty: Bool { records.isEmpty && touches.isEmpty && moves.isEmpty && removals.isEmpty }
35}
36
37/// The SQLite index. A cache: deleting it loses nothing that the files don't hold.
38public final class IndexStore: Sendable {
39 let database: DatabaseQueue
40
41 /// `path` nil opens an in-memory index.
42 public init(path: String? = nil) throws {
43 database = try path.map { try DatabaseQueue(path: $0) } ?? DatabaseQueue()
44 try Self.migrator.migrate(database)
45 }
46
47 static var migrator: DatabaseMigrator {
48 var migrator = DatabaseMigrator()
49 migrator.registerMigration("v1") { db in
50 try db.execute(sql: """
51 CREATE TABLE files (
52 id INTEGER PRIMARY KEY,
53 path TEXT NOT NULL UNIQUE,
54 root TEXT NOT NULL,
55 kind TEXT NOT NULL,
56 size INTEGER NOT NULL,
57 mtime REAL NOT NULL,
58 hash TEXT NOT NULL,
59 settings_version INTEGER NOT NULL,
60 parsed_at REAL NOT NULL
61 );
62 CREATE INDEX files_root ON files(root);
63 CREATE TABLE headings (
64 id INTEGER PRIMARY KEY,
65 file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
66 ordinal INTEGER NOT NULL,
67 parent_ordinal INTEGER,
68 start_offset INTEGER NOT NULL,
69 end_offset INTEGER NOT NULL,
70 level INTEGER NOT NULL,
71 todo TEXT,
72 is_done INTEGER NOT NULL,
73 priority TEXT,
74 title TEXT NOT NULL,
75 outline_path TEXT NOT NULL,
76 org_id TEXT,
77 archived INTEGER NOT NULL
78 );
79 CREATE INDEX headings_file ON headings(file_id);
80 CREATE INDEX headings_org_id ON headings(org_id);
81 CREATE TABLE tags (
82 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
83 tag TEXT NOT NULL,
84 inherited INTEGER NOT NULL
85 );
86 CREATE INDEX tags_heading ON tags(heading_id);
87 CREATE TABLE properties (
88 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
89 key TEXT NOT NULL,
90 value TEXT NOT NULL,
91 inherited INTEGER NOT NULL
92 );
93 CREATE INDEX properties_heading ON properties(heading_id);
94 CREATE TABLE timestamps (
95 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
96 kind TEXT NOT NULL,
97 start_at TEXT NOT NULL,
98 end_at TEXT,
99 repeater TEXT,
100 warning TEXT
101 );
102 CREATE INDEX timestamps_heading ON timestamps(heading_id);
103 CREATE TABLE clocks (
104 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
105 start_at TEXT NOT NULL,
106 end_at TEXT,
107 minutes INTEGER
108 );
109 CREATE INDEX clocks_heading ON clocks(heading_id);
110 CREATE TABLE links (
111 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
112 type TEXT NOT NULL,
113 target TEXT NOT NULL
114 );
115 CREATE INDEX links_heading ON links(heading_id);
116 CREATE VIRTUAL TABLE headings_fts USING fts5(title, body, tokenize = 'unicode61 remove_diacritics 2');
117 CREATE TRIGGER headings_fts_delete AFTER DELETE ON headings BEGIN
118 DELETE FROM headings_fts WHERE rowid = old.id;
119 END;
120 """)
121 }
122 return migrator
123 }
124
125 // MARK: - Writing
126
127 public func write(_ record: FileRecord) throws {
128 var change = IndexChange()
129 change.records = [record]
130 try apply(change)
131 }
132
133 public func apply(_ change: IndexChange) throws {
134 guard !change.isEmpty else { return }
135 try database.write { db in
136 for path in change.removals {
137 try db.execute(sql: "DELETE FROM files WHERE path = ?", arguments: [path])
138 }
139 for move in change.moves {
140 try db.execute(sql: "UPDATE files SET path = ?, mtime = ? WHERE path = ?", arguments: [move.to, move.mtime, move.from])
141 }
142 for touch in change.touches {
143 try db.execute(sql: "UPDATE files SET mtime = ? WHERE path = ?", arguments: [touch.mtime, touch.path])
144 }
145 for record in change.records {
146 try insert(record, db)
147 }
148 }
149 }
150
151 private func insert(_ record: FileRecord, _ db: Database) throws {
152 try db.execute(sql: "DELETE FROM files WHERE path = ?", arguments: [record.path])
153 try db.execute(
154 sql: """
155 INSERT INTO files (path, root, kind, size, mtime, hash, settings_version, parsed_at)
156 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
157 """,
158 arguments: [record.path, record.root, record.kind.rawValue, record.size, record.mtime, record.hash,
159 record.settingsVersion, Date().timeIntervalSince1970]
160 )
161 let fileID = db.lastInsertedRowID
162 for heading in record.headings {
163 try db.execute(
164 sql: """
165 INSERT INTO headings (file_id, ordinal, parent_ordinal, start_offset, end_offset, level, todo,
166 is_done, priority, title, outline_path, org_id, archived)
167 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
168 """,
169 arguments: [fileID, heading.ordinal, heading.parent, heading.start, heading.end, heading.level, heading.todo,
170 heading.isDone, heading.priority, heading.title, heading.outlinePath.joined(separator: "\u{1F}"),
171 heading.orgID, heading.archived]
172 )
173 let id = db.lastInsertedRowID
174 try db.execute(sql: "INSERT INTO headings_fts (rowid, title, body) VALUES (?, ?, ?)", arguments: [id, heading.title, heading.body])
175 for tag in heading.tags {
176 try db.execute(sql: "INSERT INTO tags VALUES (?, ?, ?)", arguments: [id, tag.name, tag.inherited])
177 }
178 for property in heading.properties {
179 try db.execute(sql: "INSERT INTO properties VALUES (?, ?, ?, ?)", arguments: [id, property.key, property.value, property.inherited])
180 }
181 for stamp in heading.timestamps {
182 try db.execute(
183 sql: "INSERT INTO timestamps VALUES (?, ?, ?, ?, ?, ?)",
184 arguments: [id, stamp.kind.rawValue, stamp.start, stamp.end, stamp.repeater, stamp.warning]
185 )
186 }
187 for clock in heading.clocks {
188 try db.execute(sql: "INSERT INTO clocks VALUES (?, ?, ?, ?)", arguments: [id, clock.start, clock.end, clock.minutes])
189 }
190 for link in heading.links {
191 try db.execute(sql: "INSERT INTO links VALUES (?, ?, ?)", arguments: [id, link.type, link.target])
192 }
193 }
194 }
195
196 // MARK: - Reading
197
198 /// Indexed files under `root`, by path.
199 public func fileStates(root: String) throws -> [String: FileState] {
200 try database.read { db in
201 let rows = try Row.fetchAll(db, sql: "SELECT path, kind, size, mtime, hash, settings_version FROM files WHERE root = ?", arguments: [root])
202 var states: [String: FileState] = [:]
203 for row in rows {
204 states[row["path"]] = FileState(
205 kind: FileKind(rawValue: row["kind"]) ?? .org,
206 size: row["size"], mtime: row["mtime"], hash: row["hash"], settingsVersion: row["settings_version"]
207 )
208 }
209 return states
210 }
211 }
212
213 public func files() throws -> [(path: String, kind: FileKind)] {
214 try database.read { db in
215 try Row.fetchAll(db, sql: "SELECT path, kind FROM files ORDER BY path").map {
216 (path: $0["path"], kind: FileKind(rawValue: $0["kind"]) ?? .org)
217 }
218 }
219 }
220
221 // MARK: - Queries
222 //
223 // `overlay` holds records for open documents with unsaved edits, keyed by path. Their rows
224 // replace that file's indexed rows in every query.
225
226 /// Headings whose title or body contain every word of `query` as a word prefix.
227 public func search(_ query: String, overlay: [String: FileRecord] = [:], limit: Int = 50) throws -> [HeadingLocation] {
228 let terms = query.split(whereSeparator: \.isWhitespace).map(String.init)
229 guard !terms.isEmpty else { return [] }
230 let match = terms.map { "\"" + $0.replacingOccurrences(of: "\"", with: "\"\"") + "\"*" }.joined(separator: " ")
231 let indexed = try database.read { db in
232 try Row.fetchAll(
233 db,
234 sql: """
235 SELECT f.path, f.hash, h.ordinal, h.title, h.start_offset
236 FROM headings_fts
237 JOIN headings h ON h.id = headings_fts.rowid
238 JOIN files f ON f.id = h.file_id
239 WHERE headings_fts MATCH ?
240 ORDER BY bm25(headings_fts)
241 LIMIT ?
242 """,
243 arguments: [match, limit + overlay.count * 10]
244 ).map(location)
245 }
246 // Unsaved buffers get the same word-prefix matching as the full-text index.
247 let prefixes = terms.flatMap(Self.words)
248 let live = overlay.values.sorted { $0.path < $1.path }.flatMap { record in
249 record.headings.filter { heading in
250 let words = Self.words(heading.title + "\n" + heading.body)
251 return prefixes.allSatisfy { prefix in words.contains { $0.hasPrefix(prefix) } }
252 }.map { location(record, $0) }
253 }
254 return Array((live + indexed.filter { overlay[$0.path] == nil }).prefix(limit))
255 }
256
257 /// Headings with `:ID: id`. More than one means the ID is duplicated.
258 public func headings(withID id: String, overlay: [String: FileRecord] = [:]) throws -> [HeadingLocation] {
259 let indexed = try database.read { db in
260 try Row.fetchAll(
261 db,
262 sql: """
263 SELECT f.path, f.hash, h.ordinal, h.title, h.start_offset
264 FROM headings h JOIN files f ON f.id = h.file_id
265 WHERE h.org_id = ? ORDER BY f.path, h.ordinal
266 """,
267 arguments: [id]
268 ).map(location)
269 }
270 let live = overlay.values.sorted { $0.path < $1.path }.flatMap { record in
271 record.headings.filter { $0.orgID == id }.map { location(record, $0) }
272 }
273 return indexed.filter { overlay[$0.path] == nil } + live
274 }
275
276 /// IDs used by more than one heading.
277 public func duplicateIDs(overlay: [String: FileRecord] = [:]) throws -> [String: [HeadingLocation]] {
278 let indexed = try database.read { db in
279 try Row.fetchAll(
280 db,
281 sql: """
282 SELECT h.org_id, f.path, f.hash, h.ordinal, h.title, h.start_offset
283 FROM headings h JOIN files f ON f.id = h.file_id
284 WHERE h.org_id IN (SELECT org_id FROM headings WHERE org_id IS NOT NULL GROUP BY org_id HAVING count(*) > 1)
285 OR (h.org_id IS NOT NULL AND ? > 0)
286 ORDER BY f.path, h.ordinal
287 """,
288 arguments: [overlay.count]
289 ).map { (id: $0["org_id"] as String, location: location($0)) }
290 }
291 var byID: [String: [HeadingLocation]] = [:]
292 for row in indexed where overlay[row.location.path] == nil {
293 byID[row.id, default: []].append(row.location)
294 }
295 for record in overlay.values.sorted(by: { $0.path < $1.path }) {
296 for heading in record.headings {
297 if let id = heading.orgID { byID[id, default: []].append(location(record, heading)) }
298 }
299 }
300 return byID.filter { $0.value.count > 1 }
301 }
302
303 /// Lower-cased runs of letters and digits, as the unicode61 tokenizer splits them.
304 static func words(_ text: String) -> [String] {
305 text.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
306 }
307
308 private func location(_ row: Row) -> HeadingLocation {
309 HeadingLocation(path: row["path"], ordinal: row["ordinal"], title: row["title"], start: row["start_offset"], contentHash: row["hash"])
310 }
311
312 private func location(_ record: FileRecord, _ heading: HeadingRecord) -> HeadingLocation {
313 HeadingLocation(path: record.path, ordinal: heading.ordinal, title: heading.title, start: heading.start, contentHash: record.hash)
314 }
315}
Sources/OrgWorkspace/Discovery.swift added +110
@@ -0,0 +1,110 @@
1import Foundation
2import OrgIndex
3
4/// A folder the user added. Identified by its bookmark, so a moved folder is followed.
5public struct WorkspaceRoot: Sendable, Equatable, Codable {
6 public let bookmark: Data
7 /// The path when the bookmark was made or last refreshed, for display.
8 public var path: String
9
10 public init(url: URL) throws {
11 bookmark = try url.bookmarkData(options: [], includingResourceValuesForKeys: nil, relativeTo: nil)
12 path = url.standardizedFileURL.path
13 }
14
15 /// The folder's current location. `isStale` means the bookmark should be made again.
16 public func resolve() throws -> (url: URL, isStale: Bool) {
17 var isStale = false
18 let url = try URL(resolvingBookmarkData: bookmark, options: [], relativeTo: nil, bookmarkDataIsStale: &isStale)
19 _ = url.startAccessingSecurityScopedResource()
20 return (url.standardizedFileURL, isStale)
21 }
22}
23
24public struct DiscoveredFile: Sendable, Equatable {
25 public let url: URL
26 public let kind: FileKind
27 public let size: Int
28 public let mtime: Double
29}
30
31public struct Discovery: Sendable, Equatable {
32 public var files: [DiscoveredFile] = []
33 /// iCloud files not downloaded yet, by the name they will have.
34 public var placeholders: [URL] = []
35}
36
37public enum FileClassification: Sendable, Equatable {
38 case file(FileKind)
39 /// An iCloud placeholder (`.name.org.icloud`) for the file at the given name.
40 case placeholder(String)
41 case ignored
42}
43
44public enum WorkspaceScanner {
45 /// What a file name means to the workspace. Syncthing temp files, our own temp and backup
46 /// files, Emacs backups and auto-saves, and other hidden files are ignored.
47 public static func classify(_ name: String) -> FileClassification {
48 if name.hasPrefix(".") {
49 if name.hasSuffix(".icloud") {
50 let inner = String(name.dropFirst().dropLast(".icloud".count))
51 if case .file = classify(inner) { return .placeholder(inner) }
52 }
53 return .ignored
54 }
55 if name.hasSuffix("~") || name.hasPrefix("#") { return .ignored }
56 let conflict = name.contains(".sync-conflict-")
57 if name.hasSuffix(".org") { return .file(conflict ? .conflict : .org) }
58 if name.hasSuffix(".org_archive") { return .file(conflict ? .conflict : .archive) }
59 return .ignored
60 }
61
62 /// Every org file under `root`, skipping hidden folders (`.git`, `.stfolder`, `.stversions`).
63 public static func scan(_ root: URL) throws -> Discovery {
64 let keys: [URLResourceKey] = [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey]
65 guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: keys) else {
66 return Discovery()
67 }
68 var discovery = Discovery()
69 for case let url as URL in enumerator {
70 let values = try url.resourceValues(forKeys: Set(keys))
71 let name = url.lastPathComponent
72 if values.isDirectory == true {
73 if name.hasPrefix(".") { enumerator.skipDescendants() }
74 continue
75 }
76 switch classify(name) {
77 case .file(let kind):
78 discovery.files.append(DiscoveredFile(
79 url: url.standardizedFileURL,
80 kind: kind,
81 size: values.fileSize ?? 0,
82 mtime: values.contentModificationDate?.timeIntervalSince1970 ?? 0
83 ))
84 case .placeholder(let inner):
85 discovery.placeholders.append(url.deletingLastPathComponent().appendingPathComponent(inner).standardizedFileURL)
86 case .ignored:
87 break
88 }
89 }
90 discovery.files.sort { $0.url.path < $1.url.path }
91 return discovery
92 }
93}
94
95/// Which files feed the agenda: `.org` files (not archives or conflict copies) matching one of
96/// the globs, relative to their root. No globs means every `.org` file.
97public struct AgendaScope: Sendable, Equatable {
98 public var globs: [String]
99
100 public init(globs: [String] = []) {
101 self.globs = globs
102 }
103
104 public func contains(relativePath: String, kind: FileKind) -> Bool {
105 guard kind == .org else { return false }
106 guard !globs.isEmpty else { return true }
107 // Without FNM_PATHNAME, `*` also matches `/`, so `work/*` covers nested folders.
108 return globs.contains { fnmatch($0, relativePath, 0) == 0 }
109 }
110}
Sources/OrgWorkspace/FSEventsWatcher.swift added +78
@@ -0,0 +1,78 @@
1#if os(macOS)
2import CoreServices
3import Foundation
4
5public enum WatchEvent: Sendable, Equatable {
6 case changed(URL)
7 /// Events were dropped or coalesced for this folder, or a root moved: rescan it.
8 case rescan(URL)
9}
10
11/// FSEvents with file-level events. Delivers batches on a private queue.
12public final class FSEventsWatcher: @unchecked Sendable {
13 private let paths: [String]
14 private let latency: CFTimeInterval
15 private let handler: @Sendable ([WatchEvent]) -> Void
16 private let queue = DispatchQueue(label: "orgstar.fsevents")
17 private var stream: FSEventStreamRef?
18
19 public init(roots: [URL], latency: CFTimeInterval = 0.3, handler: @escaping @Sendable ([WatchEvent]) -> Void) {
20 paths = roots.map(\.path)
21 self.latency = latency
22 self.handler = handler
23 }
24
25 deinit {
26 stop()
27 }
28
29 public func start() {
30 guard stream == nil else { return }
31 var context = FSEventStreamContext(
32 version: 0, info: Unmanaged.passUnretained(self).toOpaque(), retain: nil, release: nil, copyDescription: nil
33 )
34 let flags = FSEventStreamCreateFlags(
35 kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagUseCFTypes
36 | kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot
37 )
38 let callback: FSEventStreamCallback = { _, info, count, eventPaths, eventFlags, _ in
39 guard let info else { return }
40 let watcher = Unmanaged<FSEventsWatcher>.fromOpaque(info).takeUnretainedValue()
41 let paths = unsafeBitCast(eventPaths, to: NSArray.self) as? [String] ?? []
42 watcher.deliver(paths: paths, flags: Array(UnsafeBufferPointer(start: eventFlags, count: count)))
43 }
44 guard let stream = FSEventStreamCreate(
45 nil, callback, &context, paths as CFArray, FSEventStreamEventId(kFSEventStreamEventIdSinceNow), latency, flags
46 ) else { return }
47 FSEventStreamSetDispatchQueue(stream, queue)
48 FSEventStreamStart(stream)
49 self.stream = stream
50 }
51
52 public func stop() {
53 guard let stream else { return }
54 FSEventStreamStop(stream)
55 FSEventStreamInvalidate(stream)
56 FSEventStreamRelease(stream)
57 self.stream = nil
58 }
59
60 private func deliver(paths eventPaths: [String], flags: [FSEventStreamEventFlags]) {
61 let rescanFlags = FSEventStreamEventFlags(
62 kFSEventStreamEventFlagMustScanSubDirs | kFSEventStreamEventFlagUserDropped
63 | kFSEventStreamEventFlagKernelDropped | kFSEventStreamEventFlagRootChanged
64 )
65 var events: [WatchEvent] = []
66 for (path, flag) in zip(eventPaths, flags) {
67 let url = URL(fileURLWithPath: path).standardizedFileURL
68 if flag & rescanFlags != 0 {
69 let root = paths.first { path.hasPrefix($0) } ?? path
70 events.append(.rescan(URL(fileURLWithPath: root).standardizedFileURL))
71 } else {
72 events.append(.changed(url))
73 }
74 }
75 if !events.isEmpty { handler(events) }
76 }
77}
78#endif
Sources/OrgWorkspace/Reconciler.swift added +101
@@ -0,0 +1,101 @@
1import Foundation
2import OrgIndex
3
4public struct ReconcileReport: Sendable, Equatable {
5 public var indexed: [String] = []
6 public var touched: [String] = []
7 public var moved: [String] = []
8 public var removed: [String] = []
9 public var unchanged = 0
10 /// iCloud files asked to download; they are indexed once they arrive.
11 public var placeholders: [URL] = []
12}
13
14/// Brings the index in line with the files under a root. File events are only hints, so this
15/// is what runs at launch, after dropped events, and for every batch of changed paths.
16public struct Reconciler: Sendable {
17 public let index: IndexStore
18 public let settings: IndexSettings
19
20 public init(index: IndexStore, settings: IndexSettings) {
21 self.index = index
22 self.settings = settings
23 }
24
25 /// Reconciles the whole root.
26 public func reconcile(root: URL) throws -> ReconcileReport {
27 let discovery = try WorkspaceScanner.scan(root)
28 for placeholder in discovery.placeholders {
29 try? FileManager.default.startDownloadingUbiquitousItem(at: placeholder)
30 }
31 var report = try reconcile(root: root, found: discovery.files, known: index.fileStates(root: root.path))
32 report.placeholders = discovery.placeholders
33 return report
34 }
35
36 /// Reconciles only `paths` under the root, for a batch of file events.
37 public func reconcile(root: URL, paths: [URL]) throws -> ReconcileReport {
38 let wanted = Set(paths.map(\.standardizedFileURL.path))
39 let known = try index.fileStates(root: root.path).filter { wanted.contains($0.key) }
40 var found: [DiscoveredFile] = []
41 for url in paths.map(\.standardizedFileURL) {
42 guard case .file(let kind) = WorkspaceScanner.classify(url.lastPathComponent),
43 let values = try? url.resourceValues(forKeys: [.fileSizeKey, .contentModificationDateKey, .isDirectoryKey]),
44 values.isDirectory != true else { continue }
45 found.append(DiscoveredFile(
46 url: url, kind: kind, size: values.fileSize ?? 0,
47 mtime: values.contentModificationDate?.timeIntervalSince1970 ?? 0
48 ))
49 }
50 return try reconcile(root: root, found: found, known: known)
51 }
52
53 private func reconcile(root: URL, found: [DiscoveredFile], known: [String: FileState]) throws -> ReconcileReport {
54 var report = ReconcileReport()
55 var change = IndexChange()
56 var candidates: [(file: DiscoveredFile, bytes: [UInt8], hash: String)] = []
57
58 for file in found {
59 let path = file.url.path
60 if let state = known[path], state.size == file.size, state.mtime == file.mtime,
61 state.settingsVersion == settings.version, state.kind == file.kind {
62 report.unchanged += 1
63 continue
64 }
65 guard let data = try? Data(contentsOf: file.url) else { continue }
66 let bytes = [UInt8](data)
67 let hash = FileRecord.hash(bytes)
68 if let state = known[path], state.hash == hash, state.settingsVersion == settings.version, state.kind == file.kind {
69 change.touches.append((path, file.mtime))
70 report.touched.append(path)
71 } else {
72 candidates.append((file, bytes, hash))
73 }
74 }
75
76 // A file that vanished and one that appeared with the same content is a rename.
77 let foundPaths = Set(found.map(\.url.path))
78 var missing = known.filter { !foundPaths.contains($0.key) }
79 for candidate in candidates {
80 let path = candidate.file.url.path
81 if known[path] == nil,
82 let (oldPath, _) = missing.first(where: {
83 $0.value.hash == candidate.hash && $0.value.settingsVersion == settings.version && $0.value.kind == candidate.file.kind
84 }) {
85 missing[oldPath] = nil
86 change.moves.append((oldPath, path, candidate.file.mtime))
87 report.moved.append(path)
88 } else {
89 change.records.append(FileRecord(
90 path: path, root: root.path, kind: candidate.file.kind,
91 bytes: candidate.bytes, mtime: candidate.file.mtime, settings: settings
92 ))
93 report.indexed.append(path)
94 }
95 }
96 change.removals = missing.keys.sorted()
97 report.removed = change.removals
98 try index.apply(change)
99 return report
100 }
101}
Tests/OrgIndexTests/FileRecordTests.swift added +65
@@ -0,0 +1,65 @@
1import OrgCore
2import Testing
3@testable import OrgIndex
4
5func record(_ path: String, _ text: String, kind: FileKind = .org, settings: IndexSettings = IndexSettings()) -> FileRecord {
6 FileRecord(path: path, root: "/notes", kind: kind, bytes: Array(text.utf8), mtime: 1, settings: settings)
7}
8
9struct FileRecordTests {
10 let text = """
11 #+FILETAGS: :work:
12 #+PROPERTY: OWNER team
13 * TODO [#A] Plan the launch :big:
14 SCHEDULED: <2026-10-05 Mon +1w -2d>
15 :PROPERTIES:
16 :ID: plan
17 :CATEGORY: launch
18 :END:
19 Draft the [[https://example.com][brief]] and see [[id:other]].
20 ** DONE Child
21 CLOSED: [2026-10-04 Sun 09:00]
22 :LOGBOOK:
23 CLOCK: [2026-10-04 Sun 08:00]--[2026-10-04 Sun 09:00] => 1:00
24 :END:
25 <2026-10-06 Tue>
26
27 """
28
29 @Test func headings() throws {
30 let headings = record("/notes/a.org", text).headings
31 #expect(headings.count == 2)
32 let plan = headings[0]
33 #expect(plan.todo == "TODO" && !plan.isDone && plan.priority == "A")
34 #expect(plan.orgID == "plan")
35 #expect(plan.outlinePath == ["Plan the launch"])
36 #expect(plan.body.contains("Draft the"))
37 #expect(!plan.body.contains("Child"))
38 #expect(plan.tags == [TagRow(name: "work", inherited: true), TagRow(name: "big", inherited: false)])
39 #expect(plan.links == [LinkRow(type: "https", target: "https://example.com"), LinkRow(type: "id", target: "id:other")])
40 #expect(plan.timestamps == [TimestampRow(kind: .scheduled, start: "2026-10-05", end: nil, repeater: "+1w", warning: "-2d")])
41
42 let child = headings[1]
43 #expect(child.parent == 0 && child.isDone)
44 #expect(child.outlinePath == ["Plan the launch", "Child"])
45 #expect(child.tags.map(\.name) == ["work", "big"])
46 #expect(child.clocks == [ClockRow(start: "2026-10-04T08:00", end: "2026-10-04T09:00", minutes: 60)])
47 #expect(child.timestamps.map(\.kind) == [.closed, .active])
48 #expect(child.properties == [PropertyRow(key: "CATEGORY", value: "launch", inherited: true)])
49 }
50
51 @Test func propertyInheritanceFollowsSettings() {
52 let settings = IndexSettings(semantic: SemanticSettings(propertyInheritance: .all))
53 let child = record("/notes/a.org", text, settings: settings).headings[1]
54 #expect(child.properties.contains(PropertyRow(key: "OWNER", value: "team", inherited: true)))
55 }
56
57 @Test func archiveTag() {
58 let headings = record("/notes/a.org", "* a :ARCHIVE:\n** b\n* c\n").headings
59 #expect(headings.map(\.archived) == [true, true, false])
60 }
61
62 @Test func conflictCopiesHaveNoHeadings() {
63 #expect(record("/notes/a.sync-conflict-1.org", "* a\n", kind: .conflict).headings.isEmpty)
64 }
65}
Tests/OrgIndexTests/IndexStoreTests.swift added +66
@@ -0,0 +1,66 @@
1import OrgCore
2import Testing
3@testable import OrgIndex
4
5struct IndexStoreTests {
6 @Test func searchFindsTitlesAndBodies() throws {
7 let store = try IndexStore()
8 try store.write(record("/notes/a.org", "* Groceries\nbuy apples\n* Taxes\nfile forms\n"))
9 #expect(try store.search("appl").map(\.title) == ["Groceries"])
10 #expect(try store.search("tax").map(\.title) == ["Taxes"])
11 #expect(try store.search("file forms").map(\.title) == ["Taxes"])
12 #expect(try store.search("\"oops").isEmpty)
13 #expect(try store.search(" ").isEmpty)
14 }
15
16 @Test func rewritingAFileReplacesItsRows() throws {
17 let store = try IndexStore()
18 try store.write(record("/notes/a.org", "* Old title\n"))
19 try store.write(record("/notes/a.org", "* New title\n"))
20 #expect(try store.search("old").isEmpty)
21 #expect(try store.search("new").count == 1)
22 }
23
24 @Test func removeMoveAndTouch() throws {
25 let store = try IndexStore()
26 try store.write(record("/notes/a.org", "* Alpha\n"))
27 try store.write(record("/notes/b.org", "* Beta\n"))
28 var change = IndexChange()
29 change.removals = ["/notes/a.org"]
30 change.moves = [(from: "/notes/b.org", to: "/notes/c.org", mtime: 5)]
31 try store.apply(change)
32 #expect(try store.search("alpha").isEmpty)
33 #expect(try store.search("beta").map(\.path) == ["/notes/c.org"])
34 #expect(try store.fileStates(root: "/notes")["/notes/c.org"]?.mtime == 5)
35 }
36
37 @Test func overlayReplacesIndexedRows() throws {
38 let store = try IndexStore()
39 try store.write(record("/notes/a.org", "* Saved title\n"))
40 let overlay = ["/notes/a.org": record("/notes/a.org", "* Unsaved title\n")]
41 #expect(try store.search("saved", overlay: overlay).isEmpty)
42 #expect(try store.search("unsaved", overlay: overlay).map(\.title) == ["Unsaved title"])
43 }
44
45 @Test func idsAndDuplicates() throws {
46 let store = try IndexStore()
47 try store.write(record("/notes/a.org", "* A\n:PROPERTIES:\n:ID: x\n:END:\n"))
48 try store.write(record("/notes/b.org", "* B\n:PROPERTIES:\n:ID: y\n:END:\n"))
49 #expect(try store.headings(withID: "x").map(\.path) == ["/notes/a.org"])
50 #expect(try store.duplicateIDs().isEmpty)
51 let overlay = ["/notes/b.org": record("/notes/b.org", "* B\n:PROPERTIES:\n:ID: x\n:END:\n")]
52 #expect(try store.headings(withID: "x", overlay: overlay).map(\.path) == ["/notes/a.org", "/notes/b.org"])
53 #expect(try store.duplicateIDs(overlay: overlay).keys.sorted() == ["x"])
54 }
55
56 @Test func fileStatesAndKinds() throws {
57 let store = try IndexStore()
58 let a = record("/notes/a.org", "* A\n")
59 try store.write(a)
60 try store.write(record("/notes/a.sync-conflict-1.org", "* A\n", kind: .conflict))
61 let states = try store.fileStates(root: "/notes")
62 #expect(states["/notes/a.org"] == FileState(kind: .org, size: a.size, mtime: 1, hash: a.hash, settingsVersion: 1))
63 #expect(try store.files().map(\.kind) == [.org, .conflict])
64 #expect(try store.search("a").map(\.path) == ["/notes/a.org"])
65 }
66}
Tests/OrgWorkspaceTests/DiscoveryTests.swift added +46
@@ -0,0 +1,46 @@
1import Foundation
2import OrgIndex
3import Testing
4@testable import OrgWorkspace
5
6struct ClassificationTests {
7 @Test(arguments: [
8 ("a.org", FileClassification.file(.org)),
9 ("a.org_archive", .file(.archive)),
10 ("a.sync-conflict-20261004-120000-ABCDEFG.org", .file(.conflict)),
11 (".a.org.icloud", .placeholder("a.org")),
12 (".syncthing.a.org.tmp", .ignored),
13 (".a.org.orgstar-1234", .ignored),
14 ("a.org~", .ignored),
15 ("#a.org#", .ignored),
16 ("a.md", .ignored),
17 (".hidden.org", .ignored),
18 ])
19 func classify(name: String, expected: FileClassification) {
20 #expect(WorkspaceScanner.classify(name) == expected)
21 }
22
23 @Test func agendaScope() {
24 #expect(AgendaScope().contains(relativePath: "a.org", kind: .org))
25 #expect(!AgendaScope().contains(relativePath: "a.org_archive", kind: .archive))
26 #expect(!AgendaScope().contains(relativePath: "a.org", kind: .conflict))
27 let work = AgendaScope(globs: ["work/*"])
28 #expect(work.contains(relativePath: "work/deep/a.org", kind: .org))
29 #expect(!work.contains(relativePath: "home/a.org", kind: .org))
30 }
31}
32
33struct ScanTests {
34 @Test func skipsHiddenFoldersAndFindsPlaceholders() throws {
35 let folder = try Folder()
36 try folder.write("a.org", "* a\n")
37 try folder.write("sub/b.org_archive", "* b\n")
38 try folder.write(".git/c.org", "* c\n")
39 try folder.write(".stversions/d.org", "* d\n")
40 try folder.write("sub/.e.org.icloud", "")
41 try folder.write("notes.txt", "")
42 let discovery = try WorkspaceScanner.scan(folder.url)
43 #expect(discovery.files.map(\.url.lastPathComponent) == ["a.org", "b.org_archive"])
44 #expect(discovery.placeholders.map(\.lastPathComponent) == ["e.org"])
45 }
46}
Tests/OrgWorkspaceTests/Folder.swift added +23
@@ -0,0 +1,23 @@
1import Foundation
2
3/// A temporary folder that removes itself.
4final class Folder {
5 let url: URL
6
7 init() throws {
8 url = FileManager.default.temporaryDirectory.appendingPathComponent("orgstar-\(UUID().uuidString)").standardizedFileURL
9 try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)
10 }
11
12 deinit {
13 try? FileManager.default.removeItem(at: url)
14 }
15
16 @discardableResult
17 func write(_ path: String, _ text: String) throws -> URL {
18 let file = url.appendingPathComponent(path)
19 try FileManager.default.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true)
20 try Data(text.utf8).write(to: file)
21 return file.standardizedFileURL
22 }
23}
Tests/OrgWorkspaceTests/ReconcileTests.swift added +81
@@ -0,0 +1,81 @@
1import Foundation
2import OrgIndex
3import Testing
4@testable import OrgWorkspace
5
6struct ReconcileTests {
7 @Test func indexesUpdatesMovesAndRemoves() throws {
8 let folder = try Folder()
9 let index = try IndexStore()
10 let reconciler = Reconciler(index: index, settings: IndexSettings())
11 let a = try folder.write("a.org", "* Alpha\n")
12 try folder.write("b.org", "* Beta\n")
13
14 var report = try reconciler.reconcile(root: folder.url)
15 #expect(report.indexed.count == 2)
16 #expect(try index.search("alpha").map(\.path) == [a.path])
17
18 report = try reconciler.reconcile(root: folder.url)
19 #expect(report.unchanged == 2 && report.indexed.isEmpty)
20
21 try FileManager.default.moveItem(at: folder.url.appendingPathComponent("b.org"), to: folder.url.appendingPathComponent("c.org"))
22 try folder.write("a.org", "* Gamma\n")
23 report = try reconciler.reconcile(root: folder.url)
24 #expect(report.moved.map { URL(fileURLWithPath: $0).lastPathComponent } == ["c.org"])
25 #expect(report.indexed == [a.path])
26 #expect(try index.search("gamma").count == 1)
27 #expect(try index.search("beta").map { URL(fileURLWithPath: $0.path).lastPathComponent } == ["c.org"])
28
29 try FileManager.default.removeItem(at: a)
30 report = try reconciler.reconcile(root: folder.url)
31 #expect(report.removed == [a.path])
32 #expect(try index.search("gamma").isEmpty)
33 }
34
35 @Test func touchedFileWithSameContentIsNotReparsed() throws {
36 let folder = try Folder()
37 let index = try IndexStore()
38 let reconciler = Reconciler(index: index, settings: IndexSettings())
39 let a = try folder.write("a.org", "* Alpha\n")
40 _ = try reconciler.reconcile(root: folder.url)
41 try FileManager.default.setAttributes([.modificationDate: Date(timeIntervalSinceNow: 60)], ofItemAtPath: a.path)
42 #expect(try reconciler.reconcile(root: folder.url).touched == [a.path])
43 }
44
45 @Test func newSettingsVersionReindexes() throws {
46 let folder = try Folder()
47 let index = try IndexStore()
48 try folder.write("a.org", "* Alpha\n")
49 _ = try Reconciler(index: index, settings: IndexSettings(version: 1)).reconcile(root: folder.url)
50 #expect(try Reconciler(index: index, settings: IndexSettings(version: 2)).reconcile(root: folder.url).indexed.count == 1)
51 }
52
53 @Test func conflictCopiesAreListedButNotSearchable() throws {
54 let folder = try Folder()
55 let index = try IndexStore()
56 try folder.write("a.sync-conflict-1.org", "* Alpha\n")
57 _ = try Reconciler(index: index, settings: IndexSettings()).reconcile(root: folder.url)
58 #expect(try index.files().map(\.kind) == [.conflict])
59 #expect(try index.search("alpha").isEmpty)
60 }
61
62 @Test func reconcilingChangedPaths() throws {
63 let folder = try Folder()
64 let index = try IndexStore()
65 let reconciler = Reconciler(index: index, settings: IndexSettings())
66 let a = try folder.write("a.org", "* Alpha\n")
67 let b = try folder.write("b.org", "* Beta\n")
68 _ = try reconciler.reconcile(root: folder.url)
69 try folder.write("a.org", "* Delta\n")
70 try FileManager.default.removeItem(at: b)
71 let report = try reconciler.reconcile(root: folder.url, paths: [a, b])
72 #expect(report.indexed == [a.path])
73 #expect(report.removed == [b.path])
74 }
75
76 @Test func rootBookmarkResolves() throws {
77 let folder = try Folder()
78 let root = try WorkspaceRoot(url: folder.url)
79 #expect(try root.resolve().url.resolvingSymlinksInPath() == folder.url.resolvingSymlinksInPath())
80 }
81}
Tests/OrgWorkspaceTests/WatcherTests.swift added +38
@@ -0,0 +1,38 @@
1import Foundation
2import OrgIndex
3import Testing
4@testable import OrgWorkspace
5
6struct WatcherTests {
7 @Test func reportsAWrittenFile() throws {
8 let folder = try Folder()
9 let received = Received()
10 let watcher = FSEventsWatcher(roots: [folder.url.resolvingSymlinksInPath()], latency: 0.05) { received.add($0) }
11 watcher.start()
12 defer { watcher.stop() }
13 Thread.sleep(forTimeInterval: 0.2)
14 try folder.write("a.org", "* a\n")
15 #expect(received.wait(for: "a.org", timeout: 5))
16 }
17}
18
19final class Received: @unchecked Sendable {
20 private let lock = NSLock()
21 private var events: [WatchEvent] = []
22
23 func add(_ batch: [WatchEvent]) {
24 lock.withLock { events += batch }
25 }
26
27 func wait(for name: String, timeout: TimeInterval) -> Bool {
28 let deadline = Date(timeIntervalSinceNow: timeout)
29 while Date() < deadline {
30 let found = lock.withLock {
31 events.contains { if case .changed(let url) = $0 { return url.lastPathComponent == name } else { return false } }
32 }
33 if found { return true }
34 Thread.sleep(forTimeInterval: 0.05)
35 }
36 return false
37 }
38}
docs/plans/2026-10-04-workspace-index.md added +1387
@@ -0,0 +1,1387 @@
1# Workspace and Index Implementation Plan
2
3> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
5**Goal:** A SQLite index of every org file under the user's folders, kept in line with the files by reconciliation scans that FSEvents batches trigger, with full-text search, ID lookup and an overlay for unsaved buffers.
6
7**Architecture:** Two targets. `OrgIndex` turns a file's bytes into a `FileRecord` (pure, no database), stores records with GRDB, and answers queries; every query takes an overlay of records for unsaved buffers that replace those files' rows. `OrgWorkspace` classifies and scans folders, decides agenda membership, reconciles a root (or a batch of changed paths) against the index with rename detection, and wraps FSEvents. File events are only hints: reconciliation compares size, mtime, hash and settings version, and is safe to run at any time.
8
9**Tech Stack:** Swift 6.2 tools, Swift Testing, GRDB 7.11 (SQLite with FTS5), CryptoKit, CoreServices (FSEvents, macOS only).
10
11**Spec:** `docs/design.md`, "Workspace, storage and index".
12
13## Global Constraints
14
15- The index is a cache; nothing exists only in it.
16- Syncthing conflict copies are listed but never indexed for search, agenda or IDs.
17- Index rows record the settings version; a new version reindexes.
18- Query results carry the content hash their offsets refer to.
19- Gates (design, "Phases"): full rebuild of 10,000 files under 30 s; reconciliation with no changes under 2 s.
20
21## Out of scope
22
23Agenda queries (phase 3), security-scoped bookmark creation in a sandbox (the Mac app is unsandboxed; iOS adds it), `NSFilePresenter`/`NSMetadataQuery` watching on iOS, and conflict UI.
24
25## File structure
26
27| File | Responsibility |
28| --- | --- |
29| `Package.swift` | `OrgIndex` (GRDB) and `OrgWorkspace` targets |
30| `Sources/OrgIndex/FileRecord.swift` | `FileRecord` and its row types, built from bytes |
31| `Sources/OrgIndex/IndexStore.swift` | Schema, transactional writes, search, IDs, overlay |
32| `Sources/OrgWorkspace/Discovery.swift` | `WorkspaceRoot`, file classification, scanning, `AgendaScope` |
33| `Sources/OrgWorkspace/Reconciler.swift` | Index reconciliation with touches, moves and removals |
34| `Sources/OrgWorkspace/FSEventsWatcher.swift` | File-level FSEvents with rescan events |
35
36---
37
38### Task 1: File records
39
40**Files:**
41- Modify: `Package.swift` (add `OrgIndex` and GRDB)
42- Create: `Sources/OrgIndex/FileRecord.swift`
43- Test: `Tests/OrgIndexTests/FileRecordTests.swift`
44
45**Interfaces:**
46- Consumes: `OrgParser`, `DocumentModel`, `SourceText`, `Timestamp`.
47- Produces: `FileKind`, `IndexSettings(version:org:semantic:)`, `FileRecord(path:root:kind:bytes:mtime:settings:)`, `FileRecord.hash(_:)`, `HeadingRecord`, `TagRow`, `PropertyRow`, `TimestampRow`, `ClockRow`, `LinkRow`.
48
49- [ ] **Step 1: Add the target**
50
51```swift
52// swift-tools-version: 6.2
53import PackageDescription
54
55let package = Package(
56 name: "Orgstar",
57 platforms: [.macOS(.v26), .iOS(.v26)],
58 products: [
59 .library(name: "OrgCore", targets: ["OrgCore"]),
60 .library(name: "OrgDocument", targets: ["OrgDocument"]),
61 .library(name: "OrgIndex", targets: ["OrgIndex"])
62 ],
63 dependencies: [
64 .package(url: "https://github.com/groue/GRDB.swift", from: "7.11.1")
65 ],
66 targets: [
67 .target(name: "OrgCore"),
68 .target(name: "OrgDocument", dependencies: ["OrgCore"]),
69 .target(name: "OrgIndex", dependencies: ["OrgCore", .product(name: "GRDB", package: "GRDB.swift")]),
70 .testTarget(name: "OrgCoreTests", dependencies: ["OrgCore"]),
71 .testTarget(name: "OrgDocumentTests", dependencies: ["OrgDocument"]),
72 .testTarget(name: "OrgIndexTests", dependencies: ["OrgIndex"])
73 ]
74)
75```
76
77- [ ] **Step 2: Write the failing tests**
78
79```swift
80import OrgCore
81import Testing
82@testable import OrgIndex
83
84func record(_ path: String, _ text: String, kind: FileKind = .org, settings: IndexSettings = IndexSettings()) -> FileRecord {
85 FileRecord(path: path, root: "/notes", kind: kind, bytes: Array(text.utf8), mtime: 1, settings: settings)
86}
87
88struct FileRecordTests {
89 let text = """
90 #+FILETAGS: :work:
91 #+PROPERTY: OWNER team
92 * TODO [#A] Plan the launch :big:
93 SCHEDULED: <2026-10-05 Mon +1w -2d>
94 :PROPERTIES:
95 :ID: plan
96 :CATEGORY: launch
97 :END:
98 Draft the [[https://example.com][brief]] and see [[id:other]].
99 ** DONE Child
100 CLOSED: [2026-10-04 Sun 09:00]
101 :LOGBOOK:
102 CLOCK: [2026-10-04 Sun 08:00]--[2026-10-04 Sun 09:00] => 1:00
103 :END:
104 <2026-10-06 Tue>
105
106 """
107
108 @Test func headings() throws {
109 let headings = record("/notes/a.org", text).headings
110 #expect(headings.count == 2)
111 let plan = headings[0]
112 #expect(plan.todo == "TODO" && !plan.isDone && plan.priority == "A")
113 #expect(plan.orgID == "plan")
114 #expect(plan.outlinePath == ["Plan the launch"])
115 #expect(plan.body.contains("Draft the"))
116 #expect(!plan.body.contains("Child"))
117 #expect(plan.tags == [TagRow(name: "work", inherited: true), TagRow(name: "big", inherited: false)])
118 #expect(plan.links == [LinkRow(type: "https", target: "https://example.com"), LinkRow(type: "id", target: "id:other")])
119 #expect(plan.timestamps == [TimestampRow(kind: .scheduled, start: "2026-10-05", end: nil, repeater: "+1w", warning: "-2d")])
120
121 let child = headings[1]
122 #expect(child.parent == 0 && child.isDone)
123 #expect(child.outlinePath == ["Plan the launch", "Child"])
124 #expect(child.tags.map(\.name) == ["work", "big"])
125 #expect(child.clocks == [ClockRow(start: "2026-10-04T08:00", end: "2026-10-04T09:00", minutes: 60)])
126 #expect(child.timestamps.map(\.kind) == [.closed, .active])
127 #expect(child.properties == [PropertyRow(key: "CATEGORY", value: "launch", inherited: true)])
128 }
129
130 @Test func propertyInheritanceFollowsSettings() {
131 let settings = IndexSettings(semantic: SemanticSettings(propertyInheritance: .all))
132 let child = record("/notes/a.org", text, settings: settings).headings[1]
133 #expect(child.properties.contains(PropertyRow(key: "OWNER", value: "team", inherited: true)))
134 }
135
136 @Test func archiveTag() {
137 let headings = record("/notes/a.org", "* a :ARCHIVE:\n** b\n* c\n").headings
138 #expect(headings.map(\.archived) == [true, true, false])
139 }
140
141 @Test func conflictCopiesHaveNoHeadings() {
142 #expect(record("/notes/a.sync-conflict-1.org", "* a\n", kind: .conflict).headings.isEmpty)
143 }
144}
145```
146
147- [ ] **Step 3: Implement**
148
149```swift
150import CryptoKit
151import Foundation
152import OrgCore
153
154public enum FileKind: String, Sendable {
155 case org
156 case archive
157 /// A Syncthing conflict copy: listed, never indexed for search, agenda or IDs.
158 case conflict
159}
160
161/// Everything that decides what an index row means. Bump `version` when a semantic setting
162/// changes, so files indexed under the old settings are indexed again.
163public struct IndexSettings: Sendable, Equatable {
164 public var version: Int
165 public var org: OrgSettings
166 public var semantic: SemanticSettings
167
168 public init(version: Int = 1, org: OrgSettings = .default, semantic: SemanticSettings = .default) {
169 self.version = version
170 self.org = org
171 self.semantic = semantic
172 }
173}
174
175public struct TagRow: Sendable, Equatable {
176 public let name: String
177 public let inherited: Bool
178}
179
180public struct PropertyRow: Sendable, Equatable {
181 /// Upper-cased.
182 public let key: String
183 public let value: String
184 public let inherited: Bool
185}
186
187public struct TimestampRow: Sendable, Equatable {
188 public enum Kind: String, Sendable { case scheduled, deadline, closed, active, inactive }
189 public let kind: Kind
190 /// `2026-10-04` or `2026-10-04T10:00`.
191 public let start: String
192 public let end: String?
193 /// As written: `+1w`, `.+2d/3d`.
194 public let repeater: String?
195 /// As written: `-2d`, `--1w`.
196 public let warning: String?
197}
198
199public struct ClockRow: Sendable, Equatable {
200 public let start: String
201 public let end: String?
202 public let minutes: Int?
203}
204
205public struct LinkRow: Sendable, Equatable {
206 /// The scheme before the first colon (`id`, `file`, `https`), or `fuzzy`.
207 public let type: String
208 public let target: String
209}
210
211public struct HeadingRecord: Sendable, Equatable {
212 /// Position in document order; parents refer to it.
213 public let ordinal: Int
214 public let parent: Int?
215 /// The section's UTF-16 range in the file's text.
216 public let start: Int
217 public let end: Int
218 public let level: Int
219 public let todo: String?
220 public let isDone: Bool
221 public let priority: String?
222 public let title: String
223 public let outlinePath: [String]
224 public let orgID: String?
225 /// Tagged `ARCHIVE`, directly or by inheritance.
226 public let archived: Bool
227 /// The heading's own text, without the heading line and child sections.
228 public let body: String
229 public let tags: [TagRow]
230 public let properties: [PropertyRow]
231 public let timestamps: [TimestampRow]
232 public let clocks: [ClockRow]
233 public let links: [LinkRow]
234}
235
236/// One file's index rows, computed without touching the database.
237public struct FileRecord: Sendable, Equatable {
238 public let path: String
239 public let root: String
240 public let kind: FileKind
241 public let size: Int
242 public let mtime: Double
243 /// SHA-256 of the bytes, hex.
244 public let hash: String
245 public let settingsVersion: Int
246 public let headings: [HeadingRecord]
247
248 public static func hash(_ bytes: [UInt8]) -> String {
249 SHA256.hash(data: Data(bytes)).map { String(format: "%02x", $0) }.joined()
250 }
251
252 public init(path: String, root: String, kind: FileKind, bytes: [UInt8], mtime: Double, settings: IndexSettings) {
253 self.path = path
254 self.root = root
255 self.kind = kind
256 self.size = bytes.count
257 self.mtime = mtime
258 self.hash = Self.hash(bytes)
259 self.settingsVersion = settings.version
260 let source = SourceText(bytes: bytes)
261 headings = kind == .conflict ? [] : Self.headings(source.text, settings: settings)
262 }
263
264 static func headings(_ text: String, settings: IndexSettings) -> [HeadingRecord] {
265 let model = DocumentModel(tree: OrgParser.parse(text, defaults: settings.org), settings: settings.semantic)
266 let utf16 = text.utf16
267 func slice(_ range: Range<Int>) -> String {
268 let start = utf16.index(utf16.startIndex, offsetBy: range.lowerBound)
269 let end = utf16.index(utf16.startIndex, offsetBy: range.upperBound)
270 return String(text[start..<end])
271 }
272 var firstChild: [Int: Int] = [:]
273 for (index, heading) in model.headings.enumerated() {
274 if let parent = heading.parent, firstChild[parent] == nil { firstChild[parent] = index }
275 }
276
277 return model.headings.enumerated().map { index, heading in
278 let bodyEnd = firstChild[index].map { model.headings[$0].sectionRange.lowerBound } ?? heading.sectionRange.upperBound
279 let tags = model.tags(of: index)
280 return HeadingRecord(
281 ordinal: index,
282 parent: heading.parent,
283 start: heading.sectionRange.lowerBound,
284 end: heading.sectionRange.upperBound,
285 level: heading.level,
286 todo: heading.todo,
287 isDone: heading.isDone,
288 priority: heading.priority,
289 title: heading.title,
290 outlinePath: model.outlinePath(of: index),
291 orgID: heading.id,
292 archived: tags.contains { $0.value == "ARCHIVE" },
293 body: slice(heading.headingRange.upperBound..<bodyEnd),
294 tags: tags.map { TagRow(name: $0.value, inherited: $0.source != .heading(index)) },
295 properties: properties(model, index),
296 timestamps: timestamps(heading),
297 clocks: heading.clocks.map {
298 ClockRow(start: format($0.start), end: $0.end.map(format), minutes: $0.minutes)
299 },
300 links: heading.links.map { target in
301 let scheme = target.prefix { $0 != ":" }
302 let isScheme = scheme.count < target.count && !scheme.isEmpty && scheme.allSatisfy { $0.isLetter || $0 == "-" }
303 return LinkRow(type: isScheme ? scheme.lowercased() : "fuzzy", target: target)
304 }
305 )
306 }
307 }
308
309 /// Own properties, plus inherited ones for keys that inherit.
310 static func properties(_ model: DocumentModel, _ index: Int) -> [PropertyRow] {
311 var keys: [String] = []
312 func add(_ entries: [Property], inheritedOnly: Bool) {
313 for entry in entries {
314 let key = entry.key.uppercased()
315 if !keys.contains(key), !inheritedOnly || model.inherits(key) { keys.append(key) }
316 }
317 }
318 add(model.headings[index].properties, inheritedOnly: false)
319 add(model.fileProperties, inheritedOnly: true)
320 for ancestor in model.ancestors(of: index) { add(model.headings[ancestor].properties, inheritedOnly: true) }
321 return keys.compactMap { key in
322 model.property(key, of: index).map {
323 PropertyRow(key: key, value: $0.value, inherited: $0.source != .heading(index))
324 }
325 }
326 }
327
328 static func timestamps(_ heading: HeadingInfo) -> [TimestampRow] {
329 var rows: [TimestampRow] = []
330 func add(_ stamp: Timestamp?, _ kind: TimestampRow.Kind) {
331 guard let stamp else { return }
332 rows.append(TimestampRow(
333 kind: kind,
334 start: format(stamp.start),
335 end: stamp.end.map(format),
336 repeater: stamp.repeater.map {
337 $0.kind.rawValue + format($0.interval) + ($0.habitDeadline.map { "/" + format($0) } ?? "")
338 },
339 warning: stamp.warning.map { ($0.firstOccurrenceOnly ? "--" : "-") + format($0.interval) }
340 ))
341 }
342 add(heading.scheduled, .scheduled)
343 add(heading.deadline, .deadline)
344 add(heading.closed, .closed)
345 for stamp in heading.timestamps { add(stamp, stamp.active ? .active : .inactive) }
346 return rows
347 }
348
349 static func format(_ point: Timestamp.Point) -> String {
350 let date = String(format: "%04d-%02d-%02d", point.year, point.month, point.day)
351 guard let hour = point.hour, let minute = point.minute else { return date }
352 return date + String(format: "T%02d:%02d", hour, minute)
353 }
354
355 static func format(_ interval: Timestamp.Interval) -> String {
356 "\(interval.value)\(interval.unit.rawValue)"
357 }
358}
359```
360
361- [ ] **Step 4: Run to verify pass, then commit**
362
363Run: `swift test --filter FileRecordTests`
364
365```bash
366git add Package.swift Package.resolved Sources/OrgIndex/FileRecord.swift Tests/OrgIndexTests/FileRecordTests.swift
367git commit -m "Add index file records"
368```
369
370---
371
372### Task 2: Index store
373
374**Files:**
375- Create: `Sources/OrgIndex/IndexStore.swift`
376- Test: `Tests/OrgIndexTests/IndexStoreTests.swift`
377
378**Interfaces:**
379- Consumes: `FileRecord` (Task 1).
380- Produces: `IndexStore(path:)`, `write(_:)`, `apply(_ change: IndexChange)`, `fileStates(root:) -> [String: FileState]`, `files()`, `search(_:overlay:limit:)`, `headings(withID:overlay:)`, `duplicateIDs(overlay:)`; `FileState`, `HeadingLocation`, `IndexChange`.
381
382- [ ] **Step 1: Write the failing tests**
383
384```swift
385import OrgCore
386import Testing
387@testable import OrgIndex
388
389struct IndexStoreTests {
390 @Test func searchFindsTitlesAndBodies() throws {
391 let store = try IndexStore()
392 try store.write(record("/notes/a.org", "* Groceries\nbuy apples\n* Taxes\nfile forms\n"))
393 #expect(try store.search("appl").map(\.title) == ["Groceries"])
394 #expect(try store.search("tax").map(\.title) == ["Taxes"])
395 #expect(try store.search("file forms").map(\.title) == ["Taxes"])
396 #expect(try store.search("\"oops").isEmpty)
397 #expect(try store.search(" ").isEmpty)
398 }
399
400 @Test func rewritingAFileReplacesItsRows() throws {
401 let store = try IndexStore()
402 try store.write(record("/notes/a.org", "* Old title\n"))
403 try store.write(record("/notes/a.org", "* New title\n"))
404 #expect(try store.search("old").isEmpty)
405 #expect(try store.search("new").count == 1)
406 }
407
408 @Test func removeMoveAndTouch() throws {
409 let store = try IndexStore()
410 try store.write(record("/notes/a.org", "* Alpha\n"))
411 try store.write(record("/notes/b.org", "* Beta\n"))
412 var change = IndexChange()
413 change.removals = ["/notes/a.org"]
414 change.moves = [(from: "/notes/b.org", to: "/notes/c.org", mtime: 5)]
415 try store.apply(change)
416 #expect(try store.search("alpha").isEmpty)
417 #expect(try store.search("beta").map(\.path) == ["/notes/c.org"])
418 #expect(try store.fileStates(root: "/notes")["/notes/c.org"]?.mtime == 5)
419 }
420
421 @Test func overlayReplacesIndexedRows() throws {
422 let store = try IndexStore()
423 try store.write(record("/notes/a.org", "* Saved title\n"))
424 let overlay = ["/notes/a.org": record("/notes/a.org", "* Unsaved title\n")]
425 #expect(try store.search("saved", overlay: overlay).isEmpty)
426 #expect(try store.search("unsaved", overlay: overlay).map(\.title) == ["Unsaved title"])
427 }
428
429 @Test func idsAndDuplicates() throws {
430 let store = try IndexStore()
431 try store.write(record("/notes/a.org", "* A\n:PROPERTIES:\n:ID: x\n:END:\n"))
432 try store.write(record("/notes/b.org", "* B\n:PROPERTIES:\n:ID: y\n:END:\n"))
433 #expect(try store.headings(withID: "x").map(\.path) == ["/notes/a.org"])
434 #expect(try store.duplicateIDs().isEmpty)
435 let overlay = ["/notes/b.org": record("/notes/b.org", "* B\n:PROPERTIES:\n:ID: x\n:END:\n")]
436 #expect(try store.headings(withID: "x", overlay: overlay).map(\.path) == ["/notes/a.org", "/notes/b.org"])
437 #expect(try store.duplicateIDs(overlay: overlay).keys.sorted() == ["x"])
438 }
439
440 @Test func fileStatesAndKinds() throws {
441 let store = try IndexStore()
442 let a = record("/notes/a.org", "* A\n")
443 try store.write(a)
444 try store.write(record("/notes/a.sync-conflict-1.org", "* A\n", kind: .conflict))
445 let states = try store.fileStates(root: "/notes")
446 #expect(states["/notes/a.org"] == FileState(kind: .org, size: a.size, mtime: 1, hash: a.hash, settingsVersion: 1))
447 #expect(try store.files().map(\.kind) == [.org, .conflict])
448 #expect(try store.search("a").map(\.path) == ["/notes/a.org"])
449 }
450}
451```
452
453- [ ] **Step 2: Implement**
454
455```swift
456import Foundation
457import GRDB
458
459/// What the index knows about a file without reading it.
460public struct FileState: Sendable, Equatable {
461 public let kind: FileKind
462 public let size: Int
463 public let mtime: Double
464 public let hash: String
465 public let settingsVersion: Int
466}
467
468/// A heading as found by a query. `contentHash` is the hash of the text the offsets refer to,
469/// so a caller can tell whether they still apply to an open buffer.
470public struct HeadingLocation: Sendable, Equatable {
471 public let path: String
472 public let ordinal: Int
473 public let title: String
474 public let start: Int
475 public let contentHash: String
476}
477
478/// One reconciliation's worth of changes, applied in a single transaction.
479public struct IndexChange: Sendable {
480 public var records: [FileRecord] = []
481 /// Unchanged content with a new modification time.
482 public var touches: [(path: String, mtime: Double)] = []
483 /// Renamed files whose content didn't change.
484 public var moves: [(from: String, to: String, mtime: Double)] = []
485 public var removals: [String] = []
486
487 public init() {}
488
489 public var isEmpty: Bool { records.isEmpty && touches.isEmpty && moves.isEmpty && removals.isEmpty }
490}
491
492/// The SQLite index. A cache: deleting it loses nothing that the files don't hold.
493public final class IndexStore: Sendable {
494 let database: DatabaseQueue
495
496 /// `path` nil opens an in-memory index.
497 public init(path: String? = nil) throws {
498 database = try path.map { try DatabaseQueue(path: $0) } ?? DatabaseQueue()
499 try Self.migrator.migrate(database)
500 }
501
502 static var migrator: DatabaseMigrator {
503 var migrator = DatabaseMigrator()
504 migrator.registerMigration("v1") { db in
505 try db.execute(sql: """
506 CREATE TABLE files (
507 id INTEGER PRIMARY KEY,
508 path TEXT NOT NULL UNIQUE,
509 root TEXT NOT NULL,
510 kind TEXT NOT NULL,
511 size INTEGER NOT NULL,
512 mtime REAL NOT NULL,
513 hash TEXT NOT NULL,
514 settings_version INTEGER NOT NULL,
515 parsed_at REAL NOT NULL
516 );
517 CREATE INDEX files_root ON files(root);
518 CREATE TABLE headings (
519 id INTEGER PRIMARY KEY,
520 file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
521 ordinal INTEGER NOT NULL,
522 parent_ordinal INTEGER,
523 start_offset INTEGER NOT NULL,
524 end_offset INTEGER NOT NULL,
525 level INTEGER NOT NULL,
526 todo TEXT,
527 is_done INTEGER NOT NULL,
528 priority TEXT,
529 title TEXT NOT NULL,
530 outline_path TEXT NOT NULL,
531 org_id TEXT,
532 archived INTEGER NOT NULL
533 );
534 CREATE INDEX headings_file ON headings(file_id);
535 CREATE INDEX headings_org_id ON headings(org_id);
536 CREATE TABLE tags (
537 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
538 tag TEXT NOT NULL,
539 inherited INTEGER NOT NULL
540 );
541 CREATE INDEX tags_heading ON tags(heading_id);
542 CREATE TABLE properties (
543 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
544 key TEXT NOT NULL,
545 value TEXT NOT NULL,
546 inherited INTEGER NOT NULL
547 );
548 CREATE INDEX properties_heading ON properties(heading_id);
549 CREATE TABLE timestamps (
550 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
551 kind TEXT NOT NULL,
552 start_at TEXT NOT NULL,
553 end_at TEXT,
554 repeater TEXT,
555 warning TEXT
556 );
557 CREATE INDEX timestamps_heading ON timestamps(heading_id);
558 CREATE TABLE clocks (
559 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
560 start_at TEXT NOT NULL,
561 end_at TEXT,
562 minutes INTEGER
563 );
564 CREATE INDEX clocks_heading ON clocks(heading_id);
565 CREATE TABLE links (
566 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
567 type TEXT NOT NULL,
568 target TEXT NOT NULL
569 );
570 CREATE INDEX links_heading ON links(heading_id);
571 CREATE VIRTUAL TABLE headings_fts USING fts5(title, body, tokenize = 'unicode61 remove_diacritics 2');
572 CREATE TRIGGER headings_fts_delete AFTER DELETE ON headings BEGIN
573 DELETE FROM headings_fts WHERE rowid = old.id;
574 END;
575 """)
576 }
577 return migrator
578 }
579
580 // MARK: - Writing
581
582 public func write(_ record: FileRecord) throws {
583 var change = IndexChange()
584 change.records = [record]
585 try apply(change)
586 }
587
588 public func apply(_ change: IndexChange) throws {
589 guard !change.isEmpty else { return }
590 try database.write { db in
591 for path in change.removals {
592 try db.execute(sql: "DELETE FROM files WHERE path = ?", arguments: [path])
593 }
594 for move in change.moves {
595 try db.execute(sql: "UPDATE files SET path = ?, mtime = ? WHERE path = ?", arguments: [move.to, move.mtime, move.from])
596 }
597 for touch in change.touches {
598 try db.execute(sql: "UPDATE files SET mtime = ? WHERE path = ?", arguments: [touch.mtime, touch.path])
599 }
600 for record in change.records {
601 try insert(record, db)
602 }
603 }
604 }
605
606 private func insert(_ record: FileRecord, _ db: Database) throws {
607 try db.execute(sql: "DELETE FROM files WHERE path = ?", arguments: [record.path])
608 try db.execute(
609 sql: """
610 INSERT INTO files (path, root, kind, size, mtime, hash, settings_version, parsed_at)
611 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
612 """,
613 arguments: [record.path, record.root, record.kind.rawValue, record.size, record.mtime, record.hash,
614 record.settingsVersion, Date().timeIntervalSince1970]
615 )
616 let fileID = db.lastInsertedRowID
617 for heading in record.headings {
618 try db.execute(
619 sql: """
620 INSERT INTO headings (file_id, ordinal, parent_ordinal, start_offset, end_offset, level, todo,
621 is_done, priority, title, outline_path, org_id, archived)
622 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
623 """,
624 arguments: [fileID, heading.ordinal, heading.parent, heading.start, heading.end, heading.level, heading.todo,
625 heading.isDone, heading.priority, heading.title, heading.outlinePath.joined(separator: "\u{1F}"),
626 heading.orgID, heading.archived]
627 )
628 let id = db.lastInsertedRowID
629 try db.execute(sql: "INSERT INTO headings_fts (rowid, title, body) VALUES (?, ?, ?)", arguments: [id, heading.title, heading.body])
630 for tag in heading.tags {
631 try db.execute(sql: "INSERT INTO tags VALUES (?, ?, ?)", arguments: [id, tag.name, tag.inherited])
632 }
633 for property in heading.properties {
634 try db.execute(sql: "INSERT INTO properties VALUES (?, ?, ?, ?)", arguments: [id, property.key, property.value, property.inherited])
635 }
636 for stamp in heading.timestamps {
637 try db.execute(
638 sql: "INSERT INTO timestamps VALUES (?, ?, ?, ?, ?, ?)",
639 arguments: [id, stamp.kind.rawValue, stamp.start, stamp.end, stamp.repeater, stamp.warning]
640 )
641 }
642 for clock in heading.clocks {
643 try db.execute(sql: "INSERT INTO clocks VALUES (?, ?, ?, ?)", arguments: [id, clock.start, clock.end, clock.minutes])
644 }
645 for link in heading.links {
646 try db.execute(sql: "INSERT INTO links VALUES (?, ?, ?)", arguments: [id, link.type, link.target])
647 }
648 }
649 }
650
651 // MARK: - Reading
652
653 /// Indexed files under `root`, by path.
654 public func fileStates(root: String) throws -> [String: FileState] {
655 try database.read { db in
656 let rows = try Row.fetchAll(db, sql: "SELECT path, kind, size, mtime, hash, settings_version FROM files WHERE root = ?", arguments: [root])
657 var states: [String: FileState] = [:]
658 for row in rows {
659 states[row["path"]] = FileState(
660 kind: FileKind(rawValue: row["kind"]) ?? .org,
661 size: row["size"], mtime: row["mtime"], hash: row["hash"], settingsVersion: row["settings_version"]
662 )
663 }
664 return states
665 }
666 }
667
668 public func files() throws -> [(path: String, kind: FileKind)] {
669 try database.read { db in
670 try Row.fetchAll(db, sql: "SELECT path, kind FROM files ORDER BY path").map {
671 (path: $0["path"], kind: FileKind(rawValue: $0["kind"]) ?? .org)
672 }
673 }
674 }
675
676 // MARK: - Queries
677 //
678 // `overlay` holds records for open documents with unsaved edits, keyed by path. Their rows
679 // replace that file's indexed rows in every query.
680
681 /// Headings whose title or body contain every word of `query` as a word prefix.
682 public func search(_ query: String, overlay: [String: FileRecord] = [:], limit: Int = 50) throws -> [HeadingLocation] {
683 let terms = query.split(whereSeparator: \.isWhitespace).map(String.init)
684 guard !terms.isEmpty else { return [] }
685 let match = terms.map { "\"" + $0.replacingOccurrences(of: "\"", with: "\"\"") + "\"*" }.joined(separator: " ")
686 let indexed = try database.read { db in
687 try Row.fetchAll(
688 db,
689 sql: """
690 SELECT f.path, f.hash, h.ordinal, h.title, h.start_offset
691 FROM headings_fts
692 JOIN headings h ON h.id = headings_fts.rowid
693 JOIN files f ON f.id = h.file_id
694 WHERE headings_fts MATCH ?
695 ORDER BY bm25(headings_fts)
696 LIMIT ?
697 """,
698 arguments: [match, limit + overlay.count * 10]
699 ).map(location)
700 }
701 // Unsaved buffers get the same word-prefix matching as the full-text index.
702 let prefixes = terms.flatMap(Self.words)
703 let live = overlay.values.sorted { $0.path < $1.path }.flatMap { record in
704 record.headings.filter { heading in
705 let words = Self.words(heading.title + "\n" + heading.body)
706 return prefixes.allSatisfy { prefix in words.contains { $0.hasPrefix(prefix) } }
707 }.map { location(record, $0) }
708 }
709 return Array((live + indexed.filter { overlay[$0.path] == nil }).prefix(limit))
710 }
711
712 /// Headings with `:ID: id`. More than one means the ID is duplicated.
713 public func headings(withID id: String, overlay: [String: FileRecord] = [:]) throws -> [HeadingLocation] {
714 let indexed = try database.read { db in
715 try Row.fetchAll(
716 db,
717 sql: """
718 SELECT f.path, f.hash, h.ordinal, h.title, h.start_offset
719 FROM headings h JOIN files f ON f.id = h.file_id
720 WHERE h.org_id = ? ORDER BY f.path, h.ordinal
721 """,
722 arguments: [id]
723 ).map(location)
724 }
725 let live = overlay.values.sorted { $0.path < $1.path }.flatMap { record in
726 record.headings.filter { $0.orgID == id }.map { location(record, $0) }
727 }
728 return indexed.filter { overlay[$0.path] == nil } + live
729 }
730
731 /// IDs used by more than one heading.
732 public func duplicateIDs(overlay: [String: FileRecord] = [:]) throws -> [String: [HeadingLocation]] {
733 let indexed = try database.read { db in
734 try Row.fetchAll(
735 db,
736 sql: """
737 SELECT h.org_id, f.path, f.hash, h.ordinal, h.title, h.start_offset
738 FROM headings h JOIN files f ON f.id = h.file_id
739 WHERE h.org_id IN (SELECT org_id FROM headings WHERE org_id IS NOT NULL GROUP BY org_id HAVING count(*) > 1)
740 OR (h.org_id IS NOT NULL AND ? > 0)
741 ORDER BY f.path, h.ordinal
742 """,
743 arguments: [overlay.count]
744 ).map { (id: $0["org_id"] as String, location: location($0)) }
745 }
746 var byID: [String: [HeadingLocation]] = [:]
747 for row in indexed where overlay[row.location.path] == nil {
748 byID[row.id, default: []].append(row.location)
749 }
750 for record in overlay.values.sorted(by: { $0.path < $1.path }) {
751 for heading in record.headings {
752 if let id = heading.orgID { byID[id, default: []].append(location(record, heading)) }
753 }
754 }
755 return byID.filter { $0.value.count > 1 }
756 }
757
758 /// Lower-cased runs of letters and digits, as the unicode61 tokenizer splits them.
759 static func words(_ text: String) -> [String] {
760 text.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
761 }
762
763 private func location(_ row: Row) -> HeadingLocation {
764 HeadingLocation(path: row["path"], ordinal: row["ordinal"], title: row["title"], start: row["start_offset"], contentHash: row["hash"])
765 }
766
767 private func location(_ record: FileRecord, _ heading: HeadingRecord) -> HeadingLocation {
768 HeadingLocation(path: record.path, ordinal: heading.ordinal, title: heading.title, start: heading.start, contentHash: record.hash)
769 }
770}
771```
772
773- [ ] **Step 3: Run to verify pass, then commit**
774
775Run: `swift test --filter IndexStoreTests`
776
777```bash
778git add Sources/OrgIndex/IndexStore.swift Tests/OrgIndexTests/IndexStoreTests.swift
779git commit -m "Add SQLite index store with search and overlay"
780```
781
782---
783
784### Task 3: Discovery and agenda scope
785
786**Files:**
787- Modify: `Package.swift` (add `OrgWorkspace`)
788- Create: `Sources/OrgWorkspace/Discovery.swift`
789- Test: `Tests/OrgWorkspaceTests/Folder.swift`, `Tests/OrgWorkspaceTests/DiscoveryTests.swift`
790
791**Interfaces:**
792- Consumes: `FileKind` (Task 1).
793- Produces: `WorkspaceRoot(url:)` with `resolve()`, `DiscoveredFile`, `Discovery`, `FileClassification`, `WorkspaceScanner.classify(_:)`, `WorkspaceScanner.scan(_:)`, `AgendaScope(globs:)` with `contains(relativePath:kind:)`; test helper `Folder`.
794
795- [ ] **Step 1: Add the target**
796
797```swift
798// swift-tools-version: 6.2
799import PackageDescription
800
801let package = Package(
802 name: "Orgstar",
803 platforms: [.macOS(.v26), .iOS(.v26)],
804 products: [
805 .library(name: "OrgCore", targets: ["OrgCore"]),
806 .library(name: "OrgDocument", targets: ["OrgDocument"]),
807 .library(name: "OrgIndex", targets: ["OrgIndex"]),
808 .library(name: "OrgWorkspace", targets: ["OrgWorkspace"])
809 ],
810 dependencies: [
811 .package(url: "https://github.com/groue/GRDB.swift", from: "7.11.1")
812 ],
813 targets: [
814 .target(name: "OrgCore"),
815 .target(name: "OrgDocument", dependencies: ["OrgCore"]),
816 .target(name: "OrgIndex", dependencies: ["OrgCore", .product(name: "GRDB", package: "GRDB.swift")]),
817 .target(name: "OrgWorkspace", dependencies: ["OrgIndex"]),
818 .testTarget(name: "OrgCoreTests", dependencies: ["OrgCore"]),
819 .testTarget(name: "OrgDocumentTests", dependencies: ["OrgDocument"]),
820 .testTarget(name: "OrgIndexTests", dependencies: ["OrgIndex"]),
821 .testTarget(name: "OrgWorkspaceTests", dependencies: ["OrgWorkspace"])
822 ]
823)
824```
825
826- [ ] **Step 2: Write the failing tests**
827
828```swift
829import Foundation
830
831/// A temporary folder that removes itself.
832final class Folder {
833 let url: URL
834
835 init() throws {
836 url = FileManager.default.temporaryDirectory.appendingPathComponent("orgstar-\(UUID().uuidString)").standardizedFileURL
837 try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)
838 }
839
840 deinit {
841 try? FileManager.default.removeItem(at: url)
842 }
843
844 @discardableResult
845 func write(_ path: String, _ text: String) throws -> URL {
846 let file = url.appendingPathComponent(path)
847 try FileManager.default.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true)
848 try Data(text.utf8).write(to: file)
849 return file.standardizedFileURL
850 }
851}
852```
853
854```swift
855import Foundation
856import OrgIndex
857import Testing
858@testable import OrgWorkspace
859
860struct ClassificationTests {
861 @Test(arguments: [
862 ("a.org", FileClassification.file(.org)),
863 ("a.org_archive", .file(.archive)),
864 ("a.sync-conflict-20261004-120000-ABCDEFG.org", .file(.conflict)),
865 (".a.org.icloud", .placeholder("a.org")),
866 (".syncthing.a.org.tmp", .ignored),
867 (".a.org.orgstar-1234", .ignored),
868 ("a.org~", .ignored),
869 ("#a.org#", .ignored),
870 ("a.md", .ignored),
871 (".hidden.org", .ignored),
872 ])
873 func classify(name: String, expected: FileClassification) {
874 #expect(WorkspaceScanner.classify(name) == expected)
875 }
876
877 @Test func agendaScope() {
878 #expect(AgendaScope().contains(relativePath: "a.org", kind: .org))
879 #expect(!AgendaScope().contains(relativePath: "a.org_archive", kind: .archive))
880 #expect(!AgendaScope().contains(relativePath: "a.org", kind: .conflict))
881 let work = AgendaScope(globs: ["work/*"])
882 #expect(work.contains(relativePath: "work/deep/a.org", kind: .org))
883 #expect(!work.contains(relativePath: "home/a.org", kind: .org))
884 }
885}
886
887struct ScanTests {
888 @Test func skipsHiddenFoldersAndFindsPlaceholders() throws {
889 let folder = try Folder()
890 try folder.write("a.org", "* a\n")
891 try folder.write("sub/b.org_archive", "* b\n")
892 try folder.write(".git/c.org", "* c\n")
893 try folder.write(".stversions/d.org", "* d\n")
894 try folder.write("sub/.e.org.icloud", "")
895 try folder.write("notes.txt", "")
896 let discovery = try WorkspaceScanner.scan(folder.url)
897 #expect(discovery.files.map(\.url.lastPathComponent) == ["a.org", "b.org_archive"])
898 #expect(discovery.placeholders.map(\.lastPathComponent) == ["e.org"])
899 }
900}
901```
902
903- [ ] **Step 3: Implement**
904
905```swift
906import Foundation
907import OrgIndex
908
909/// A folder the user added. Identified by its bookmark, so a moved folder is followed.
910public struct WorkspaceRoot: Sendable, Equatable, Codable {
911 public let bookmark: Data
912 /// The path when the bookmark was made or last refreshed, for display.
913 public var path: String
914
915 public init(url: URL) throws {
916 bookmark = try url.bookmarkData(options: [], includingResourceValuesForKeys: nil, relativeTo: nil)
917 path = url.standardizedFileURL.path
918 }
919
920 /// The folder's current location. `isStale` means the bookmark should be made again.
921 public func resolve() throws -> (url: URL, isStale: Bool) {
922 var isStale = false
923 let url = try URL(resolvingBookmarkData: bookmark, options: [], relativeTo: nil, bookmarkDataIsStale: &isStale)
924 _ = url.startAccessingSecurityScopedResource()
925 return (url.standardizedFileURL, isStale)
926 }
927}
928
929public struct DiscoveredFile: Sendable, Equatable {
930 public let url: URL
931 public let kind: FileKind
932 public let size: Int
933 public let mtime: Double
934}
935
936public struct Discovery: Sendable, Equatable {
937 public var files: [DiscoveredFile] = []
938 /// iCloud files not downloaded yet, by the name they will have.
939 public var placeholders: [URL] = []
940}
941
942public enum FileClassification: Sendable, Equatable {
943 case file(FileKind)
944 /// An iCloud placeholder (`.name.org.icloud`) for the file at the given name.
945 case placeholder(String)
946 case ignored
947}
948
949public enum WorkspaceScanner {
950 /// What a file name means to the workspace. Syncthing temp files, our own temp and backup
951 /// files, Emacs backups and auto-saves, and other hidden files are ignored.
952 public static func classify(_ name: String) -> FileClassification {
953 if name.hasPrefix(".") {
954 if name.hasSuffix(".icloud") {
955 let inner = String(name.dropFirst().dropLast(".icloud".count))
956 if case .file = classify(inner) { return .placeholder(inner) }
957 }
958 return .ignored
959 }
960 if name.hasSuffix("~") || name.hasPrefix("#") { return .ignored }
961 let conflict = name.contains(".sync-conflict-")
962 if name.hasSuffix(".org") { return .file(conflict ? .conflict : .org) }
963 if name.hasSuffix(".org_archive") { return .file(conflict ? .conflict : .archive) }
964 return .ignored
965 }
966
967 /// Every org file under `root`, skipping hidden folders (`.git`, `.stfolder`, `.stversions`).
968 public static func scan(_ root: URL) throws -> Discovery {
969 let keys: [URLResourceKey] = [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey]
970 guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: keys) else {
971 return Discovery()
972 }
973 var discovery = Discovery()
974 for case let url as URL in enumerator {
975 let values = try url.resourceValues(forKeys: Set(keys))
976 let name = url.lastPathComponent
977 if values.isDirectory == true {
978 if name.hasPrefix(".") { enumerator.skipDescendants() }
979 continue
980 }
981 switch classify(name) {
982 case .file(let kind):
983 discovery.files.append(DiscoveredFile(
984 url: url.standardizedFileURL,
985 kind: kind,
986 size: values.fileSize ?? 0,
987 mtime: values.contentModificationDate?.timeIntervalSince1970 ?? 0
988 ))
989 case .placeholder(let inner):
990 discovery.placeholders.append(url.deletingLastPathComponent().appendingPathComponent(inner).standardizedFileURL)
991 case .ignored:
992 break
993 }
994 }
995 discovery.files.sort { $0.url.path < $1.url.path }
996 return discovery
997 }
998}
999
1000/// Which files feed the agenda: `.org` files (not archives or conflict copies) matching one of
1001/// the globs, relative to their root. No globs means every `.org` file.
1002public struct AgendaScope: Sendable, Equatable {
1003 public var globs: [String]
1004
1005 public init(globs: [String] = []) {
1006 self.globs = globs
1007 }
1008
1009 public func contains(relativePath: String, kind: FileKind) -> Bool {
1010 guard kind == .org else { return false }
1011 guard !globs.isEmpty else { return true }
1012 // Without FNM_PATHNAME, `*` also matches `/`, so `work/*` covers nested folders.
1013 return globs.contains { fnmatch($0, relativePath, 0) == 0 }
1014 }
1015}
1016```
1017
1018- [ ] **Step 4: Run to verify pass, then commit**
1019
1020Run: `swift test --filter "ClassificationTests|ScanTests"`
1021
1022```bash
1023git add Package.swift Sources/OrgWorkspace/Discovery.swift Tests/OrgWorkspaceTests/Folder.swift Tests/OrgWorkspaceTests/DiscoveryTests.swift
1024git commit -m "Add workspace discovery and agenda scope"
1025```
1026
1027---
1028
1029### Task 4: Reconciliation
1030
1031**Files:**
1032- Create: `Sources/OrgWorkspace/Reconciler.swift`
1033- Test: `Tests/OrgWorkspaceTests/ReconcileTests.swift`
1034
1035**Interfaces:**
1036- Consumes: Tasks 1–3.
1037- Produces: `Reconciler(index:settings:)` with `reconcile(root:)` and `reconcile(root:paths:)`, `ReconcileReport`.
1038
1039- [ ] **Step 1: Write the failing tests**
1040
1041```swift
1042import Foundation
1043import OrgIndex
1044import Testing
1045@testable import OrgWorkspace
1046
1047struct ReconcileTests {
1048 @Test func indexesUpdatesMovesAndRemoves() throws {
1049 let folder = try Folder()
1050 let index = try IndexStore()
1051 let reconciler = Reconciler(index: index, settings: IndexSettings())
1052 let a = try folder.write("a.org", "* Alpha\n")
1053 try folder.write("b.org", "* Beta\n")
1054
1055 var report = try reconciler.reconcile(root: folder.url)
1056 #expect(report.indexed.count == 2)
1057 #expect(try index.search("alpha").map(\.path) == [a.path])
1058
1059 report = try reconciler.reconcile(root: folder.url)
1060 #expect(report.unchanged == 2 && report.indexed.isEmpty)
1061
1062 try FileManager.default.moveItem(at: folder.url.appendingPathComponent("b.org"), to: folder.url.appendingPathComponent("c.org"))
1063 try folder.write("a.org", "* Gamma\n")
1064 report = try reconciler.reconcile(root: folder.url)
1065 #expect(report.moved.map { URL(fileURLWithPath: $0).lastPathComponent } == ["c.org"])
1066 #expect(report.indexed == [a.path])
1067 #expect(try index.search("gamma").count == 1)
1068 #expect(try index.search("beta").map { URL(fileURLWithPath: $0.path).lastPathComponent } == ["c.org"])
1069
1070 try FileManager.default.removeItem(at: a)
1071 report = try reconciler.reconcile(root: folder.url)
1072 #expect(report.removed == [a.path])
1073 #expect(try index.search("gamma").isEmpty)
1074 }
1075
1076 @Test func touchedFileWithSameContentIsNotReparsed() throws {
1077 let folder = try Folder()
1078 let index = try IndexStore()
1079 let reconciler = Reconciler(index: index, settings: IndexSettings())
1080 let a = try folder.write("a.org", "* Alpha\n")
1081 _ = try reconciler.reconcile(root: folder.url)
1082 try FileManager.default.setAttributes([.modificationDate: Date(timeIntervalSinceNow: 60)], ofItemAtPath: a.path)
1083 #expect(try reconciler.reconcile(root: folder.url).touched == [a.path])
1084 }
1085
1086 @Test func newSettingsVersionReindexes() throws {
1087 let folder = try Folder()
1088 let index = try IndexStore()
1089 try folder.write("a.org", "* Alpha\n")
1090 _ = try Reconciler(index: index, settings: IndexSettings(version: 1)).reconcile(root: folder.url)
1091 #expect(try Reconciler(index: index, settings: IndexSettings(version: 2)).reconcile(root: folder.url).indexed.count == 1)
1092 }
1093
1094 @Test func conflictCopiesAreListedButNotSearchable() throws {
1095 let folder = try Folder()
1096 let index = try IndexStore()
1097 try folder.write("a.sync-conflict-1.org", "* Alpha\n")
1098 _ = try Reconciler(index: index, settings: IndexSettings()).reconcile(root: folder.url)
1099 #expect(try index.files().map(\.kind) == [.conflict])
1100 #expect(try index.search("alpha").isEmpty)
1101 }
1102
1103 @Test func reconcilingChangedPaths() throws {
1104 let folder = try Folder()
1105 let index = try IndexStore()
1106 let reconciler = Reconciler(index: index, settings: IndexSettings())
1107 let a = try folder.write("a.org", "* Alpha\n")
1108 let b = try folder.write("b.org", "* Beta\n")
1109 _ = try reconciler.reconcile(root: folder.url)
1110 try folder.write("a.org", "* Delta\n")
1111 try FileManager.default.removeItem(at: b)
1112 let report = try reconciler.reconcile(root: folder.url, paths: [a, b])
1113 #expect(report.indexed == [a.path])
1114 #expect(report.removed == [b.path])
1115 }
1116
1117 @Test func rootBookmarkResolves() throws {
1118 let folder = try Folder()
1119 let root = try WorkspaceRoot(url: folder.url)
1120 #expect(try root.resolve().url.resolvingSymlinksInPath() == folder.url.resolvingSymlinksInPath())
1121 }
1122}
1123```
1124
1125- [ ] **Step 2: Implement**
1126
1127```swift
1128import Foundation
1129import OrgIndex
1130
1131public struct ReconcileReport: Sendable, Equatable {
1132 public var indexed: [String] = []
1133 public var touched: [String] = []
1134 public var moved: [String] = []
1135 public var removed: [String] = []
1136 public var unchanged = 0
1137 /// iCloud files asked to download; they are indexed once they arrive.
1138 public var placeholders: [URL] = []
1139}
1140
1141/// Brings the index in line with the files under a root. File events are only hints, so this
1142/// is what runs at launch, after dropped events, and for every batch of changed paths.
1143public struct Reconciler: Sendable {
1144 public let index: IndexStore
1145 public let settings: IndexSettings
1146
1147 public init(index: IndexStore, settings: IndexSettings) {
1148 self.index = index
1149 self.settings = settings
1150 }
1151
1152 /// Reconciles the whole root.
1153 public func reconcile(root: URL) throws -> ReconcileReport {
1154 let discovery = try WorkspaceScanner.scan(root)
1155 for placeholder in discovery.placeholders {
1156 try? FileManager.default.startDownloadingUbiquitousItem(at: placeholder)
1157 }
1158 var report = try reconcile(root: root, found: discovery.files, known: index.fileStates(root: root.path))
1159 report.placeholders = discovery.placeholders
1160 return report
1161 }
1162
1163 /// Reconciles only `paths` under the root, for a batch of file events.
1164 public func reconcile(root: URL, paths: [URL]) throws -> ReconcileReport {
1165 let wanted = Set(paths.map(\.standardizedFileURL.path))
1166 let known = try index.fileStates(root: root.path).filter { wanted.contains($0.key) }
1167 var found: [DiscoveredFile] = []
1168 for url in paths.map(\.standardizedFileURL) {
1169 guard case .file(let kind) = WorkspaceScanner.classify(url.lastPathComponent),
1170 let values = try? url.resourceValues(forKeys: [.fileSizeKey, .contentModificationDateKey, .isDirectoryKey]),
1171 values.isDirectory != true else { continue }
1172 found.append(DiscoveredFile(
1173 url: url, kind: kind, size: values.fileSize ?? 0,
1174 mtime: values.contentModificationDate?.timeIntervalSince1970 ?? 0
1175 ))
1176 }
1177 return try reconcile(root: root, found: found, known: known)
1178 }
1179
1180 private func reconcile(root: URL, found: [DiscoveredFile], known: [String: FileState]) throws -> ReconcileReport {
1181 var report = ReconcileReport()
1182 var change = IndexChange()
1183 var candidates: [(file: DiscoveredFile, bytes: [UInt8], hash: String)] = []
1184
1185 for file in found {
1186 let path = file.url.path
1187 if let state = known[path], state.size == file.size, state.mtime == file.mtime,
1188 state.settingsVersion == settings.version, state.kind == file.kind {
1189 report.unchanged += 1
1190 continue
1191 }
1192 guard let data = try? Data(contentsOf: file.url) else { continue }
1193 let bytes = [UInt8](data)
1194 let hash = FileRecord.hash(bytes)
1195 if let state = known[path], state.hash == hash, state.settingsVersion == settings.version, state.kind == file.kind {
1196 change.touches.append((path, file.mtime))
1197 report.touched.append(path)
1198 } else {
1199 candidates.append((file, bytes, hash))
1200 }
1201 }
1202
1203 // A file that vanished and one that appeared with the same content is a rename.
1204 let foundPaths = Set(found.map(\.url.path))
1205 var missing = known.filter { !foundPaths.contains($0.key) }
1206 for candidate in candidates {
1207 let path = candidate.file.url.path
1208 if known[path] == nil,
1209 let (oldPath, _) = missing.first(where: {
1210 $0.value.hash == candidate.hash && $0.value.settingsVersion == settings.version && $0.value.kind == candidate.file.kind
1211 }) {
1212 missing[oldPath] = nil
1213 change.moves.append((oldPath, path, candidate.file.mtime))
1214 report.moved.append(path)
1215 } else {
1216 change.records.append(FileRecord(
1217 path: path, root: root.path, kind: candidate.file.kind,
1218 bytes: candidate.bytes, mtime: candidate.file.mtime, settings: settings
1219 ))
1220 report.indexed.append(path)
1221 }
1222 }
1223 change.removals = missing.keys.sorted()
1224 report.removed = change.removals
1225 try index.apply(change)
1226 return report
1227 }
1228}
1229```
1230
1231- [ ] **Step 3: Run to verify pass, then commit**
1232
1233Run: `swift test --filter ReconcileTests`
1234
1235```bash
1236git add Sources/OrgWorkspace/Reconciler.swift Tests/OrgWorkspaceTests/ReconcileTests.swift
1237git commit -m "Add index reconciliation"
1238```
1239
1240---
1241
1242### Task 5: FSEvents watcher
1243
1244**Files:**
1245- Create: `Sources/OrgWorkspace/FSEventsWatcher.swift`
1246- Test: `Tests/OrgWorkspaceTests/WatcherTests.swift`
1247
1248**Interfaces:**
1249- Produces (macOS): `WatchEvent` (`.changed(URL)`, `.rescan(URL)`), `FSEventsWatcher(roots:latency:handler:)` with `start()`, `stop()`.
1250
1251- [ ] **Step 1: Write the failing test**
1252
1253```swift
1254import Foundation
1255import OrgIndex
1256import Testing
1257@testable import OrgWorkspace
1258
1259struct WatcherTests {
1260 @Test func reportsAWrittenFile() throws {
1261 let folder = try Folder()
1262 let received = Received()
1263 let watcher = FSEventsWatcher(roots: [folder.url.resolvingSymlinksInPath()], latency: 0.05) { received.add($0) }
1264 watcher.start()
1265 defer { watcher.stop() }
1266 Thread.sleep(forTimeInterval: 0.2)
1267 try folder.write("a.org", "* a\n")
1268 #expect(received.wait(for: "a.org", timeout: 5))
1269 }
1270}
1271
1272final class Received: @unchecked Sendable {
1273 private let lock = NSLock()
1274 private var events: [WatchEvent] = []
1275
1276 func add(_ batch: [WatchEvent]) {
1277 lock.withLock { events += batch }
1278 }
1279
1280 func wait(for name: String, timeout: TimeInterval) -> Bool {
1281 let deadline = Date(timeIntervalSinceNow: timeout)
1282 while Date() < deadline {
1283 let found = lock.withLock {
1284 events.contains { if case .changed(let url) = $0 { return url.lastPathComponent == name } else { return false } }
1285 }
1286 if found { return true }
1287 Thread.sleep(forTimeInterval: 0.05)
1288 }
1289 return false
1290 }
1291}
1292```
1293
1294- [ ] **Step 2: Implement**
1295
1296```swift
1297#if os(macOS)
1298import CoreServices
1299import Foundation
1300
1301public enum WatchEvent: Sendable, Equatable {
1302 case changed(URL)
1303 /// Events were dropped or coalesced for this folder, or a root moved: rescan it.
1304 case rescan(URL)
1305}
1306
1307/// FSEvents with file-level events. Delivers batches on a private queue.
1308public final class FSEventsWatcher: @unchecked Sendable {
1309 private let paths: [String]
1310 private let latency: CFTimeInterval
1311 private let handler: @Sendable ([WatchEvent]) -> Void
1312 private let queue = DispatchQueue(label: "orgstar.fsevents")
1313 private var stream: FSEventStreamRef?
1314
1315 public init(roots: [URL], latency: CFTimeInterval = 0.3, handler: @escaping @Sendable ([WatchEvent]) -> Void) {
1316 paths = roots.map(\.path)
1317 self.latency = latency
1318 self.handler = handler
1319 }
1320
1321 deinit {
1322 stop()
1323 }
1324
1325 public func start() {
1326 guard stream == nil else { return }
1327 var context = FSEventStreamContext(
1328 version: 0, info: Unmanaged.passUnretained(self).toOpaque(), retain: nil, release: nil, copyDescription: nil
1329 )
1330 let flags = FSEventStreamCreateFlags(
1331 kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagUseCFTypes
1332 | kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot
1333 )
1334 let callback: FSEventStreamCallback = { _, info, count, eventPaths, eventFlags, _ in
1335 guard let info else { return }
1336 let watcher = Unmanaged<FSEventsWatcher>.fromOpaque(info).takeUnretainedValue()
1337 let paths = unsafeBitCast(eventPaths, to: NSArray.self) as? [String] ?? []
1338 watcher.deliver(paths: paths, flags: Array(UnsafeBufferPointer(start: eventFlags, count: count)))
1339 }
1340 guard let stream = FSEventStreamCreate(
1341 nil, callback, &context, paths as CFArray, FSEventStreamEventId(kFSEventStreamEventIdSinceNow), latency, flags
1342 ) else { return }
1343 FSEventStreamSetDispatchQueue(stream, queue)
1344 FSEventStreamStart(stream)
1345 self.stream = stream
1346 }
1347
1348 public func stop() {
1349 guard let stream else { return }
1350 FSEventStreamStop(stream)
1351 FSEventStreamInvalidate(stream)
1352 FSEventStreamRelease(stream)
1353 self.stream = nil
1354 }
1355
1356 private func deliver(paths eventPaths: [String], flags: [FSEventStreamEventFlags]) {
1357 let rescanFlags = FSEventStreamEventFlags(
1358 kFSEventStreamEventFlagMustScanSubDirs | kFSEventStreamEventFlagUserDropped
1359 | kFSEventStreamEventFlagKernelDropped | kFSEventStreamEventFlagRootChanged
1360 )
1361 var events: [WatchEvent] = []
1362 for (path, flag) in zip(eventPaths, flags) {
1363 let url = URL(fileURLWithPath: path).standardizedFileURL
1364 if flag & rescanFlags != 0 {
1365 let root = paths.first { path.hasPrefix($0) } ?? path
1366 events.append(.rescan(URL(fileURLWithPath: root).standardizedFileURL))
1367 } else {
1368 events.append(.changed(url))
1369 }
1370 }
1371 if !events.isEmpty { handler(events) }
1372 }
1373}
1374#endif
1375```
1376
1377- [ ] **Step 3: Run everything, the iOS build and the gates**
1378
1379Run: `swift test`, then `xcodebuild -scheme OrgWorkspace -destination 'generic/platform=iOS' build`.
1380Gates: reconcile a folder of 10,000 org files into an empty index (under 30 s), then again with no changes (under 2 s), in a release build on the reference machine.
1381
1382- [ ] **Step 4: Commit**
1383
1384```bash
1385git add Sources/OrgWorkspace/FSEventsWatcher.swift Tests/OrgWorkspaceTests/WatcherTests.swift
1386git commit -m "Add FSEvents watcher"
1387```