docs/plans/2026-10-04-workspace-index.md
1387 lines · 57081 bytes
1# Workspace and Index Implementation Plan
2
3> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
5**Goal:** A SQLite index of every org file under the user's folders, kept in line with the files by reconciliation scans that FSEvents batches trigger, with full-text search, ID lookup and an overlay for unsaved buffers.
6
7**Architecture:** Two targets. `OrgIndex` turns a file's bytes into a `FileRecord` (pure, no database), stores records with GRDB, and answers queries; every query takes an overlay of records for unsaved buffers that replace those files' rows. `OrgWorkspace` classifies and scans folders, decides agenda membership, reconciles a root (or a batch of changed paths) against the index with rename detection, and wraps FSEvents. File events are only hints: reconciliation compares size, mtime, hash and settings version, and is safe to run at any time.
8
9**Tech Stack:** Swift 6.2 tools, Swift Testing, GRDB 7.11 (SQLite with FTS5), CryptoKit, CoreServices (FSEvents, macOS only).
10
11**Spec:** `docs/design.md`, "Workspace, storage and index".
12
13## Global Constraints
14
15- The index is a cache; nothing exists only in it.
16- Syncthing conflict copies are listed but never indexed for search, agenda or IDs.
17- Index rows record the settings version; a new version reindexes.
18- Query results carry the content hash their offsets refer to.
19- Gates (design, "Phases"): full rebuild of 10,000 files under 30 s; reconciliation with no changes under 2 s.
20
21## Out of scope
22
23Agenda queries (phase 3), security-scoped bookmark creation in a sandbox (the Mac app is unsandboxed; iOS adds it), `NSFilePresenter`/`NSMetadataQuery` watching on iOS, and conflict UI.
24
25## File structure
26
27| File | Responsibility |
28| --- | --- |
29| `Package.swift` | `OrgIndex` (GRDB) and `OrgWorkspace` targets |
30| `Sources/OrgIndex/FileRecord.swift` | `FileRecord` and its row types, built from bytes |
31| `Sources/OrgIndex/IndexStore.swift` | Schema, transactional writes, search, IDs, overlay |
32| `Sources/OrgWorkspace/Discovery.swift` | `WorkspaceRoot`, file classification, scanning, `AgendaScope` |
33| `Sources/OrgWorkspace/Reconciler.swift` | Index reconciliation with touches, moves and removals |
34| `Sources/OrgWorkspace/FSEventsWatcher.swift` | File-level FSEvents with rescan events |
35
36---
37
38### Task 1: File records
39
40**Files:**
41- Modify: `Package.swift` (add `OrgIndex` and GRDB)
42- Create: `Sources/OrgIndex/FileRecord.swift`
43- Test: `Tests/OrgIndexTests/FileRecordTests.swift`
44
45**Interfaces:**
46- Consumes: `OrgParser`, `DocumentModel`, `SourceText`, `Timestamp`.
47- Produces: `FileKind`, `IndexSettings(version:org:semantic:)`, `FileRecord(path:root:kind:bytes:mtime:settings:)`, `FileRecord.hash(_:)`, `HeadingRecord`, `TagRow`, `PropertyRow`, `TimestampRow`, `ClockRow`, `LinkRow`.
48
49- [ ] **Step 1: Add the target**
50
51```swift
52// swift-tools-version: 6.2
53import PackageDescription
54
55let package = Package(
56 name: "Orgstar",
57 platforms: [.macOS(.v26), .iOS(.v26)],
58 products: [
59 .library(name: "OrgCore", targets: ["OrgCore"]),
60 .library(name: "OrgDocument", targets: ["OrgDocument"]),
61 .library(name: "OrgIndex", targets: ["OrgIndex"])
62 ],
63 dependencies: [
64 .package(url: "https://github.com/groue/GRDB.swift", from: "7.11.1")
65 ],
66 targets: [
67 .target(name: "OrgCore"),
68 .target(name: "OrgDocument", dependencies: ["OrgCore"]),
69 .target(name: "OrgIndex", dependencies: ["OrgCore", .product(name: "GRDB", package: "GRDB.swift")]),
70 .testTarget(name: "OrgCoreTests", dependencies: ["OrgCore"]),
71 .testTarget(name: "OrgDocumentTests", dependencies: ["OrgDocument"]),
72 .testTarget(name: "OrgIndexTests", dependencies: ["OrgIndex"])
73 ]
74)
75```
76
77- [ ] **Step 2: Write the failing tests**
78
79```swift
80import OrgCore
81import Testing
82@testable import OrgIndex
83
84func record(_ path: String, _ text: String, kind: FileKind = .org, settings: IndexSettings = IndexSettings()) -> FileRecord {
85 FileRecord(path: path, root: "/notes", kind: kind, bytes: Array(text.utf8), mtime: 1, settings: settings)
86}
87
88struct FileRecordTests {
89 let text = """
90 #+FILETAGS: :work:
91 #+PROPERTY: OWNER team
92 * TODO [#A] Plan the launch :big:
93 SCHEDULED: <2026-10-05 Mon +1w -2d>
94 :PROPERTIES:
95 :ID: plan
96 :CATEGORY: launch
97 :END:
98 Draft the [[https://example.com][brief]] and see [[id:other]].
99 ** DONE Child
100 CLOSED: [2026-10-04 Sun 09:00]
101 :LOGBOOK:
102 CLOCK: [2026-10-04 Sun 08:00]--[2026-10-04 Sun 09:00] => 1:00
103 :END:
104 <2026-10-06 Tue>
105
106 """
107
108 @Test func headings() throws {
109 let headings = record("/notes/a.org", text).headings
110 #expect(headings.count == 2)
111 let plan = headings[0]
112 #expect(plan.todo == "TODO" && !plan.isDone && plan.priority == "A")
113 #expect(plan.orgID == "plan")
114 #expect(plan.outlinePath == ["Plan the launch"])
115 #expect(plan.body.contains("Draft the"))
116 #expect(!plan.body.contains("Child"))
117 #expect(plan.tags == [TagRow(name: "work", inherited: true), TagRow(name: "big", inherited: false)])
118 #expect(plan.links == [LinkRow(type: "https", target: "https://example.com"), LinkRow(type: "id", target: "id:other")])
119 #expect(plan.timestamps == [TimestampRow(kind: .scheduled, start: "2026-10-05", end: nil, repeater: "+1w", warning: "-2d")])
120
121 let child = headings[1]
122 #expect(child.parent == 0 && child.isDone)
123 #expect(child.outlinePath == ["Plan the launch", "Child"])
124 #expect(child.tags.map(\.name) == ["work", "big"])
125 #expect(child.clocks == [ClockRow(start: "2026-10-04T08:00", end: "2026-10-04T09:00", minutes: 60)])
126 #expect(child.timestamps.map(\.kind) == [.closed, .active])
127 #expect(child.properties == [PropertyRow(key: "CATEGORY", value: "launch", inherited: true)])
128 }
129
130 @Test func propertyInheritanceFollowsSettings() {
131 let settings = IndexSettings(semantic: SemanticSettings(propertyInheritance: .all))
132 let child = record("/notes/a.org", text, settings: settings).headings[1]
133 #expect(child.properties.contains(PropertyRow(key: "OWNER", value: "team", inherited: true)))
134 }
135
136 @Test func archiveTag() {
137 let headings = record("/notes/a.org", "* a :ARCHIVE:\n** b\n* c\n").headings
138 #expect(headings.map(\.archived) == [true, true, false])
139 }
140
141 @Test func conflictCopiesHaveNoHeadings() {
142 #expect(record("/notes/a.sync-conflict-1.org", "* a\n", kind: .conflict).headings.isEmpty)
143 }
144}
145```
146
147- [ ] **Step 3: Implement**
148
149```swift
150import CryptoKit
151import Foundation
152import OrgCore
153
154public enum FileKind: String, Sendable {
155 case org
156 case archive
157 /// A Syncthing conflict copy: listed, never indexed for search, agenda or IDs.
158 case conflict
159}
160
161/// Everything that decides what an index row means. Bump `version` when a semantic setting
162/// changes, so files indexed under the old settings are indexed again.
163public struct IndexSettings: Sendable, Equatable {
164 public var version: Int
165 public var org: OrgSettings
166 public var semantic: SemanticSettings
167
168 public init(version: Int = 1, org: OrgSettings = .default, semantic: SemanticSettings = .default) {
169 self.version = version
170 self.org = org
171 self.semantic = semantic
172 }
173}
174
175public struct TagRow: Sendable, Equatable {
176 public let name: String
177 public let inherited: Bool
178}
179
180public struct PropertyRow: Sendable, Equatable {
181 /// Upper-cased.
182 public let key: String
183 public let value: String
184 public let inherited: Bool
185}
186
187public struct TimestampRow: Sendable, Equatable {
188 public enum Kind: String, Sendable { case scheduled, deadline, closed, active, inactive }
189 public let kind: Kind
190 /// `2026-10-04` or `2026-10-04T10:00`.
191 public let start: String
192 public let end: String?
193 /// As written: `+1w`, `.+2d/3d`.
194 public let repeater: String?
195 /// As written: `-2d`, `--1w`.
196 public let warning: String?
197}
198
199public struct ClockRow: Sendable, Equatable {
200 public let start: String
201 public let end: String?
202 public let minutes: Int?
203}
204
205public struct LinkRow: Sendable, Equatable {
206 /// The scheme before the first colon (`id`, `file`, `https`), or `fuzzy`.
207 public let type: String
208 public let target: String
209}
210
211public struct HeadingRecord: Sendable, Equatable {
212 /// Position in document order; parents refer to it.
213 public let ordinal: Int
214 public let parent: Int?
215 /// The section's UTF-16 range in the file's text.
216 public let start: Int
217 public let end: Int
218 public let level: Int
219 public let todo: String?
220 public let isDone: Bool
221 public let priority: String?
222 public let title: String
223 public let outlinePath: [String]
224 public let orgID: String?
225 /// Tagged `ARCHIVE`, directly or by inheritance.
226 public let archived: Bool
227 /// The heading's own text, without the heading line and child sections.
228 public let body: String
229 public let tags: [TagRow]
230 public let properties: [PropertyRow]
231 public let timestamps: [TimestampRow]
232 public let clocks: [ClockRow]
233 public let links: [LinkRow]
234}
235
236/// One file's index rows, computed without touching the database.
237public struct FileRecord: Sendable, Equatable {
238 public let path: String
239 public let root: String
240 public let kind: FileKind
241 public let size: Int
242 public let mtime: Double
243 /// SHA-256 of the bytes, hex.
244 public let hash: String
245 public let settingsVersion: Int
246 public let headings: [HeadingRecord]
247
248 public static func hash(_ bytes: [UInt8]) -> String {
249 SHA256.hash(data: Data(bytes)).map { String(format: "%02x", $0) }.joined()
250 }
251
252 public init(path: String, root: String, kind: FileKind, bytes: [UInt8], mtime: Double, settings: IndexSettings) {
253 self.path = path
254 self.root = root
255 self.kind = kind
256 self.size = bytes.count
257 self.mtime = mtime
258 self.hash = Self.hash(bytes)
259 self.settingsVersion = settings.version
260 let source = SourceText(bytes: bytes)
261 headings = kind == .conflict ? [] : Self.headings(source.text, settings: settings)
262 }
263
264 static func headings(_ text: String, settings: IndexSettings) -> [HeadingRecord] {
265 let model = DocumentModel(tree: OrgParser.parse(text, defaults: settings.org), settings: settings.semantic)
266 let utf16 = text.utf16
267 func slice(_ range: Range<Int>) -> String {
268 let start = utf16.index(utf16.startIndex, offsetBy: range.lowerBound)
269 let end = utf16.index(utf16.startIndex, offsetBy: range.upperBound)
270 return String(text[start..<end])
271 }
272 var firstChild: [Int: Int] = [:]
273 for (index, heading) in model.headings.enumerated() {
274 if let parent = heading.parent, firstChild[parent] == nil { firstChild[parent] = index }
275 }
276
277 return model.headings.enumerated().map { index, heading in
278 let bodyEnd = firstChild[index].map { model.headings[$0].sectionRange.lowerBound } ?? heading.sectionRange.upperBound
279 let tags = model.tags(of: index)
280 return HeadingRecord(
281 ordinal: index,
282 parent: heading.parent,
283 start: heading.sectionRange.lowerBound,
284 end: heading.sectionRange.upperBound,
285 level: heading.level,
286 todo: heading.todo,
287 isDone: heading.isDone,
288 priority: heading.priority,
289 title: heading.title,
290 outlinePath: model.outlinePath(of: index),
291 orgID: heading.id,
292 archived: tags.contains { $0.value == "ARCHIVE" },
293 body: slice(heading.headingRange.upperBound..<bodyEnd),
294 tags: tags.map { TagRow(name: $0.value, inherited: $0.source != .heading(index)) },
295 properties: properties(model, index),
296 timestamps: timestamps(heading),
297 clocks: heading.clocks.map {
298 ClockRow(start: format($0.start), end: $0.end.map(format), minutes: $0.minutes)
299 },
300 links: heading.links.map { target in
301 let scheme = target.prefix { $0 != ":" }
302 let isScheme = scheme.count < target.count && !scheme.isEmpty && scheme.allSatisfy { $0.isLetter || $0 == "-" }
303 return LinkRow(type: isScheme ? scheme.lowercased() : "fuzzy", target: target)
304 }
305 )
306 }
307 }
308
309 /// Own properties, plus inherited ones for keys that inherit.
310 static func properties(_ model: DocumentModel, _ index: Int) -> [PropertyRow] {
311 var keys: [String] = []
312 func add(_ entries: [Property], inheritedOnly: Bool) {
313 for entry in entries {
314 let key = entry.key.uppercased()
315 if !keys.contains(key), !inheritedOnly || model.inherits(key) { keys.append(key) }
316 }
317 }
318 add(model.headings[index].properties, inheritedOnly: false)
319 add(model.fileProperties, inheritedOnly: true)
320 for ancestor in model.ancestors(of: index) { add(model.headings[ancestor].properties, inheritedOnly: true) }
321 return keys.compactMap { key in
322 model.property(key, of: index).map {
323 PropertyRow(key: key, value: $0.value, inherited: $0.source != .heading(index))
324 }
325 }
326 }
327
328 static func timestamps(_ heading: HeadingInfo) -> [TimestampRow] {
329 var rows: [TimestampRow] = []
330 func add(_ stamp: Timestamp?, _ kind: TimestampRow.Kind) {
331 guard let stamp else { return }
332 rows.append(TimestampRow(
333 kind: kind,
334 start: format(stamp.start),
335 end: stamp.end.map(format),
336 repeater: stamp.repeater.map {
337 $0.kind.rawValue + format($0.interval) + ($0.habitDeadline.map { "/" + format($0) } ?? "")
338 },
339 warning: stamp.warning.map { ($0.firstOccurrenceOnly ? "--" : "-") + format($0.interval) }
340 ))
341 }
342 add(heading.scheduled, .scheduled)
343 add(heading.deadline, .deadline)
344 add(heading.closed, .closed)
345 for stamp in heading.timestamps { add(stamp, stamp.active ? .active : .inactive) }
346 return rows
347 }
348
349 static func format(_ point: Timestamp.Point) -> String {
350 let date = String(format: "%04d-%02d-%02d", point.year, point.month, point.day)
351 guard let hour = point.hour, let minute = point.minute else { return date }
352 return date + String(format: "T%02d:%02d", hour, minute)
353 }
354
355 static func format(_ interval: Timestamp.Interval) -> String {
356 "\(interval.value)\(interval.unit.rawValue)"
357 }
358}
359```
360
361- [ ] **Step 4: Run to verify pass, then commit**
362
363Run: `swift test --filter FileRecordTests`
364
365```bash
366git add Package.swift Package.resolved Sources/OrgIndex/FileRecord.swift Tests/OrgIndexTests/FileRecordTests.swift
367git commit -m "Add index file records"
368```
369
370---
371
372### Task 2: Index store
373
374**Files:**
375- Create: `Sources/OrgIndex/IndexStore.swift`
376- Test: `Tests/OrgIndexTests/IndexStoreTests.swift`
377
378**Interfaces:**
379- Consumes: `FileRecord` (Task 1).
380- Produces: `IndexStore(path:)`, `write(_:)`, `apply(_ change: IndexChange)`, `fileStates(root:) -> [String: FileState]`, `files()`, `search(_:overlay:limit:)`, `headings(withID:overlay:)`, `duplicateIDs(overlay:)`; `FileState`, `HeadingLocation`, `IndexChange`.
381
382- [ ] **Step 1: Write the failing tests**
383
384```swift
385import OrgCore
386import Testing
387@testable import OrgIndex
388
389struct IndexStoreTests {
390 @Test func searchFindsTitlesAndBodies() throws {
391 let store = try IndexStore()
392 try store.write(record("/notes/a.org", "* Groceries\nbuy apples\n* Taxes\nfile forms\n"))
393 #expect(try store.search("appl").map(\.title) == ["Groceries"])
394 #expect(try store.search("tax").map(\.title) == ["Taxes"])
395 #expect(try store.search("file forms").map(\.title) == ["Taxes"])
396 #expect(try store.search("\"oops").isEmpty)
397 #expect(try store.search(" ").isEmpty)
398 }
399
400 @Test func rewritingAFileReplacesItsRows() throws {
401 let store = try IndexStore()
402 try store.write(record("/notes/a.org", "* Old title\n"))
403 try store.write(record("/notes/a.org", "* New title\n"))
404 #expect(try store.search("old").isEmpty)
405 #expect(try store.search("new").count == 1)
406 }
407
408 @Test func removeMoveAndTouch() throws {
409 let store = try IndexStore()
410 try store.write(record("/notes/a.org", "* Alpha\n"))
411 try store.write(record("/notes/b.org", "* Beta\n"))
412 var change = IndexChange()
413 change.removals = ["/notes/a.org"]
414 change.moves = [(from: "/notes/b.org", to: "/notes/c.org", mtime: 5)]
415 try store.apply(change)
416 #expect(try store.search("alpha").isEmpty)
417 #expect(try store.search("beta").map(\.path) == ["/notes/c.org"])
418 #expect(try store.fileStates(root: "/notes")["/notes/c.org"]?.mtime == 5)
419 }
420
421 @Test func overlayReplacesIndexedRows() throws {
422 let store = try IndexStore()
423 try store.write(record("/notes/a.org", "* Saved title\n"))
424 let overlay = ["/notes/a.org": record("/notes/a.org", "* Unsaved title\n")]
425 #expect(try store.search("saved", overlay: overlay).isEmpty)
426 #expect(try store.search("unsaved", overlay: overlay).map(\.title) == ["Unsaved title"])
427 }
428
429 @Test func idsAndDuplicates() throws {
430 let store = try IndexStore()
431 try store.write(record("/notes/a.org", "* A\n:PROPERTIES:\n:ID: x\n:END:\n"))
432 try store.write(record("/notes/b.org", "* B\n:PROPERTIES:\n:ID: y\n:END:\n"))
433 #expect(try store.headings(withID: "x").map(\.path) == ["/notes/a.org"])
434 #expect(try store.duplicateIDs().isEmpty)
435 let overlay = ["/notes/b.org": record("/notes/b.org", "* B\n:PROPERTIES:\n:ID: x\n:END:\n")]
436 #expect(try store.headings(withID: "x", overlay: overlay).map(\.path) == ["/notes/a.org", "/notes/b.org"])
437 #expect(try store.duplicateIDs(overlay: overlay).keys.sorted() == ["x"])
438 }
439
440 @Test func fileStatesAndKinds() throws {
441 let store = try IndexStore()
442 let a = record("/notes/a.org", "* A\n")
443 try store.write(a)
444 try store.write(record("/notes/a.sync-conflict-1.org", "* A\n", kind: .conflict))
445 let states = try store.fileStates(root: "/notes")
446 #expect(states["/notes/a.org"] == FileState(kind: .org, size: a.size, mtime: 1, hash: a.hash, settingsVersion: 1))
447 #expect(try store.files().map(\.kind) == [.org, .conflict])
448 #expect(try store.search("a").map(\.path) == ["/notes/a.org"])
449 }
450}
451```
452
453- [ ] **Step 2: Implement**
454
455```swift
456import Foundation
457import GRDB
458
459/// What the index knows about a file without reading it.
460public struct FileState: Sendable, Equatable {
461 public let kind: FileKind
462 public let size: Int
463 public let mtime: Double
464 public let hash: String
465 public let settingsVersion: Int
466}
467
468/// A heading as found by a query. `contentHash` is the hash of the text the offsets refer to,
469/// so a caller can tell whether they still apply to an open buffer.
470public struct HeadingLocation: Sendable, Equatable {
471 public let path: String
472 public let ordinal: Int
473 public let title: String
474 public let start: Int
475 public let contentHash: String
476}
477
478/// One reconciliation's worth of changes, applied in a single transaction.
479public struct IndexChange: Sendable {
480 public var records: [FileRecord] = []
481 /// Unchanged content with a new modification time.
482 public var touches: [(path: String, mtime: Double)] = []
483 /// Renamed files whose content didn't change.
484 public var moves: [(from: String, to: String, mtime: Double)] = []
485 public var removals: [String] = []
486
487 public init() {}
488
489 public var isEmpty: Bool { records.isEmpty && touches.isEmpty && moves.isEmpty && removals.isEmpty }
490}
491
492/// The SQLite index. A cache: deleting it loses nothing that the files don't hold.
493public final class IndexStore: Sendable {
494 let database: DatabaseQueue
495
496 /// `path` nil opens an in-memory index.
497 public init(path: String? = nil) throws {
498 database = try path.map { try DatabaseQueue(path: $0) } ?? DatabaseQueue()
499 try Self.migrator.migrate(database)
500 }
501
502 static var migrator: DatabaseMigrator {
503 var migrator = DatabaseMigrator()
504 migrator.registerMigration("v1") { db in
505 try db.execute(sql: """
506 CREATE TABLE files (
507 id INTEGER PRIMARY KEY,
508 path TEXT NOT NULL UNIQUE,
509 root TEXT NOT NULL,
510 kind TEXT NOT NULL,
511 size INTEGER NOT NULL,
512 mtime REAL NOT NULL,
513 hash TEXT NOT NULL,
514 settings_version INTEGER NOT NULL,
515 parsed_at REAL NOT NULL
516 );
517 CREATE INDEX files_root ON files(root);
518 CREATE TABLE headings (
519 id INTEGER PRIMARY KEY,
520 file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
521 ordinal INTEGER NOT NULL,
522 parent_ordinal INTEGER,
523 start_offset INTEGER NOT NULL,
524 end_offset INTEGER NOT NULL,
525 level INTEGER NOT NULL,
526 todo TEXT,
527 is_done INTEGER NOT NULL,
528 priority TEXT,
529 title TEXT NOT NULL,
530 outline_path TEXT NOT NULL,
531 org_id TEXT,
532 archived INTEGER NOT NULL
533 );
534 CREATE INDEX headings_file ON headings(file_id);
535 CREATE INDEX headings_org_id ON headings(org_id);
536 CREATE TABLE tags (
537 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
538 tag TEXT NOT NULL,
539 inherited INTEGER NOT NULL
540 );
541 CREATE INDEX tags_heading ON tags(heading_id);
542 CREATE TABLE properties (
543 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
544 key TEXT NOT NULL,
545 value TEXT NOT NULL,
546 inherited INTEGER NOT NULL
547 );
548 CREATE INDEX properties_heading ON properties(heading_id);
549 CREATE TABLE timestamps (
550 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
551 kind TEXT NOT NULL,
552 start_at TEXT NOT NULL,
553 end_at TEXT,
554 repeater TEXT,
555 warning TEXT
556 );
557 CREATE INDEX timestamps_heading ON timestamps(heading_id);
558 CREATE TABLE clocks (
559 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
560 start_at TEXT NOT NULL,
561 end_at TEXT,
562 minutes INTEGER
563 );
564 CREATE INDEX clocks_heading ON clocks(heading_id);
565 CREATE TABLE links (
566 heading_id INTEGER NOT NULL REFERENCES headings(id) ON DELETE CASCADE,
567 type TEXT NOT NULL,
568 target TEXT NOT NULL
569 );
570 CREATE INDEX links_heading ON links(heading_id);
571 CREATE VIRTUAL TABLE headings_fts USING fts5(title, body, tokenize = 'unicode61 remove_diacritics 2');
572 CREATE TRIGGER headings_fts_delete AFTER DELETE ON headings BEGIN
573 DELETE FROM headings_fts WHERE rowid = old.id;
574 END;
575 """)
576 }
577 return migrator
578 }
579
580 // MARK: - Writing
581
582 public func write(_ record: FileRecord) throws {
583 var change = IndexChange()
584 change.records = [record]
585 try apply(change)
586 }
587
588 public func apply(_ change: IndexChange) throws {
589 guard !change.isEmpty else { return }
590 try database.write { db in
591 for path in change.removals {
592 try db.execute(sql: "DELETE FROM files WHERE path = ?", arguments: [path])
593 }
594 for move in change.moves {
595 try db.execute(sql: "UPDATE files SET path = ?, mtime = ? WHERE path = ?", arguments: [move.to, move.mtime, move.from])
596 }
597 for touch in change.touches {
598 try db.execute(sql: "UPDATE files SET mtime = ? WHERE path = ?", arguments: [touch.mtime, touch.path])
599 }
600 for record in change.records {
601 try insert(record, db)
602 }
603 }
604 }
605
606 private func insert(_ record: FileRecord, _ db: Database) throws {
607 try db.execute(sql: "DELETE FROM files WHERE path = ?", arguments: [record.path])
608 try db.execute(
609 sql: """
610 INSERT INTO files (path, root, kind, size, mtime, hash, settings_version, parsed_at)
611 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
612 """,
613 arguments: [record.path, record.root, record.kind.rawValue, record.size, record.mtime, record.hash,
614 record.settingsVersion, Date().timeIntervalSince1970]
615 )
616 let fileID = db.lastInsertedRowID
617 for heading in record.headings {
618 try db.execute(
619 sql: """
620 INSERT INTO headings (file_id, ordinal, parent_ordinal, start_offset, end_offset, level, todo,
621 is_done, priority, title, outline_path, org_id, archived)
622 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
623 """,
624 arguments: [fileID, heading.ordinal, heading.parent, heading.start, heading.end, heading.level, heading.todo,
625 heading.isDone, heading.priority, heading.title, heading.outlinePath.joined(separator: "\u{1F}"),
626 heading.orgID, heading.archived]
627 )
628 let id = db.lastInsertedRowID
629 try db.execute(sql: "INSERT INTO headings_fts (rowid, title, body) VALUES (?, ?, ?)", arguments: [id, heading.title, heading.body])
630 for tag in heading.tags {
631 try db.execute(sql: "INSERT INTO tags VALUES (?, ?, ?)", arguments: [id, tag.name, tag.inherited])
632 }
633 for property in heading.properties {
634 try db.execute(sql: "INSERT INTO properties VALUES (?, ?, ?, ?)", arguments: [id, property.key, property.value, property.inherited])
635 }
636 for stamp in heading.timestamps {
637 try db.execute(
638 sql: "INSERT INTO timestamps VALUES (?, ?, ?, ?, ?, ?)",
639 arguments: [id, stamp.kind.rawValue, stamp.start, stamp.end, stamp.repeater, stamp.warning]
640 )
641 }
642 for clock in heading.clocks {
643 try db.execute(sql: "INSERT INTO clocks VALUES (?, ?, ?, ?)", arguments: [id, clock.start, clock.end, clock.minutes])
644 }
645 for link in heading.links {
646 try db.execute(sql: "INSERT INTO links VALUES (?, ?, ?)", arguments: [id, link.type, link.target])
647 }
648 }
649 }
650
651 // MARK: - Reading
652
653 /// Indexed files under `root`, by path.
654 public func fileStates(root: String) throws -> [String: FileState] {
655 try database.read { db in
656 let rows = try Row.fetchAll(db, sql: "SELECT path, kind, size, mtime, hash, settings_version FROM files WHERE root = ?", arguments: [root])
657 var states: [String: FileState] = [:]
658 for row in rows {
659 states[row["path"]] = FileState(
660 kind: FileKind(rawValue: row["kind"]) ?? .org,
661 size: row["size"], mtime: row["mtime"], hash: row["hash"], settingsVersion: row["settings_version"]
662 )
663 }
664 return states
665 }
666 }
667
668 public func files() throws -> [(path: String, kind: FileKind)] {
669 try database.read { db in
670 try Row.fetchAll(db, sql: "SELECT path, kind FROM files ORDER BY path").map {
671 (path: $0["path"], kind: FileKind(rawValue: $0["kind"]) ?? .org)
672 }
673 }
674 }
675
676 // MARK: - Queries
677 //
678 // `overlay` holds records for open documents with unsaved edits, keyed by path. Their rows
679 // replace that file's indexed rows in every query.
680
681 /// Headings whose title or body contain every word of `query` as a word prefix.
682 public func search(_ query: String, overlay: [String: FileRecord] = [:], limit: Int = 50) throws -> [HeadingLocation] {
683 let terms = query.split(whereSeparator: \.isWhitespace).map(String.init)
684 guard !terms.isEmpty else { return [] }
685 let match = terms.map { "\"" + $0.replacingOccurrences(of: "\"", with: "\"\"") + "\"*" }.joined(separator: " ")
686 let indexed = try database.read { db in
687 try Row.fetchAll(
688 db,
689 sql: """
690 SELECT f.path, f.hash, h.ordinal, h.title, h.start_offset
691 FROM headings_fts
692 JOIN headings h ON h.id = headings_fts.rowid
693 JOIN files f ON f.id = h.file_id
694 WHERE headings_fts MATCH ?
695 ORDER BY bm25(headings_fts)
696 LIMIT ?
697 """,
698 arguments: [match, limit + overlay.count * 10]
699 ).map(location)
700 }
701 // Unsaved buffers get the same word-prefix matching as the full-text index.
702 let prefixes = terms.flatMap(Self.words)
703 let live = overlay.values.sorted { $0.path < $1.path }.flatMap { record in
704 record.headings.filter { heading in
705 let words = Self.words(heading.title + "\n" + heading.body)
706 return prefixes.allSatisfy { prefix in words.contains { $0.hasPrefix(prefix) } }
707 }.map { location(record, $0) }
708 }
709 return Array((live + indexed.filter { overlay[$0.path] == nil }).prefix(limit))
710 }
711
712 /// Headings with `:ID: id`. More than one means the ID is duplicated.
713 public func headings(withID id: String, overlay: [String: FileRecord] = [:]) throws -> [HeadingLocation] {
714 let indexed = try database.read { db in
715 try Row.fetchAll(
716 db,
717 sql: """
718 SELECT f.path, f.hash, h.ordinal, h.title, h.start_offset
719 FROM headings h JOIN files f ON f.id = h.file_id
720 WHERE h.org_id = ? ORDER BY f.path, h.ordinal
721 """,
722 arguments: [id]
723 ).map(location)
724 }
725 let live = overlay.values.sorted { $0.path < $1.path }.flatMap { record in
726 record.headings.filter { $0.orgID == id }.map { location(record, $0) }
727 }
728 return indexed.filter { overlay[$0.path] == nil } + live
729 }
730
731 /// IDs used by more than one heading.
732 public func duplicateIDs(overlay: [String: FileRecord] = [:]) throws -> [String: [HeadingLocation]] {
733 let indexed = try database.read { db in
734 try Row.fetchAll(
735 db,
736 sql: """
737 SELECT h.org_id, f.path, f.hash, h.ordinal, h.title, h.start_offset
738 FROM headings h JOIN files f ON f.id = h.file_id
739 WHERE h.org_id IN (SELECT org_id FROM headings WHERE org_id IS NOT NULL GROUP BY org_id HAVING count(*) > 1)
740 OR (h.org_id IS NOT NULL AND ? > 0)
741 ORDER BY f.path, h.ordinal
742 """,
743 arguments: [overlay.count]
744 ).map { (id: $0["org_id"] as String, location: location($0)) }
745 }
746 var byID: [String: [HeadingLocation]] = [:]
747 for row in indexed where overlay[row.location.path] == nil {
748 byID[row.id, default: []].append(row.location)
749 }
750 for record in overlay.values.sorted(by: { $0.path < $1.path }) {
751 for heading in record.headings {
752 if let id = heading.orgID { byID[id, default: []].append(location(record, heading)) }
753 }
754 }
755 return byID.filter { $0.value.count > 1 }
756 }
757
758 /// Lower-cased runs of letters and digits, as the unicode61 tokenizer splits them.
759 static func words(_ text: String) -> [String] {
760 text.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
761 }
762
763 private func location(_ row: Row) -> HeadingLocation {
764 HeadingLocation(path: row["path"], ordinal: row["ordinal"], title: row["title"], start: row["start_offset"], contentHash: row["hash"])
765 }
766
767 private func location(_ record: FileRecord, _ heading: HeadingRecord) -> HeadingLocation {
768 HeadingLocation(path: record.path, ordinal: heading.ordinal, title: heading.title, start: heading.start, contentHash: record.hash)
769 }
770}
771```
772
773- [ ] **Step 3: Run to verify pass, then commit**
774
775Run: `swift test --filter IndexStoreTests`
776
777```bash
778git add Sources/OrgIndex/IndexStore.swift Tests/OrgIndexTests/IndexStoreTests.swift
779git commit -m "Add SQLite index store with search and overlay"
780```
781
782---
783
784### Task 3: Discovery and agenda scope
785
786**Files:**
787- Modify: `Package.swift` (add `OrgWorkspace`)
788- Create: `Sources/OrgWorkspace/Discovery.swift`
789- Test: `Tests/OrgWorkspaceTests/Folder.swift`, `Tests/OrgWorkspaceTests/DiscoveryTests.swift`
790
791**Interfaces:**
792- Consumes: `FileKind` (Task 1).
793- Produces: `WorkspaceRoot(url:)` with `resolve()`, `DiscoveredFile`, `Discovery`, `FileClassification`, `WorkspaceScanner.classify(_:)`, `WorkspaceScanner.scan(_:)`, `AgendaScope(globs:)` with `contains(relativePath:kind:)`; test helper `Folder`.
794
795- [ ] **Step 1: Add the target**
796
797```swift
798// swift-tools-version: 6.2
799import PackageDescription
800
801let package = Package(
802 name: "Orgstar",
803 platforms: [.macOS(.v26), .iOS(.v26)],
804 products: [
805 .library(name: "OrgCore", targets: ["OrgCore"]),
806 .library(name: "OrgDocument", targets: ["OrgDocument"]),
807 .library(name: "OrgIndex", targets: ["OrgIndex"]),
808 .library(name: "OrgWorkspace", targets: ["OrgWorkspace"])
809 ],
810 dependencies: [
811 .package(url: "https://github.com/groue/GRDB.swift", from: "7.11.1")
812 ],
813 targets: [
814 .target(name: "OrgCore"),
815 .target(name: "OrgDocument", dependencies: ["OrgCore"]),
816 .target(name: "OrgIndex", dependencies: ["OrgCore", .product(name: "GRDB", package: "GRDB.swift")]),
817 .target(name: "OrgWorkspace", dependencies: ["OrgIndex"]),
818 .testTarget(name: "OrgCoreTests", dependencies: ["OrgCore"]),
819 .testTarget(name: "OrgDocumentTests", dependencies: ["OrgDocument"]),
820 .testTarget(name: "OrgIndexTests", dependencies: ["OrgIndex"]),
821 .testTarget(name: "OrgWorkspaceTests", dependencies: ["OrgWorkspace"])
822 ]
823)
824```
825
826- [ ] **Step 2: Write the failing tests**
827
828```swift
829import Foundation
830
831/// A temporary folder that removes itself.
832final class Folder {
833 let url: URL
834
835 init() throws {
836 url = FileManager.default.temporaryDirectory.appendingPathComponent("orgstar-\(UUID().uuidString)").standardizedFileURL
837 try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)
838 }
839
840 deinit {
841 try? FileManager.default.removeItem(at: url)
842 }
843
844 @discardableResult
845 func write(_ path: String, _ text: String) throws -> URL {
846 let file = url.appendingPathComponent(path)
847 try FileManager.default.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true)
848 try Data(text.utf8).write(to: file)
849 return file.standardizedFileURL
850 }
851}
852```
853
854```swift
855import Foundation
856import OrgIndex
857import Testing
858@testable import OrgWorkspace
859
860struct ClassificationTests {
861 @Test(arguments: [
862 ("a.org", FileClassification.file(.org)),
863 ("a.org_archive", .file(.archive)),
864 ("a.sync-conflict-20261004-120000-ABCDEFG.org", .file(.conflict)),
865 (".a.org.icloud", .placeholder("a.org")),
866 (".syncthing.a.org.tmp", .ignored),
867 (".a.org.orgstar-1234", .ignored),
868 ("a.org~", .ignored),
869 ("#a.org#", .ignored),
870 ("a.md", .ignored),
871 (".hidden.org", .ignored),
872 ])
873 func classify(name: String, expected: FileClassification) {
874 #expect(WorkspaceScanner.classify(name) == expected)
875 }
876
877 @Test func agendaScope() {
878 #expect(AgendaScope().contains(relativePath: "a.org", kind: .org))
879 #expect(!AgendaScope().contains(relativePath: "a.org_archive", kind: .archive))
880 #expect(!AgendaScope().contains(relativePath: "a.org", kind: .conflict))
881 let work = AgendaScope(globs: ["work/*"])
882 #expect(work.contains(relativePath: "work/deep/a.org", kind: .org))
883 #expect(!work.contains(relativePath: "home/a.org", kind: .org))
884 }
885}
886
887struct ScanTests {
888 @Test func skipsHiddenFoldersAndFindsPlaceholders() throws {
889 let folder = try Folder()
890 try folder.write("a.org", "* a\n")
891 try folder.write("sub/b.org_archive", "* b\n")
892 try folder.write(".git/c.org", "* c\n")
893 try folder.write(".stversions/d.org", "* d\n")
894 try folder.write("sub/.e.org.icloud", "")
895 try folder.write("notes.txt", "")
896 let discovery = try WorkspaceScanner.scan(folder.url)
897 #expect(discovery.files.map(\.url.lastPathComponent) == ["a.org", "b.org_archive"])
898 #expect(discovery.placeholders.map(\.lastPathComponent) == ["e.org"])
899 }
900}
901```
902
903- [ ] **Step 3: Implement**
904
905```swift
906import Foundation
907import OrgIndex
908
909/// A folder the user added. Identified by its bookmark, so a moved folder is followed.
910public struct WorkspaceRoot: Sendable, Equatable, Codable {
911 public let bookmark: Data
912 /// The path when the bookmark was made or last refreshed, for display.
913 public var path: String
914
915 public init(url: URL) throws {
916 bookmark = try url.bookmarkData(options: [], includingResourceValuesForKeys: nil, relativeTo: nil)
917 path = url.standardizedFileURL.path
918 }
919
920 /// The folder's current location. `isStale` means the bookmark should be made again.
921 public func resolve() throws -> (url: URL, isStale: Bool) {
922 var isStale = false
923 let url = try URL(resolvingBookmarkData: bookmark, options: [], relativeTo: nil, bookmarkDataIsStale: &isStale)
924 _ = url.startAccessingSecurityScopedResource()
925 return (url.standardizedFileURL, isStale)
926 }
927}
928
929public struct DiscoveredFile: Sendable, Equatable {
930 public let url: URL
931 public let kind: FileKind
932 public let size: Int
933 public let mtime: Double
934}
935
936public struct Discovery: Sendable, Equatable {
937 public var files: [DiscoveredFile] = []
938 /// iCloud files not downloaded yet, by the name they will have.
939 public var placeholders: [URL] = []
940}
941
942public enum FileClassification: Sendable, Equatable {
943 case file(FileKind)
944 /// An iCloud placeholder (`.name.org.icloud`) for the file at the given name.
945 case placeholder(String)
946 case ignored
947}
948
949public enum WorkspaceScanner {
950 /// What a file name means to the workspace. Syncthing temp files, our own temp and backup
951 /// files, Emacs backups and auto-saves, and other hidden files are ignored.
952 public static func classify(_ name: String) -> FileClassification {
953 if name.hasPrefix(".") {
954 if name.hasSuffix(".icloud") {
955 let inner = String(name.dropFirst().dropLast(".icloud".count))
956 if case .file = classify(inner) { return .placeholder(inner) }
957 }
958 return .ignored
959 }
960 if name.hasSuffix("~") || name.hasPrefix("#") { return .ignored }
961 let conflict = name.contains(".sync-conflict-")
962 if name.hasSuffix(".org") { return .file(conflict ? .conflict : .org) }
963 if name.hasSuffix(".org_archive") { return .file(conflict ? .conflict : .archive) }
964 return .ignored
965 }
966
967 /// Every org file under `root`, skipping hidden folders (`.git`, `.stfolder`, `.stversions`).
968 public static func scan(_ root: URL) throws -> Discovery {
969 let keys: [URLResourceKey] = [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey]
970 guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: keys) else {
971 return Discovery()
972 }
973 var discovery = Discovery()
974 for case let url as URL in enumerator {
975 let values = try url.resourceValues(forKeys: Set(keys))
976 let name = url.lastPathComponent
977 if values.isDirectory == true {
978 if name.hasPrefix(".") { enumerator.skipDescendants() }
979 continue
980 }
981 switch classify(name) {
982 case .file(let kind):
983 discovery.files.append(DiscoveredFile(
984 url: url.standardizedFileURL,
985 kind: kind,
986 size: values.fileSize ?? 0,
987 mtime: values.contentModificationDate?.timeIntervalSince1970 ?? 0
988 ))
989 case .placeholder(let inner):
990 discovery.placeholders.append(url.deletingLastPathComponent().appendingPathComponent(inner).standardizedFileURL)
991 case .ignored:
992 break
993 }
994 }
995 discovery.files.sort { $0.url.path < $1.url.path }
996 return discovery
997 }
998}
999
1000/// Which files feed the agenda: `.org` files (not archives or conflict copies) matching one of
1001/// the globs, relative to their root. No globs means every `.org` file.
1002public struct AgendaScope: Sendable, Equatable {
1003 public var globs: [String]
1004
1005 public init(globs: [String] = []) {
1006 self.globs = globs
1007 }
1008
1009 public func contains(relativePath: String, kind: FileKind) -> Bool {
1010 guard kind == .org else { return false }
1011 guard !globs.isEmpty else { return true }
1012 // Without FNM_PATHNAME, `*` also matches `/`, so `work/*` covers nested folders.
1013 return globs.contains { fnmatch($0, relativePath, 0) == 0 }
1014 }
1015}
1016```
1017
1018- [ ] **Step 4: Run to verify pass, then commit**
1019
1020Run: `swift test --filter "ClassificationTests|ScanTests"`
1021
1022```bash
1023git add Package.swift Sources/OrgWorkspace/Discovery.swift Tests/OrgWorkspaceTests/Folder.swift Tests/OrgWorkspaceTests/DiscoveryTests.swift
1024git commit -m "Add workspace discovery and agenda scope"
1025```
1026
1027---
1028
1029### Task 4: Reconciliation
1030
1031**Files:**
1032- Create: `Sources/OrgWorkspace/Reconciler.swift`
1033- Test: `Tests/OrgWorkspaceTests/ReconcileTests.swift`
1034
1035**Interfaces:**
1036- Consumes: Tasks 1–3.
1037- Produces: `Reconciler(index:settings:)` with `reconcile(root:)` and `reconcile(root:paths:)`, `ReconcileReport`.
1038
1039- [ ] **Step 1: Write the failing tests**
1040
1041```swift
1042import Foundation
1043import OrgIndex
1044import Testing
1045@testable import OrgWorkspace
1046
1047struct ReconcileTests {
1048 @Test func indexesUpdatesMovesAndRemoves() throws {
1049 let folder = try Folder()
1050 let index = try IndexStore()
1051 let reconciler = Reconciler(index: index, settings: IndexSettings())
1052 let a = try folder.write("a.org", "* Alpha\n")
1053 try folder.write("b.org", "* Beta\n")
1054
1055 var report = try reconciler.reconcile(root: folder.url)
1056 #expect(report.indexed.count == 2)
1057 #expect(try index.search("alpha").map(\.path) == [a.path])
1058
1059 report = try reconciler.reconcile(root: folder.url)
1060 #expect(report.unchanged == 2 && report.indexed.isEmpty)
1061
1062 try FileManager.default.moveItem(at: folder.url.appendingPathComponent("b.org"), to: folder.url.appendingPathComponent("c.org"))
1063 try folder.write("a.org", "* Gamma\n")
1064 report = try reconciler.reconcile(root: folder.url)
1065 #expect(report.moved.map { URL(fileURLWithPath: $0).lastPathComponent } == ["c.org"])
1066 #expect(report.indexed == [a.path])
1067 #expect(try index.search("gamma").count == 1)
1068 #expect(try index.search("beta").map { URL(fileURLWithPath: $0.path).lastPathComponent } == ["c.org"])
1069
1070 try FileManager.default.removeItem(at: a)
1071 report = try reconciler.reconcile(root: folder.url)
1072 #expect(report.removed == [a.path])
1073 #expect(try index.search("gamma").isEmpty)
1074 }
1075
1076 @Test func touchedFileWithSameContentIsNotReparsed() throws {
1077 let folder = try Folder()
1078 let index = try IndexStore()
1079 let reconciler = Reconciler(index: index, settings: IndexSettings())
1080 let a = try folder.write("a.org", "* Alpha\n")
1081 _ = try reconciler.reconcile(root: folder.url)
1082 try FileManager.default.setAttributes([.modificationDate: Date(timeIntervalSinceNow: 60)], ofItemAtPath: a.path)
1083 #expect(try reconciler.reconcile(root: folder.url).touched == [a.path])
1084 }
1085
1086 @Test func newSettingsVersionReindexes() throws {
1087 let folder = try Folder()
1088 let index = try IndexStore()
1089 try folder.write("a.org", "* Alpha\n")
1090 _ = try Reconciler(index: index, settings: IndexSettings(version: 1)).reconcile(root: folder.url)
1091 #expect(try Reconciler(index: index, settings: IndexSettings(version: 2)).reconcile(root: folder.url).indexed.count == 1)
1092 }
1093
1094 @Test func conflictCopiesAreListedButNotSearchable() throws {
1095 let folder = try Folder()
1096 let index = try IndexStore()
1097 try folder.write("a.sync-conflict-1.org", "* Alpha\n")
1098 _ = try Reconciler(index: index, settings: IndexSettings()).reconcile(root: folder.url)
1099 #expect(try index.files().map(\.kind) == [.conflict])
1100 #expect(try index.search("alpha").isEmpty)
1101 }
1102
1103 @Test func reconcilingChangedPaths() throws {
1104 let folder = try Folder()
1105 let index = try IndexStore()
1106 let reconciler = Reconciler(index: index, settings: IndexSettings())
1107 let a = try folder.write("a.org", "* Alpha\n")
1108 let b = try folder.write("b.org", "* Beta\n")
1109 _ = try reconciler.reconcile(root: folder.url)
1110 try folder.write("a.org", "* Delta\n")
1111 try FileManager.default.removeItem(at: b)
1112 let report = try reconciler.reconcile(root: folder.url, paths: [a, b])
1113 #expect(report.indexed == [a.path])
1114 #expect(report.removed == [b.path])
1115 }
1116
1117 @Test func rootBookmarkResolves() throws {
1118 let folder = try Folder()
1119 let root = try WorkspaceRoot(url: folder.url)
1120 #expect(try root.resolve().url.resolvingSymlinksInPath() == folder.url.resolvingSymlinksInPath())
1121 }
1122}
1123```
1124
1125- [ ] **Step 2: Implement**
1126
1127```swift
1128import Foundation
1129import OrgIndex
1130
1131public struct ReconcileReport: Sendable, Equatable {
1132 public var indexed: [String] = []
1133 public var touched: [String] = []
1134 public var moved: [String] = []
1135 public var removed: [String] = []
1136 public var unchanged = 0
1137 /// iCloud files asked to download; they are indexed once they arrive.
1138 public var placeholders: [URL] = []
1139}
1140
1141/// Brings the index in line with the files under a root. File events are only hints, so this
1142/// is what runs at launch, after dropped events, and for every batch of changed paths.
1143public struct Reconciler: Sendable {
1144 public let index: IndexStore
1145 public let settings: IndexSettings
1146
1147 public init(index: IndexStore, settings: IndexSettings) {
1148 self.index = index
1149 self.settings = settings
1150 }
1151
1152 /// Reconciles the whole root.
1153 public func reconcile(root: URL) throws -> ReconcileReport {
1154 let discovery = try WorkspaceScanner.scan(root)
1155 for placeholder in discovery.placeholders {
1156 try? FileManager.default.startDownloadingUbiquitousItem(at: placeholder)
1157 }
1158 var report = try reconcile(root: root, found: discovery.files, known: index.fileStates(root: root.path))
1159 report.placeholders = discovery.placeholders
1160 return report
1161 }
1162
1163 /// Reconciles only `paths` under the root, for a batch of file events.
1164 public func reconcile(root: URL, paths: [URL]) throws -> ReconcileReport {
1165 let wanted = Set(paths.map(\.standardizedFileURL.path))
1166 let known = try index.fileStates(root: root.path).filter { wanted.contains($0.key) }
1167 var found: [DiscoveredFile] = []
1168 for url in paths.map(\.standardizedFileURL) {
1169 guard case .file(let kind) = WorkspaceScanner.classify(url.lastPathComponent),
1170 let values = try? url.resourceValues(forKeys: [.fileSizeKey, .contentModificationDateKey, .isDirectoryKey]),
1171 values.isDirectory != true else { continue }
1172 found.append(DiscoveredFile(
1173 url: url, kind: kind, size: values.fileSize ?? 0,
1174 mtime: values.contentModificationDate?.timeIntervalSince1970 ?? 0
1175 ))
1176 }
1177 return try reconcile(root: root, found: found, known: known)
1178 }
1179
1180 private func reconcile(root: URL, found: [DiscoveredFile], known: [String: FileState]) throws -> ReconcileReport {
1181 var report = ReconcileReport()
1182 var change = IndexChange()
1183 var candidates: [(file: DiscoveredFile, bytes: [UInt8], hash: String)] = []
1184
1185 for file in found {
1186 let path = file.url.path
1187 if let state = known[path], state.size == file.size, state.mtime == file.mtime,
1188 state.settingsVersion == settings.version, state.kind == file.kind {
1189 report.unchanged += 1
1190 continue
1191 }
1192 guard let data = try? Data(contentsOf: file.url) else { continue }
1193 let bytes = [UInt8](data)
1194 let hash = FileRecord.hash(bytes)
1195 if let state = known[path], state.hash == hash, state.settingsVersion == settings.version, state.kind == file.kind {
1196 change.touches.append((path, file.mtime))
1197 report.touched.append(path)
1198 } else {
1199 candidates.append((file, bytes, hash))
1200 }
1201 }
1202
1203 // A file that vanished and one that appeared with the same content is a rename.
1204 let foundPaths = Set(found.map(\.url.path))
1205 var missing = known.filter { !foundPaths.contains($0.key) }
1206 for candidate in candidates {
1207 let path = candidate.file.url.path
1208 if known[path] == nil,
1209 let (oldPath, _) = missing.first(where: {
1210 $0.value.hash == candidate.hash && $0.value.settingsVersion == settings.version && $0.value.kind == candidate.file.kind
1211 }) {
1212 missing[oldPath] = nil
1213 change.moves.append((oldPath, path, candidate.file.mtime))
1214 report.moved.append(path)
1215 } else {
1216 change.records.append(FileRecord(
1217 path: path, root: root.path, kind: candidate.file.kind,
1218 bytes: candidate.bytes, mtime: candidate.file.mtime, settings: settings
1219 ))
1220 report.indexed.append(path)
1221 }
1222 }
1223 change.removals = missing.keys.sorted()
1224 report.removed = change.removals
1225 try index.apply(change)
1226 return report
1227 }
1228}
1229```
1230
1231- [ ] **Step 3: Run to verify pass, then commit**
1232
1233Run: `swift test --filter ReconcileTests`
1234
1235```bash
1236git add Sources/OrgWorkspace/Reconciler.swift Tests/OrgWorkspaceTests/ReconcileTests.swift
1237git commit -m "Add index reconciliation"
1238```
1239
1240---
1241
1242### Task 5: FSEvents watcher
1243
1244**Files:**
1245- Create: `Sources/OrgWorkspace/FSEventsWatcher.swift`
1246- Test: `Tests/OrgWorkspaceTests/WatcherTests.swift`
1247
1248**Interfaces:**
1249- Produces (macOS): `WatchEvent` (`.changed(URL)`, `.rescan(URL)`), `FSEventsWatcher(roots:latency:handler:)` with `start()`, `stop()`.
1250
1251- [ ] **Step 1: Write the failing test**
1252
1253```swift
1254import Foundation
1255import OrgIndex
1256import Testing
1257@testable import OrgWorkspace
1258
1259struct WatcherTests {
1260 @Test func reportsAWrittenFile() throws {
1261 let folder = try Folder()
1262 let received = Received()
1263 let watcher = FSEventsWatcher(roots: [folder.url.resolvingSymlinksInPath()], latency: 0.05) { received.add($0) }
1264 watcher.start()
1265 defer { watcher.stop() }
1266 Thread.sleep(forTimeInterval: 0.2)
1267 try folder.write("a.org", "* a\n")
1268 #expect(received.wait(for: "a.org", timeout: 5))
1269 }
1270}
1271
1272final class Received: @unchecked Sendable {
1273 private let lock = NSLock()
1274 private var events: [WatchEvent] = []
1275
1276 func add(_ batch: [WatchEvent]) {
1277 lock.withLock { events += batch }
1278 }
1279
1280 func wait(for name: String, timeout: TimeInterval) -> Bool {
1281 let deadline = Date(timeIntervalSinceNow: timeout)
1282 while Date() < deadline {
1283 let found = lock.withLock {
1284 events.contains { if case .changed(let url) = $0 { return url.lastPathComponent == name } else { return false } }
1285 }
1286 if found { return true }
1287 Thread.sleep(forTimeInterval: 0.05)
1288 }
1289 return false
1290 }
1291}
1292```
1293
1294- [ ] **Step 2: Implement**
1295
1296```swift
1297#if os(macOS)
1298import CoreServices
1299import Foundation
1300
1301public enum WatchEvent: Sendable, Equatable {
1302 case changed(URL)
1303 /// Events were dropped or coalesced for this folder, or a root moved: rescan it.
1304 case rescan(URL)
1305}
1306
1307/// FSEvents with file-level events. Delivers batches on a private queue.
1308public final class FSEventsWatcher: @unchecked Sendable {
1309 private let paths: [String]
1310 private let latency: CFTimeInterval
1311 private let handler: @Sendable ([WatchEvent]) -> Void
1312 private let queue = DispatchQueue(label: "orgstar.fsevents")
1313 private var stream: FSEventStreamRef?
1314
1315 public init(roots: [URL], latency: CFTimeInterval = 0.3, handler: @escaping @Sendable ([WatchEvent]) -> Void) {
1316 paths = roots.map(\.path)
1317 self.latency = latency
1318 self.handler = handler
1319 }
1320
1321 deinit {
1322 stop()
1323 }
1324
1325 public func start() {
1326 guard stream == nil else { return }
1327 var context = FSEventStreamContext(
1328 version: 0, info: Unmanaged.passUnretained(self).toOpaque(), retain: nil, release: nil, copyDescription: nil
1329 )
1330 let flags = FSEventStreamCreateFlags(
1331 kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagUseCFTypes
1332 | kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot
1333 )
1334 let callback: FSEventStreamCallback = { _, info, count, eventPaths, eventFlags, _ in
1335 guard let info else { return }
1336 let watcher = Unmanaged<FSEventsWatcher>.fromOpaque(info).takeUnretainedValue()
1337 let paths = unsafeBitCast(eventPaths, to: NSArray.self) as? [String] ?? []
1338 watcher.deliver(paths: paths, flags: Array(UnsafeBufferPointer(start: eventFlags, count: count)))
1339 }
1340 guard let stream = FSEventStreamCreate(
1341 nil, callback, &context, paths as CFArray, FSEventStreamEventId(kFSEventStreamEventIdSinceNow), latency, flags
1342 ) else { return }
1343 FSEventStreamSetDispatchQueue(stream, queue)
1344 FSEventStreamStart(stream)
1345 self.stream = stream
1346 }
1347
1348 public func stop() {
1349 guard let stream else { return }
1350 FSEventStreamStop(stream)
1351 FSEventStreamInvalidate(stream)
1352 FSEventStreamRelease(stream)
1353 self.stream = nil
1354 }
1355
1356 private func deliver(paths eventPaths: [String], flags: [FSEventStreamEventFlags]) {
1357 let rescanFlags = FSEventStreamEventFlags(
1358 kFSEventStreamEventFlagMustScanSubDirs | kFSEventStreamEventFlagUserDropped
1359 | kFSEventStreamEventFlagKernelDropped | kFSEventStreamEventFlagRootChanged
1360 )
1361 var events: [WatchEvent] = []
1362 for (path, flag) in zip(eventPaths, flags) {
1363 let url = URL(fileURLWithPath: path).standardizedFileURL
1364 if flag & rescanFlags != 0 {
1365 let root = paths.first { path.hasPrefix($0) } ?? path
1366 events.append(.rescan(URL(fileURLWithPath: root).standardizedFileURL))
1367 } else {
1368 events.append(.changed(url))
1369 }
1370 }
1371 if !events.isEmpty { handler(events) }
1372 }
1373}
1374#endif
1375```
1376
1377- [ ] **Step 3: Run everything, the iOS build and the gates**
1378
1379Run: `swift test`, then `xcodebuild -scheme OrgWorkspace -destination 'generic/platform=iOS' build`.
1380Gates: reconcile a folder of 10,000 org files into an empty index (under 30 s), then again with no changes (under 2 s), in a release build on the reference machine.
1381
1382- [ ] **Step 4: Commit**
1383
1384```bash
1385git add Sources/OrgWorkspace/FSEventsWatcher.swift Tests/OrgWorkspaceTests/WatcherTests.swift
1386git commit -m "Add FSEvents watcher"
1387```