krz/domain-dig
an ios app for DNS & SSL analysis
clone: git clone https://gitbay.org/krz/domain-dig.git
v4.5.0: DomainDig/ExternalDataService.swift · raw
1import Foundation
2
3actor ExternalDataService {
4 static let shared = ExternalDataService()
5
6 private enum RequestKey: Hashable {
7 case ownershipHistory(String)
8 case dnsHistory(String)
9 case extendedSubdomains(String)
10 case pricing(String)
11 }
12
13 private enum RateLimitBucket: Hashable {
14 case history
15 case subdomains
16 case pricing
17
18 var minimumSpacing: TimeInterval {
19 switch self {
20 case .history:
21 return 1.0
22 case .subdomains:
23 return 1.5
24 case .pricing:
25 return 1.0
26 }
27 }
28 }
29
30 private enum CachedPayload {
31 case ownershipHistory(ServiceResult<[DomainOwnershipHistoryEvent]>)
32 case dnsHistory(ServiceResult<[DNSHistoryEvent]>)
33 case extendedSubdomains(ServiceResult<[DiscoveredSubdomain]>)
34 case pricing(ServiceResult<DomainPricingInsight>)
35 }
36
37 private struct CacheEntry {
38 let payload: CachedPayload
39 let expiresAt: Date
40 }
41
42 private struct Configuration {
43 let ownershipHistoryURL: String?
44 let dnsHistoryURL: String?
45 let extendedSubdomainsURL: String?
46 let pricingURL: String?
47 }
48
49 private let ttl: TimeInterval = 900
50 private let session: URLSession = .shared
51 private var cache: [RequestKey: CacheEntry] = [:]
52 private var inFlight: [RequestKey: Task<CachedPayload, Never>] = [:]
53 private var nextAllowedAt: [RateLimitBucket: Date] = [:]
54
55 func clearCache() {
56 cache.removeAll()
57 inFlight.values.forEach { $0.cancel() }
58 inFlight.removeAll()
59 nextAllowedAt.removeAll()
60 }
61
62 func ownershipHistory(
63 domain: String,
64 currentOwnership: DomainOwnership?,
65 historyEntries: [HistoryEntry]
66 ) async -> CachedLookupResult<ServiceResult<[DomainOwnershipHistoryEvent]>> {
67 let normalizedDomain = Self.normalize(domain)
68 return await execute(
69 key: .ownershipHistory(normalizedDomain),
70 rateLimitBucket: .history,
71 extract: { payload in
72 guard case let .ownershipHistory(result) = payload else { return nil }
73 return result
74 },
75 operation: { [configuration = configuration()] in
76 let localEvents = Self.localOwnershipHistory(
77 domain: normalizedDomain,
78 currentOwnership: currentOwnership,
79 historyEntries: historyEntries
80 )
81 let externalEvents = await self.fetchOwnershipHistory(domain: normalizedDomain, configuration: configuration)
82 return .ownershipHistory(Self.mergeOwnershipHistory(localEvents: localEvents, externalEvents: externalEvents))
83 }
84 )
85 }
86
87 func dnsHistory(
88 domain: String,
89 dnsSections: [DNSSection],
90 historyEntries: [HistoryEntry]
91 ) async -> CachedLookupResult<ServiceResult<[DNSHistoryEvent]>> {
92 let normalizedDomain = Self.normalize(domain)
93 return await execute(
94 key: .dnsHistory(normalizedDomain),
95 rateLimitBucket: .history,
96 extract: { payload in
97 guard case let .dnsHistory(result) = payload else { return nil }
98 return result
99 },
100 operation: { [configuration = configuration()] in
101 let localEvents = Self.localDNSHistory(
102 domain: normalizedDomain,
103 dnsSections: dnsSections,
104 historyEntries: historyEntries
105 )
106 let externalEvents = await self.fetchDNSHistory(domain: normalizedDomain, configuration: configuration)
107 return .dnsHistory(Self.mergeDNSHistory(localEvents: localEvents, externalEvents: externalEvents))
108 }
109 )
110 }
111
112 func extendedSubdomains(
113 domain: String,
114 existing: [DiscoveredSubdomain]
115 ) async -> CachedLookupResult<ServiceResult<[DiscoveredSubdomain]>> {
116 let normalizedDomain = Self.normalize(domain)
117 return await execute(
118 key: .extendedSubdomains(normalizedDomain),
119 rateLimitBucket: .subdomains,
120 extract: { payload in
121 guard case let .extendedSubdomains(result) = payload else { return nil }
122 return result
123 },
124 operation: { [configuration = configuration()] in
125 var merged = existing
126 switch await SubdomainDiscoveryService.discover(for: normalizedDomain, limit: 100) {
127 case let .success(results):
128 merged = Self.mergeSubdomains(primary: merged, additional: results.map {
129 DiscoveredSubdomain(hostname: $0.hostname, source: $0.source ?? "crt.sh", isExtended: true)
130 })
131 case .empty, .error:
132 break
133 }
134
135 if let external = await self.fetchExtendedSubdomains(domain: normalizedDomain, configuration: configuration) {
136 merged = Self.mergeSubdomains(primary: merged, additional: external)
137 }
138
139 if merged.count <= existing.count {
140 return .extendedSubdomains(.empty("No extended subdomains available"))
141 }
142
143 let onlyExtended = merged.filter(\.isExtended)
144 return .extendedSubdomains(.success(onlyExtended.sorted { $0.hostname < $1.hostname }))
145 }
146 )
147 }
148
149 func pricing(domain: String) async -> CachedLookupResult<ServiceResult<DomainPricingInsight>> {
150 let normalizedDomain = Self.normalize(domain)
151 return await execute(
152 key: .pricing(normalizedDomain),
153 rateLimitBucket: .pricing,
154 extract: { payload in
155 guard case let .pricing(result) = payload else { return nil }
156 return result
157 },
158 operation: { [configuration = configuration()] in
159 .pricing(await self.fetchPricing(domain: normalizedDomain, configuration: configuration))
160 }
161 )
162 }
163
164 private func execute<T>(
165 key: RequestKey,
166 rateLimitBucket: RateLimitBucket,
167 extract: @escaping (CachedPayload) -> T?,
168 operation: @escaping @Sendable () async -> CachedPayload
169 ) async -> CachedLookupResult<T> {
170 if let cachedEntry = cache[key], cachedEntry.expiresAt > Date(), let value = extract(cachedEntry.payload) {
171 return CachedLookupResult(value: value, source: .cached)
172 }
173
174 if let task = inFlight[key], let value = extract(await task.value) {
175 return CachedLookupResult(value: value, source: .mixed)
176 }
177
178 let task = Task<CachedPayload, Never> {
179 await self.enforceRateLimit(for: rateLimitBucket)
180 return await operation()
181 }
182 inFlight[key] = task
183
184 let payload = await task.value
185 cache[key] = CacheEntry(payload: payload, expiresAt: Date().addingTimeInterval(ttl))
186 inFlight[key] = nil
187
188 guard let value = extract(payload) else {
189 fatalError("ExternalDataService payload extraction mismatch")
190 }
191
192 return CachedLookupResult(value: value, source: .live)
193 }
194
195 private func enforceRateLimit(for bucket: RateLimitBucket) async {
196 let now = Date()
197 if let nextAllowed = nextAllowedAt[bucket], nextAllowed > now {
198 let delay = nextAllowed.timeIntervalSince(now)
199 if delay > 0 {
200 try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
201 }
202 }
203 nextAllowedAt[bucket] = Date().addingTimeInterval(bucket.minimumSpacing)
204 }
205
206 private func configuration() -> Configuration {
207 let defaults = UserDefaults.standard
208 return Configuration(
209 ownershipHistoryURL: defaults.string(forKey: "externalData.ownershipHistoryURL")
210 ?? Bundle.main.object(forInfoDictionaryKey: "ExternalOwnershipHistoryURL") as? String,
211 dnsHistoryURL: defaults.string(forKey: "externalData.dnsHistoryURL")
212 ?? Bundle.main.object(forInfoDictionaryKey: "ExternalDNSHistoryURL") as? String,
213 extendedSubdomainsURL: defaults.string(forKey: "externalData.extendedSubdomainsURL")
214 ?? Bundle.main.object(forInfoDictionaryKey: "ExternalExtendedSubdomainsURL") as? String,
215 pricingURL: defaults.string(forKey: "externalData.pricingURL")
216 ?? Bundle.main.object(forInfoDictionaryKey: "ExternalPricingURL") as? String
217 )
218 }
219
220 private func fetchOwnershipHistory(
221 domain: String,
222 configuration: Configuration
223 ) async -> [DomainOwnershipHistoryEvent] {
224 guard let template = configuration.ownershipHistoryURL,
225 let url = Self.url(from: template, domain: domain) else {
226 return []
227 }
228
229 switch await requestData(url: url) {
230 case let .success(data):
231 return parseOwnershipHistoryEvents(from: data)
232 case .empty, .error:
233 return []
234 }
235 }
236
237 private func fetchDNSHistory(
238 domain: String,
239 configuration: Configuration
240 ) async -> [DNSHistoryEvent] {
241 guard let template = configuration.dnsHistoryURL,
242 let url = Self.url(from: template, domain: domain) else {
243 return []
244 }
245
246 switch await requestData(url: url) {
247 case let .success(data):
248 return parseDNSHistoryEvents(from: data)
249 case .empty, .error:
250 return []
251 }
252 }
253
254 private func fetchExtendedSubdomains(
255 domain: String,
256 configuration: Configuration
257 ) async -> [DiscoveredSubdomain]? {
258 guard let template = configuration.extendedSubdomainsURL,
259 let url = Self.url(from: template, domain: domain) else {
260 return nil
261 }
262
263 switch await requestData(url: url) {
264 case let .success(data):
265 return parseSubdomains(from: data)
266 case .empty, .error:
267 return nil
268 }
269 }
270
271 private func fetchPricing(
272 domain: String,
273 configuration: Configuration
274 ) async -> ServiceResult<DomainPricingInsight> {
275 guard let template = configuration.pricingURL,
276 let url = Self.url(from: template, domain: domain) else {
277 return .empty("External pricing unavailable")
278 }
279
280 switch await requestData(url: url) {
281 case let .success(data):
282 guard let pricing = parsePricing(from: data) else {
283 return .error("Invalid external response")
284 }
285 return .success(pricing)
286 case let .empty(message):
287 return .empty(message)
288 case let .error(message):
289 return .error(message)
290 }
291 }
292
293 private func requestData(url: URL) async -> ServiceResult<Data> {
294 for attempt in 0..<2 {
295 do {
296 let (data, response) = try await session.data(for: URLRequest(url: url, timeoutInterval: 8))
297 guard let httpResponse = response as? HTTPURLResponse else {
298 return .error("External data unavailable")
299 }
300
301 switch httpResponse.statusCode {
302 case 200:
303 return .success(data)
304 case 204, 404:
305 return .empty("No external data available")
306 case 429:
307 return .error("Rate limit reached")
308 case 500...599 where attempt == 0:
309 continue
310 default:
311 return .error("External provider failed")
312 }
313 } catch is DecodingError {
314 return .error("Invalid external response")
315 } catch {
316 if attempt == 0 {
317 continue
318 }
319 return .error(error.localizedDescription)
320 }
321 }
322
323 return .error("External provider failed")
324 }
325
326 private func parseOwnershipHistoryEvents(from data: Data) -> [DomainOwnershipHistoryEvent] {
327 guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
328 let rawEvents = json["events"] as? [[String: Any]] else {
329 return []
330 }
331
332 return rawEvents.compactMap { event in
333 guard let dateString = event["date"] as? String,
334 let date = Self.iso8601DateFormatter.date(from: dateString) else {
335 return nil
336 }
337
338 return DomainOwnershipHistoryEvent(
339 date: date,
340 summary: event["summary"] as? String ?? "Ownership change observed",
341 registrar: event["registrar"] as? String,
342 registrant: event["registrant"] as? String,
343 nameservers: event["nameservers"] as? [String] ?? [],
344 source: event["source"] as? String ?? "Configured external history feed",
345 isExternal: true
346 )
347 }
348 }
349
350 private func parseDNSHistoryEvents(from data: Data) -> [DNSHistoryEvent] {
351 guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
352 let rawEvents = json["events"] as? [[String: Any]] else {
353 return []
354 }
355
356 return rawEvents.compactMap { event in
357 guard let dateString = event["date"] as? String,
358 let date = Self.iso8601DateFormatter.date(from: dateString) else {
359 return nil
360 }
361
362 return DNSHistoryEvent(
363 date: date,
364 summary: event["summary"] as? String ?? "DNS change observed",
365 aRecords: event["a_records"] as? [String] ?? [],
366 nameservers: event["nameservers"] as? [String] ?? [],
367 recordSnapshots: parseDNSRecordSnapshots(from: event),
368 changedRecordTypes: parseDNSChangedRecordTypes(from: event),
369 source: event["source"] as? String ?? "Configured external history feed",
370 isExternal: true
371 )
372 }
373 }
374
375 private func parseSubdomains(from data: Data) -> [DiscoveredSubdomain] {
376 guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
377 let rawSubdomains = json["subdomains"] as? [[String: Any]] else {
378 return []
379 }
380
381 return rawSubdomains.compactMap { item in
382 guard let hostname = item["hostname"] as? String else { return nil }
383 return DiscoveredSubdomain(
384 hostname: hostname,
385 source: item["source"] as? String ?? "Configured external subdomain feed",
386 isExtended: true
387 )
388 }
389 }
390
391 private func parsePricing(from data: Data) -> DomainPricingInsight? {
392 guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
393 return nil
394 }
395
396 return DomainPricingInsight(
397 estimatedPrice: json["estimated_price"] as? String,
398 premiumIndicator: json["premium"] as? Bool,
399 resaleSignal: json["resale_signal"] as? String,
400 auctionSignal: json["auction_signal"] as? String,
401 source: json["source"] as? String ?? "Configured external pricing feed",
402 collectedAt: Date()
403 )
404 }
405
406 private static func localOwnershipHistory(
407 domain: String,
408 currentOwnership: DomainOwnership?,
409 historyEntries: [HistoryEntry]
410 ) -> [DomainOwnershipHistoryEvent] {
411 let domainHistory = historyEntries
412 .filter { $0.domain.caseInsensitiveCompare(domain) == .orderedSame }
413 .sorted { $0.timestamp < $1.timestamp }
414
415 var events: [DomainOwnershipHistoryEvent] = []
416 var previousOwnership: DomainOwnership?
417
418 for entry in domainHistory {
419 guard let ownership = entry.ownership else { continue }
420 let summary = ownershipSummaryChange(previous: previousOwnership, current: ownership)
421 if let summary {
422 events.append(
423 DomainOwnershipHistoryEvent(
424 date: entry.timestamp,
425 summary: summary,
426 registrar: ownership.registrar,
427 registrant: ownership.registrant,
428 nameservers: ownership.nameservers,
429 source: "Local observations",
430 isExternal: false
431 )
432 )
433 }
434 previousOwnership = ownership
435 }
436
437 if let currentOwnership, events.isEmpty {
438 events.append(
439 DomainOwnershipHistoryEvent(
440 date: Date(),
441 summary: "Current ownership snapshot",
442 registrar: currentOwnership.registrar,
443 registrant: currentOwnership.registrant,
444 nameservers: currentOwnership.nameservers,
445 source: "Local observations",
446 isExternal: false
447 )
448 )
449 }
450
451 return events.sorted { $0.date > $1.date }
452 }
453
454 private static func localDNSHistory(
455 domain: String,
456 dnsSections: [DNSSection],
457 historyEntries: [HistoryEntry]
458 ) -> [DNSHistoryEvent] {
459 let domainHistory = historyEntries
460 .filter { $0.domain.caseInsensitiveCompare(domain) == .orderedSame }
461 .sorted { $0.timestamp < $1.timestamp }
462
463 var events: [DNSHistoryEvent] = []
464 var previousRecordValues: [DNSRecordType: [String]] = [:]
465
466 for entry in domainHistory {
467 let currentRecordValues = Self.historyRecordValues(in: entry.dnsSections)
468 let changedRecordTypes = Self.changedRecordTypes(previous: previousRecordValues, current: currentRecordValues)
469 let summary = dnsSummaryChange(previous: previousRecordValues, current: currentRecordValues)
470
471 if let summary {
472 events.append(
473 DNSHistoryEvent(
474 date: entry.timestamp,
475 summary: summary,
476 aRecords: currentRecordValues[.A] ?? [],
477 nameservers: currentRecordValues[.NS] ?? [],
478 recordSnapshots: currentRecordValues.map { DNSHistoryRecordSnapshot(recordType: $0.key, values: $0.value) }
479 .sorted { $0.recordType.rawValue < $1.recordType.rawValue },
480 changedRecordTypes: changedRecordTypes,
481 source: "Local observations",
482 isExternal: false
483 )
484 )
485 }
486
487 previousRecordValues = currentRecordValues
488 }
489
490 if events.isEmpty {
491 let currentRecordValues = Self.historyRecordValues(in: dnsSections)
492 if !currentRecordValues.isEmpty {
493 events.append(
494 DNSHistoryEvent(
495 date: Date(),
496 summary: "Current DNS snapshot",
497 aRecords: currentRecordValues[.A] ?? [],
498 nameservers: currentRecordValues[.NS] ?? [],
499 recordSnapshots: currentRecordValues.map { DNSHistoryRecordSnapshot(recordType: $0.key, values: $0.value) }
500 .sorted { $0.recordType.rawValue < $1.recordType.rawValue },
501 changedRecordTypes: Array(currentRecordValues.keys).sorted { $0.rawValue < $1.rawValue },
502 source: "Local observations",
503 isExternal: false
504 )
505 )
506 }
507 }
508
509 return events.sorted { $0.date > $1.date }
510 }
511
512 private static func mergeOwnershipHistory(
513 localEvents: [DomainOwnershipHistoryEvent],
514 externalEvents: [DomainOwnershipHistoryEvent]
515 ) -> ServiceResult<[DomainOwnershipHistoryEvent]> {
516 let merged = (externalEvents + localEvents)
517 .sorted { $0.date > $1.date }
518 .reduce(into: [DomainOwnershipHistoryEvent]()) { partialResult, event in
519 let duplicate = partialResult.contains {
520 $0.date == event.date
521 && $0.summary == event.summary
522 && $0.registrar == event.registrar
523 && $0.nameservers == event.nameservers
524 }
525 if !duplicate {
526 partialResult.append(event)
527 }
528 }
529
530 return merged.isEmpty ? .empty("No ownership history available") : .success(merged)
531 }
532
533 private static func mergeDNSHistory(
534 localEvents: [DNSHistoryEvent],
535 externalEvents: [DNSHistoryEvent]
536 ) -> ServiceResult<[DNSHistoryEvent]> {
537 let merged = (externalEvents + localEvents)
538 .sorted { $0.date > $1.date }
539 .reduce(into: [DNSHistoryEvent]()) { partialResult, event in
540 let duplicate = partialResult.contains {
541 $0.date == event.date
542 && $0.summary == event.summary
543 && compareDNSRecordSnapshots($0.recordSnapshots, event.recordSnapshots)
544 }
545 if !duplicate {
546 partialResult.append(event)
547 }
548 }
549
550 return merged.isEmpty ? .empty("No DNS history available") : .success(merged)
551 }
552
553 private static func mergeSubdomains(
554 primary: [DiscoveredSubdomain],
555 additional: [DiscoveredSubdomain]
556 ) -> [DiscoveredSubdomain] {
557 var seen = Set(primary.map { $0.hostname.lowercased() })
558 var merged = primary
559
560 for subdomain in additional where seen.insert(subdomain.hostname.lowercased()).inserted {
561 merged.append(subdomain)
562 }
563
564 return merged.sorted { $0.hostname < $1.hostname }
565 }
566
567 private static func ownershipSummaryChange(previous: DomainOwnership?, current: DomainOwnership) -> String? {
568 guard let previous else {
569 return "Initial ownership observation"
570 }
571
572 var changes: [String] = []
573 if previous.registrar != current.registrar {
574 changes.append("Registrar changed")
575 }
576 if previous.registrant != current.registrant, current.registrant != nil {
577 changes.append("Ownership changed")
578 }
579 if previous.nameservers != current.nameservers {
580 changes.append("Nameservers changed")
581 }
582
583 return changes.isEmpty ? nil : changes.joined(separator: " • ")
584 }
585
586 private static func dnsSummaryChange(
587 previous: [DNSRecordType: [String]],
588 current: [DNSRecordType: [String]]
589 ) -> String? {
590 var changes: [String] = []
591 for type in [DNSRecordType.A, .AAAA, .MX, .NS, .TXT, .CNAME] {
592 let previousValues = previous[type] ?? []
593 let currentValues = current[type] ?? []
594 if previousValues != currentValues, !currentValues.isEmpty {
595 changes.append("\(type.rawValue) records changed")
596 }
597 }
598 if previous.isEmpty && !current.isEmpty {
599 changes.append("Initial DNS observation")
600 }
601 return changes.isEmpty ? nil : changes.joined(separator: " • ")
602 }
603
604 private static func url(from template: String, domain: String) -> URL? {
605 URL(string: template.replacingOccurrences(of: "{domain}", with: domain))
606 }
607
608 private static func normalize(_ domain: String) -> String {
609 domain.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
610 }
611
612 private static func dnsValues(for type: DNSRecordType, in sections: [DNSSection]) -> [String] {
613 sections
614 .first(where: { $0.recordType == type })?
615 .records
616 .map(\.value)
617 .sorted() ?? []
618 }
619
620 private func parseDNSRecordSnapshots(from event: [String: Any]) -> [DNSHistoryRecordSnapshot] {
621 if let snapshots = event["record_snapshots"] as? [[String: Any]] {
622 return snapshots.compactMap { item in
623 guard let typeName = item["type"] as? String,
624 let type = DNSRecordType(rawValue: typeName) else {
625 return nil
626 }
627 return DNSHistoryRecordSnapshot(recordType: type, values: item["values"] as? [String] ?? [])
628 }
629 }
630 var snapshots: [DNSHistoryRecordSnapshot] = []
631 if let aRecords = event["a_records"] as? [String], !aRecords.isEmpty {
632 snapshots.append(DNSHistoryRecordSnapshot(recordType: .A, values: aRecords))
633 }
634 if let nameservers = event["nameservers"] as? [String], !nameservers.isEmpty {
635 snapshots.append(DNSHistoryRecordSnapshot(recordType: .NS, values: nameservers))
636 }
637 return snapshots
638 }
639
640 private func parseDNSChangedRecordTypes(from event: [String: Any]) -> [DNSRecordType] {
641 if let rawTypes = event["changed_record_types"] as? [String] {
642 return rawTypes.compactMap(DNSRecordType.init(rawValue:))
643 }
644 return parseDNSRecordSnapshots(from: event).map(\.recordType)
645 }
646
647 private static func historyRecordValues(in sections: [DNSSection]) -> [DNSRecordType: [String]] {
648 let trackedTypes: [DNSRecordType] = [.A, .AAAA, .MX, .NS, .TXT, .CNAME]
649 return trackedTypes.reduce(into: [DNSRecordType: [String]]()) { result, type in
650 let values = dnsValues(for: type, in: sections)
651 if !values.isEmpty {
652 result[type] = values
653 }
654 }
655 }
656
657 private static func changedRecordTypes(
658 previous: [DNSRecordType: [String]],
659 current: [DNSRecordType: [String]]
660 ) -> [DNSRecordType] {
661 Array(Set(previous.keys).union(current.keys))
662 .filter { previous[$0] != current[$0] }
663 .sorted { $0.rawValue < $1.rawValue }
664 }
665
666 private static func compareDNSRecordSnapshots(
667 _ lhs: [DNSHistoryRecordSnapshot],
668 _ rhs: [DNSHistoryRecordSnapshot]
669 ) -> Bool {
670 guard lhs.count == rhs.count else { return false }
671 return zip(lhs, rhs).allSatisfy { left, right in
672 left.recordType == right.recordType && left.values == right.values
673 }
674 }
675
676 private static let iso8601DateFormatter: ISO8601DateFormatter = {
677 let formatter = ISO8601DateFormatter()
678 formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
679 return formatter
680 }()
681}