app/parsers.go
490 lines · 16162 bytes · executable
1package app
2
3import (
4 "encoding/json"
5 "net/url"
6 "path"
7 "strconv"
8 "strings"
9 "time"
10
11 "github.com/krazywarez/devianter"
12 "golang.org/x/net/html"
13)
14
15// ParseComments renders a comment thread, nesting replies under the comment they
16// answer. It returns a placeholder message rather than failing if the upstream
17// fetch errored.
18func (s skunkyart) ParseComments(c devianter.Comments, daError devianter.Error) string {
19 if daError.RAW != nil {
20 return esc(T(s.Lang, "comments.failed"))
21 }
22
23 var cmmts strings.Builder
24 replied := make(map[int]string)
25
26 cmmts.WriteString("<details><summary>")
27 cmmts.WriteString(esc(T(s.Lang, "deviation.comments")))
28 cmmts.WriteString(": <b>")
29 cmmts.WriteString(strconv.Itoa(c.Total))
30 cmmts.WriteString("</b></summary>")
31 for _, x := range c.Thread {
32 replied[x.ID] = x.User.Username
33 cmmts.WriteString(`<div class="msg`)
34 if x.Parent > 0 {
35 cmmts.WriteString(` reply`)
36 }
37 cmmts.WriteString(`"><p id="`)
38 cmmts.WriteString(strconv.Itoa(x.ID))
39 cmmts.WriteString(`"><img src="`)
40 cmmts.WriteString(esc(URLBuilder(s.Host, "media", "emojitar", x.User.Username, "?type=a")))
41 cmmts.WriteString(`" width="30px" height="30px" alt="`)
42 cmmts.WriteString(esc(x.User.Username))
43 cmmts.WriteString(`"><a href="`)
44 cmmts.WriteString(esc(URLBuilder(s.Host, "group_user", "?q=", x.User.Username, "&type=a")))
45 cmmts.WriteString(`"><b`)
46 cmmts.WriteString(` class="`)
47 if x.User.Banned {
48 cmmts.WriteString(`banned`)
49 }
50 if x.Author {
51 cmmts.WriteString(`author`)
52 }
53 cmmts.WriteString(`">`)
54 cmmts.WriteString(esc(x.User.Username))
55 cmmts.WriteString("</b></a> ")
56
57 if x.Parent > 0 {
58 cmmts.WriteString(" ")
59 cmmts.WriteString(esc(T(s.Lang, "comments.reply_to")))
60 cmmts.WriteString(` <a href="`)
61 cmmts.WriteString(esc(s._pth))
62 cmmts.WriteString("#")
63 cmmts.WriteString(strconv.Itoa(x.Parent))
64 cmmts.WriteString(`">`)
65 if replied[x.Parent] == "" {
66 cmmts.WriteString("???")
67 } else {
68 cmmts.WriteString(esc(replied[x.Parent]))
69 }
70 cmmts.WriteString("</a>")
71 }
72 cmmts.WriteString(" [")
73 cmmts.WriteString(x.Posted.UTC().String())
74 cmmts.WriteString("]<p>")
75
76 cmmts.WriteString(ParseDescription(s.Host, x.TextContent))
77 cmmts.WriteString("<p>👍: ")
78 cmmts.WriteString(strconv.Itoa(x.Likes))
79 cmmts.WriteString(" ⏩: ")
80 cmmts.WriteString(strconv.Itoa(x.Replies))
81 cmmts.WriteString("</p></div>\n")
82 }
83 cmmts.WriteString(s.NavBase(DeviationList{
84 Pages: 0,
85 More: c.HasMore,
86 }))
87 cmmts.WriteString("</details>")
88 return cmmts.String()
89}
90
91// VisibleDeviation reports whether a deviation may be shown by this instance.
92//
93// Both the HTML listing and the JSON API ask this, deliberately: an API that
94// returned what the pages hide would leak exactly what hide-ai and the NSFW
95// setting exist to withhold, and a second copy of the rule is a second thing to
96// forget to update.
97func VisibleDeviation(d *devianter.Deviation) bool {
98 if d.AI && CFG.HideAI {
99 return false
100 }
101 return !d.NSFW || CFG.Nsfw
102}
103
104// DeviationList renders devs as an HTML grid, or as an Atom feed when the
105// request asked for one and allowAtom permits it. NSFW entries are dropped
106// unless the instance allows them. Passing content adds a navigation bar.
107func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, content ...DeviationList) string {
108 if s.Atom && s.Page > 1 {
109 s.ReturnHTTPError(400)
110 return ""
111 }
112
113 var list, listContent strings.Builder
114 var newest time.Time // the latest entry, for the feed's own <updated>
115
116 for i, l := 0, len(devs); i < l; i++ {
117 data := &devs[i]
118 if !VisibleDeviation(data) {
119 continue
120 }
121 // Escaped once here: the same values go into both the HTML grid and the
122 // Atom feed, and html.EscapeString produces entities XML accepts too.
123 author, title := esc(data.Author.Username), esc(data.Title)
124 postURL := esc(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url))
125 if preview, fullview := esc(ParseMedia(s.Host, data.Media, 320)), esc(ParseMedia(s.Host, data.Media)); true {
126 if allowAtom && s.Atom {
127 s.Writer.Header().Add("Content-Type", "application/atom+xml")
128 // The entry id is the post's URL here: Atom wants an IRI, and a
129 // bare number is not one. Timestamps are RFC 3339, the only form
130 // the format allows.
131 entryURL := esc(URLBuilder(s.Host, "post", data.Author.Username, "atom-"+strconv.Itoa(data.ID)))
132 published := data.PublishedTime.UTC()
133 if published.After(newest) {
134 newest = published
135 }
136 listContent.WriteString(`<entry><author><name>`)
137 listContent.WriteString(author)
138 listContent.WriteString(`</name></author><title>`)
139 listContent.WriteString(title)
140 listContent.WriteString(`</title><link rel="alternate" type="text/html" href="`)
141 listContent.WriteString(entryURL)
142 listContent.WriteString(`"/><id>`)
143 listContent.WriteString(entryURL)
144 listContent.WriteString(`</id><published>`)
145 listContent.WriteString(published.Format(time.RFC3339))
146 listContent.WriteString(`</published><updated>`)
147 listContent.WriteString(published.Format(time.RFC3339))
148 listContent.WriteString(`</updated>`)
149 listContent.WriteString(`<media:group><media:title>`)
150 listContent.WriteString(title)
151 listContent.WriteString(`</media:title><media:thumbnail url="`)
152 listContent.WriteString(preview)
153 listContent.WriteString(`"/></media:group><content type="xhtml"><div xmlns="http://www.w3.org/1999/xhtml"><a href="`)
154 listContent.WriteString(postURL)
155 listContent.WriteString(`"><img src="`)
156 listContent.WriteString(fullview)
157 listContent.WriteString(`"/></a><p>`)
158 listContent.WriteString(ParseDescription(s.Host, data.TextContent))
159 listContent.WriteString(`</p></div></content></entry>`)
160 } else {
161 listContent.WriteString(`<div class="block">`)
162 if fullview != "" && preview != "" {
163 listContent.WriteString(`<a title="` + esc(T(s.Lang, "deviation.open")) + `" href="`)
164 listContent.WriteString(fullview)
165 listContent.WriteString(`"><img loading="lazy" src="`)
166 listContent.WriteString(preview)
167 listContent.WriteString(`" width="15%" alt="`)
168 listContent.WriteString(author)
169 listContent.WriteString(" - ")
170 listContent.WriteString(title)
171 listContent.WriteString(`"></a>`)
172 } else {
173 listContent.WriteString("<h1>[ " + esc(T(s.Lang, "list.text")) + " ]</h1>")
174 }
175 listContent.WriteString(`<br><a href="`)
176 listContent.WriteString(postURL)
177 listContent.WriteString(`">`)
178 listContent.WriteString(author)
179 listContent.WriteString(" - ")
180 listContent.WriteString(title)
181
182 if data.NSFW {
183 listContent.WriteString(` [<span class="nsfw">NSFW</span>]`)
184 }
185 if data.AI {
186 listContent.WriteString(" [🤖]")
187 }
188 if data.DD {
189 listContent.WriteString(` [<span class="dd">DD</span>]`)
190 }
191
192 listContent.WriteString("</a></div>")
193 }
194 }
195 }
196
197 if allowAtom && s.Atom {
198 list.WriteString(`<?xml version="1.0" encoding="UTF-8"?><feed xmlns:media="http://search.yahoo.com/mrss/" xmlns="http://www.w3.org/2005/Atom">`)
199
200 list.WriteString(`<title>`)
201 switch {
202 case s.Type == 0:
203 list.WriteString("Daily Deviations")
204 case s.Type == 'g' && len(devs) != 0:
205 list.WriteString(esc(devs[0].Author.Username))
206 default:
207 list.WriteString("SkunkyArt")
208 }
209 list.WriteString(`</title>`)
210
211 // Atom requires a feed id and updated time; readers reject a feed
212 // without them. The id is the feed's own URL on this instance.
213 feedURL := esc(s.Host + s._pth + "?" + s.Args.Encode())
214 if newest.IsZero() {
215 newest = time.Now().UTC()
216 }
217 list.WriteString(`<id>`)
218 list.WriteString(feedURL)
219 list.WriteString(`</id><updated>`)
220 list.WriteString(newest.Format(time.RFC3339))
221 list.WriteString(`</updated><link rel="self" type="application/atom+xml" href="`)
222 list.WriteString(feedURL)
223 list.WriteString(`"/>`)
224
225 list.WriteString(`<link rel="alternate" href="`)
226 list.WriteString(esc(s.Host))
227 list.WriteString(`"/>`)
228
229 list.WriteString(listContent.String())
230
231 list.WriteString("</feed>")
232 wr(s.Writer, list.String())
233 } else {
234 list.WriteString(`<div class="content">`)
235
236 list.WriteString(listContent.String())
237
238 list.WriteString("</div>")
239 if content != nil {
240 list.WriteString(s.NavBase(content[0]))
241 }
242 }
243
244 return list.String()
245}
246
247/* DESCRIPTION/COMMENT PARSER */
248
249// emoticonURL maps an e.deviantart.net emoticon image URL to this instance's
250// emote route, by the file's base name without its extension, which is what
251// devianter.AEmedia takes. Anything else yields "".
252func emoticonURL(host, raw string) string {
253 u, err := url.Parse(raw)
254 if err != nil || u.Host != "e.deviantart.net" {
255 return ""
256 }
257 name := path.Base(u.Path)
258 name = strings.TrimSuffix(name, path.Ext(name))
259 if name == "" || name == "." || name == "/" {
260 return ""
261 }
262 return URLBuilder(host, "media", "emojitar", name, "?type=e")
263}
264
265// text is one styled run within a description: the rendered HTML, the raw source
266// it came from, and the offsets it spans in the original block.
267type text struct {
268 Txt string
269 TxtRaw string
270 From int
271 To int
272}
273
274// ParseDescription renders a DeviantArt description into HTML, handling both the
275// Draft.js-style JSON payload and the plain HTML markup DeviantArt returns, and
276// rewriting embedded links and artwork references to point at this instance.
277// host is the request's scheme and host, as taken by URLBuilder.
278//
279// Three formats arrive: the current editor's document JSON (a "document"
280// key), handled by renderDocument; the older Draft.js JSON ("blocks" and
281// "entityMap"), handled below and kept for the posts that still carry it; and
282// plain HTML markup from before either.
283func ParseDescription(host string, dscr devianter.Text) string {
284 if doc, ok := parseDocument(dscr.Html.Markup); ok {
285 return renderDocument(host, doc)
286 }
287
288 var parsedDescription strings.Builder
289 TagBuilder := func(content string, tags ...string) string {
290 l := len(tags)
291 for x := range l {
292 var htm strings.Builder
293 htm.WriteString("<")
294 htm.WriteString(tags[x])
295 htm.WriteString(">")
296
297 htm.WriteString(content)
298
299 htm.WriteString("</")
300 htm.WriteString(tags[x])
301 htm.WriteString(">")
302 content = htm.String()
303 }
304 return content
305 }
306 DeleteTrackingFromURL := deleteTrackingFromURL
307
308 if description, dl := dscr.Html.Markup, len(dscr.Html.Markup); dl != 0 &&
309 description[0] == '{' &&
310 description[dl-1] == '}' {
311 var descr struct {
312 Blocks []struct {
313 Text string `json:"text"`
314 Type string `json:"type"`
315 InlineStyleRanges []struct {
316 Offset int `json:"offset"`
317 Length int `json:"length"`
318 Style string `json:"style"`
319 } `json:"inlineStyleRanges"`
320 EntityRanges []struct {
321 Offset int `json:"offset"`
322 Length int `json:"length"`
323 Key int `json:"key"`
324 } `json:"entityRanges"`
325 Data struct {
326 TextAlignment string `json:"textAlignment"`
327 } `json:"data"`
328 } `json:"blocks"`
329 EntityMap map[string]struct {
330 Type string `json:"type"`
331 Data struct {
332 URL string `json:"url"`
333 Config struct {
334 // "aligment" is DeviantArt's own spelling; do not correct it.
335 Aligment string `json:"aligment"`
336 Width int `json:"width"`
337 } `json:"config"`
338 Data devianter.Deviation `json:"data"`
339 } `json:"data"`
340 } `json:"entityMap"`
341 }
342 e := json.Unmarshal([]byte(description), &descr)
343 try(e)
344
345 entities := make(map[int]devianter.Deviation)
346 urls := make(map[int]string)
347 for n, x := range descr.EntityMap {
348 num, _ := strconv.Atoi(n)
349 if x.Data.URL != "" {
350 urls[num] = DeleteTrackingFromURL(x.Data.URL)
351 }
352 entities[num] = x.Data.Data
353 }
354
355 for _, x := range descr.Blocks {
356 Styles := make([]text, len(x.InlineStyleRanges))
357
358 if len(x.InlineStyleRanges) != 0 {
359 var tags = make(map[int][]string)
360 for n, rngs := range x.InlineStyleRanges {
361 Styles := &Styles[n]
362 switch rngs.Style {
363 case "BOLD":
364 rngs.Style = "b"
365 case "UNDERLINE":
366 rngs.Style = "u"
367 case "ITALIC":
368 rngs.Style = "i"
369 }
370 Styles.From = rngs.Offset
371 Styles.To = rngs.Offset + rngs.Length
372 FT := Styles.From * Styles.To
373 tags[FT] = append(tags[FT], rngs.Style)
374 }
375 for n := range Styles {
376 Styles := &Styles[n]
377 Styles.TxtRaw = x.Text[Styles.From:Styles.To]
378 Styles.Txt = TagBuilder(esc(Styles.TxtRaw), tags[Styles.From*Styles.To]...)
379 }
380 }
381
382 switch x.Type {
383 case "atomic":
384 if len(x.EntityRanges) != 0 {
385 d := entities[x.EntityRanges[0].Key]
386 parsedDescription.WriteString(`<a href="`)
387 parsedDescription.WriteString(esc(ConvertDeviantArtURLToSkunkyArt(host, d.Url)))
388 parsedDescription.WriteString(`"><img width="50%" src="`)
389 parsedDescription.WriteString(esc(ParseMedia(host, d.Media)))
390 parsedDescription.WriteString(`" alt="`)
391 parsedDescription.WriteString(esc(d.Author.Username))
392 parsedDescription.WriteString(" - ")
393 parsedDescription.WriteString(esc(d.Title))
394 parsedDescription.WriteString(`" title="`)
395 parsedDescription.WriteString(esc(d.Author.Username))
396 parsedDescription.WriteString(" - ")
397 parsedDescription.WriteString(esc(d.Title))
398 parsedDescription.WriteString(`"></a>`)
399 }
400 case "unstyled":
401 if l := len(Styles); l != 0 {
402 for n, r := range Styles {
403 var tag string
404 if x.Type == "header-two" {
405 tag = "h2"
406 }
407
408 parsedDescription.WriteString(esc(x.Text[:r.From]))
409 if len(urls) != 0 && len(x.EntityRanges) != 0 {
410 ra := &x.EntityRanges[0]
411
412 parsedDescription.WriteString(`<a target="_blank" href="`)
413 parsedDescription.WriteString(esc(urls[ra.Key]))
414 parsedDescription.WriteString(`">`)
415 parsedDescription.WriteString(r.Txt)
416 parsedDescription.WriteString(`</a>`)
417 } else if l > n+1 {
418 parsedDescription.WriteString(r.Txt)
419 }
420 parsedDescription.WriteString(TagBuilder(tag, esc(x.Text[r.To:])))
421 }
422 } else {
423 parsedDescription.WriteString(esc(x.Text))
424 }
425 }
426 parsedDescription.WriteString("<br>")
427 }
428 } else if dl != 0 {
429 for tt := html.NewTokenizer(strings.NewReader(dscr.Html.Markup)); ; {
430 switch tt.Next() {
431 case html.ErrorToken:
432 return parsedDescription.String()
433 case html.CommentToken, html.DoctypeToken:
434 // No renderable content; skip.
435 case html.StartTagToken, html.EndTagToken, html.SelfClosingTagToken:
436 token := tt.Token()
437 switch token.Data {
438 case "a":
439 for _, a := range token.Attr {
440 if a.Key == "href" {
441 url := DeleteTrackingFromURL(a.Val)
442 parsedDescription.WriteString(`<a target="_blank" href="`)
443 parsedDescription.WriteString(esc(url))
444 parsedDescription.WriteString(`">`)
445 parsedDescription.WriteString(esc(GetValueOfTag(tt)))
446 parsedDescription.WriteString("</a> ")
447 }
448 }
449 case "img":
450 // Only DeviantArt's emoticons are carried over, served
451 // through this instance; any other image is dropped.
452 var uri, title string
453 for _, a := range token.Attr {
454 switch a.Key {
455 case "src":
456 uri = emoticonURL(host, a.Val)
457 case "title":
458 title = a.Val
459 }
460 }
461 if uri != "" {
462 parsedDescription.WriteString(`<img src="`)
463 parsedDescription.WriteString(esc(uri))
464 parsedDescription.WriteString(`" alt="`)
465 parsedDescription.WriteString(esc(title))
466 parsedDescription.WriteString(`" title="`)
467 parsedDescription.WriteString(esc(title))
468 parsedDescription.WriteString(`">`)
469 }
470 case "br", "li", "ul", "p", "b":
471 // The bare tag, not token.String(): that would carry over
472 // whatever attributes DeviantArt's markup put on it.
473 if token.Type == html.EndTagToken {
474 parsedDescription.WriteString("</")
475 } else {
476 parsedDescription.WriteString("<")
477 }
478 parsedDescription.WriteString(token.Data)
479 parsedDescription.WriteString(">")
480 case "div":
481 parsedDescription.WriteString("<p> ")
482 }
483 case html.TextToken:
484 parsedDescription.WriteString(esc(string(tt.Text())))
485 }
486 }
487 }
488
489 return parsedDescription.String()
490}