app/parsers.go
470 lines · 15341 bytes · executable
1package app
2
3import (
4 "encoding/json"
5 "strconv"
6 "strings"
7 "time"
8
9 "github.com/krazywarez/devianter"
10 "golang.org/x/net/html"
11)
12
13// ParseComments renders a comment thread, nesting replies under the comment they
14// answer. It returns a placeholder message rather than failing if the upstream
15// fetch errored.
16func (s skunkyart) ParseComments(c devianter.Comments, daError devianter.Error) string {
17 if daError.RAW != nil {
18 return esc(T(s.Lang, "comments.failed"))
19 }
20
21 var cmmts strings.Builder
22 replied := make(map[int]string)
23
24 cmmts.WriteString("<details><summary>")
25 cmmts.WriteString(esc(T(s.Lang, "deviation.comments")))
26 cmmts.WriteString(": <b>")
27 cmmts.WriteString(strconv.Itoa(c.Total))
28 cmmts.WriteString("</b></summary>")
29 for _, x := range c.Thread {
30 replied[x.ID] = x.User.Username
31 cmmts.WriteString(`<div class="msg`)
32 if x.Parent > 0 {
33 cmmts.WriteString(` reply`)
34 }
35 cmmts.WriteString(`"><p id="`)
36 cmmts.WriteString(strconv.Itoa(x.ID))
37 cmmts.WriteString(`"><img src="`)
38 cmmts.WriteString(esc(URLBuilder(s.Host, "media", "emojitar", x.User.Username, "?type=a")))
39 cmmts.WriteString(`" width="30px" height="30px" alt="`)
40 cmmts.WriteString(esc(x.User.Username))
41 cmmts.WriteString(`"><a href="`)
42 cmmts.WriteString(esc(URLBuilder(s.Host, "group_user", "?q=", x.User.Username, "&type=a")))
43 cmmts.WriteString(`"><b`)
44 cmmts.WriteString(` class="`)
45 if x.User.Banned {
46 cmmts.WriteString(`banned`)
47 }
48 if x.Author {
49 cmmts.WriteString(`author`)
50 }
51 cmmts.WriteString(`">`)
52 cmmts.WriteString(esc(x.User.Username))
53 cmmts.WriteString("</b></a> ")
54
55 if x.Parent > 0 {
56 cmmts.WriteString(" ")
57 cmmts.WriteString(esc(T(s.Lang, "comments.reply_to")))
58 cmmts.WriteString(` <a href="`)
59 cmmts.WriteString(esc(s._pth))
60 cmmts.WriteString("#")
61 cmmts.WriteString(strconv.Itoa(x.Parent))
62 cmmts.WriteString(`">`)
63 if replied[x.Parent] == "" {
64 cmmts.WriteString("???")
65 } else {
66 cmmts.WriteString(esc(replied[x.Parent]))
67 }
68 cmmts.WriteString("</a>")
69 }
70 cmmts.WriteString(" [")
71 cmmts.WriteString(x.Posted.UTC().String())
72 cmmts.WriteString("]<p>")
73
74 cmmts.WriteString(ParseDescription(s.Host, x.TextContent))
75 cmmts.WriteString("<p>👍: ")
76 cmmts.WriteString(strconv.Itoa(x.Likes))
77 cmmts.WriteString(" ⏩: ")
78 cmmts.WriteString(strconv.Itoa(x.Replies))
79 cmmts.WriteString("</p></div>\n")
80 }
81 cmmts.WriteString(s.NavBase(DeviationList{
82 Pages: 0,
83 More: c.HasMore,
84 }))
85 cmmts.WriteString("</details>")
86 return cmmts.String()
87}
88
89// VisibleDeviation reports whether a deviation may be shown by this instance.
90//
91// Both the HTML listing and the JSON API ask this, deliberately: an API that
92// returned what the pages hide would leak exactly what hide-ai and the NSFW
93// setting exist to withhold, and a second copy of the rule is a second thing to
94// forget to update.
95func VisibleDeviation(d *devianter.Deviation) bool {
96 if d.AI && CFG.HideAI {
97 return false
98 }
99 return !d.NSFW || CFG.Nsfw
100}
101
102// DeviationList renders devs as an HTML grid, or as an Atom feed when the
103// request asked for one and allowAtom permits it. NSFW entries are dropped
104// unless the instance allows them. Passing content adds a navigation bar.
105func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, content ...DeviationList) string {
106 if s.Atom && s.Page > 1 {
107 s.ReturnHTTPError(400)
108 return ""
109 }
110
111 var list, listContent strings.Builder
112 var newest time.Time // the latest entry, for the feed's own <updated>
113
114 for i, l := 0, len(devs); i < l; i++ {
115 data := &devs[i]
116 if !VisibleDeviation(data) {
117 continue
118 }
119 // Escaped once here: the same values go into both the HTML grid and the
120 // Atom feed, and html.EscapeString produces entities XML accepts too.
121 author, title := esc(data.Author.Username), esc(data.Title)
122 postURL := esc(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url))
123 if preview, fullview := esc(ParseMedia(s.Host, data.Media, 320)), esc(ParseMedia(s.Host, data.Media)); true {
124 if allowAtom && s.Atom {
125 s.Writer.Header().Add("Content-Type", "application/atom+xml")
126 // The entry id is the post's URL here: Atom wants an IRI, and a
127 // bare number is not one. Timestamps are RFC 3339, the only form
128 // the format allows.
129 entryURL := esc(URLBuilder(s.Host, "post", data.Author.Username, "atom-"+strconv.Itoa(data.ID)))
130 published := data.PublishedTime.UTC()
131 if published.After(newest) {
132 newest = published
133 }
134 listContent.WriteString(`<entry><author><name>`)
135 listContent.WriteString(author)
136 listContent.WriteString(`</name></author><title>`)
137 listContent.WriteString(title)
138 listContent.WriteString(`</title><link rel="alternate" type="text/html" href="`)
139 listContent.WriteString(entryURL)
140 listContent.WriteString(`"/><id>`)
141 listContent.WriteString(entryURL)
142 listContent.WriteString(`</id><published>`)
143 listContent.WriteString(published.Format(time.RFC3339))
144 listContent.WriteString(`</published><updated>`)
145 listContent.WriteString(published.Format(time.RFC3339))
146 listContent.WriteString(`</updated>`)
147 listContent.WriteString(`<media:group><media:title>`)
148 listContent.WriteString(title)
149 listContent.WriteString(`</media:title><media:thumbnail url="`)
150 listContent.WriteString(preview)
151 listContent.WriteString(`"/></media:group><content type="xhtml"><div xmlns="http://www.w3.org/1999/xhtml"><a href="`)
152 listContent.WriteString(postURL)
153 listContent.WriteString(`"><img src="`)
154 listContent.WriteString(fullview)
155 listContent.WriteString(`"/></a><p>`)
156 listContent.WriteString(ParseDescription(s.Host, data.TextContent))
157 listContent.WriteString(`</p></div></content></entry>`)
158 } else {
159 listContent.WriteString(`<div class="block">`)
160 if fullview != "" && preview != "" {
161 listContent.WriteString(`<a title="` + esc(T(s.Lang, "deviation.open")) + `" href="`)
162 listContent.WriteString(fullview)
163 listContent.WriteString(`"><img loading="lazy" src="`)
164 listContent.WriteString(preview)
165 listContent.WriteString(`" width="15%" alt="`)
166 listContent.WriteString(author)
167 listContent.WriteString(" - ")
168 listContent.WriteString(title)
169 listContent.WriteString(`"></a>`)
170 } else {
171 listContent.WriteString("<h1>[ " + esc(T(s.Lang, "list.text")) + " ]</h1>")
172 }
173 listContent.WriteString(`<br><a href="`)
174 listContent.WriteString(postURL)
175 listContent.WriteString(`">`)
176 listContent.WriteString(author)
177 listContent.WriteString(" - ")
178 listContent.WriteString(title)
179
180 if data.NSFW {
181 listContent.WriteString(` [<span class="nsfw">NSFW</span>]`)
182 }
183 if data.AI {
184 listContent.WriteString(" [🤖]")
185 }
186 if data.DD {
187 listContent.WriteString(` [<span class="dd">DD</span>]`)
188 }
189
190 listContent.WriteString("</a></div>")
191 }
192 }
193 }
194
195 if allowAtom && s.Atom {
196 list.WriteString(`<?xml version="1.0" encoding="UTF-8"?><feed xmlns:media="http://search.yahoo.com/mrss/" xmlns="http://www.w3.org/2005/Atom">`)
197
198 list.WriteString(`<title>`)
199 switch {
200 case s.Type == 0:
201 list.WriteString("Daily Deviations")
202 case s.Type == 'g' && len(devs) != 0:
203 list.WriteString(esc(devs[0].Author.Username))
204 default:
205 list.WriteString("SkunkyArt")
206 }
207 list.WriteString(`</title>`)
208
209 // Atom requires a feed id and updated time; readers reject a feed
210 // without them. The id is the feed's own URL on this instance.
211 feedURL := esc(s.Host + s._pth + "?" + s.Args.Encode())
212 if newest.IsZero() {
213 newest = time.Now().UTC()
214 }
215 list.WriteString(`<id>`)
216 list.WriteString(feedURL)
217 list.WriteString(`</id><updated>`)
218 list.WriteString(newest.Format(time.RFC3339))
219 list.WriteString(`</updated><link rel="self" type="application/atom+xml" href="`)
220 list.WriteString(feedURL)
221 list.WriteString(`"/>`)
222
223 list.WriteString(`<link rel="alternate" href="`)
224 list.WriteString(esc(s.Host))
225 list.WriteString(`"/>`)
226
227 list.WriteString(listContent.String())
228
229 list.WriteString("</feed>")
230 wr(s.Writer, list.String())
231 } else {
232 list.WriteString(`<div class="content">`)
233
234 list.WriteString(listContent.String())
235
236 list.WriteString("</div>")
237 if content != nil {
238 list.WriteString(s.NavBase(content[0]))
239 }
240 }
241
242 return list.String()
243}
244
245/* DESCRIPTION/COMMENT PARSER */
246
247// text is one styled run within a description: the rendered HTML, the raw source
248// it came from, and the offsets it spans in the original block.
249type text struct {
250 Txt string
251 TxtRaw string
252 From int
253 To int
254}
255
256// ParseDescription renders a DeviantArt description into HTML, handling both the
257// Draft.js-style JSON payload and the plain HTML markup DeviantArt returns, and
258// rewriting embedded links and artwork references to point at this instance.
259// host is the request's scheme and host, as taken by URLBuilder.
260//
261// TODO: rewrite this whole mess.
262func ParseDescription(host string, dscr devianter.Text) string {
263 var parsedDescription strings.Builder
264 TagBuilder := func(content string, tags ...string) string {
265 l := len(tags)
266 for x := range l {
267 var htm strings.Builder
268 htm.WriteString("<")
269 htm.WriteString(tags[x])
270 htm.WriteString(">")
271
272 htm.WriteString(content)
273
274 htm.WriteString("</")
275 htm.WriteString(tags[x])
276 htm.WriteString(">")
277 content = htm.String()
278 }
279 return content
280 }
281 DeleteTrackingFromURL := func(url string) string {
282 if len(url) > 42 && url[:42] == "https://www.deviantart.com/users/outgoing?" {
283 url = url[42:]
284 }
285 return url
286 }
287
288 if description, dl := dscr.Html.Markup, len(dscr.Html.Markup); dl != 0 &&
289 description[0] == '{' &&
290 description[dl-1] == '}' {
291 var descr struct {
292 Blocks []struct {
293 Text string `json:"text"`
294 Type string `json:"type"`
295 InlineStyleRanges []struct {
296 Offset int `json:"offset"`
297 Length int `json:"length"`
298 Style string `json:"style"`
299 } `json:"inlineStyleRanges"`
300 EntityRanges []struct {
301 Offset int `json:"offset"`
302 Length int `json:"length"`
303 Key int `json:"key"`
304 } `json:"entityRanges"`
305 Data struct {
306 TextAlignment string `json:"textAlignment"`
307 } `json:"data"`
308 } `json:"blocks"`
309 EntityMap map[string]struct {
310 Type string `json:"type"`
311 Data struct {
312 URL string `json:"url"`
313 Config struct {
314 // "aligment" is DeviantArt's own spelling; do not correct it.
315 Aligment string `json:"aligment"`
316 Width int `json:"width"`
317 } `json:"config"`
318 Data devianter.Deviation `json:"data"`
319 } `json:"data"`
320 } `json:"entityMap"`
321 }
322 e := json.Unmarshal([]byte(description), &descr)
323 try(e)
324
325 entities := make(map[int]devianter.Deviation)
326 urls := make(map[int]string)
327 for n, x := range descr.EntityMap {
328 num, _ := strconv.Atoi(n)
329 if x.Data.URL != "" {
330 urls[num] = DeleteTrackingFromURL(x.Data.URL)
331 }
332 entities[num] = x.Data.Data
333 }
334
335 for _, x := range descr.Blocks {
336 Styles := make([]text, len(x.InlineStyleRanges))
337
338 if len(x.InlineStyleRanges) != 0 {
339 var tags = make(map[int][]string)
340 for n, rngs := range x.InlineStyleRanges {
341 Styles := &Styles[n]
342 switch rngs.Style {
343 case "BOLD":
344 rngs.Style = "b"
345 case "UNDERLINE":
346 rngs.Style = "u"
347 case "ITALIC":
348 rngs.Style = "i"
349 }
350 Styles.From = rngs.Offset
351 Styles.To = rngs.Offset + rngs.Length
352 FT := Styles.From * Styles.To
353 tags[FT] = append(tags[FT], rngs.Style)
354 }
355 for n := range Styles {
356 Styles := &Styles[n]
357 Styles.TxtRaw = x.Text[Styles.From:Styles.To]
358 Styles.Txt = TagBuilder(esc(Styles.TxtRaw), tags[Styles.From*Styles.To]...)
359 }
360 }
361
362 switch x.Type {
363 case "atomic":
364 if len(x.EntityRanges) != 0 {
365 d := entities[x.EntityRanges[0].Key]
366 parsedDescription.WriteString(`<a href="`)
367 parsedDescription.WriteString(esc(ConvertDeviantArtURLToSkunkyArt(host, d.Url)))
368 parsedDescription.WriteString(`"><img width="50%" src="`)
369 parsedDescription.WriteString(esc(ParseMedia(host, d.Media)))
370 parsedDescription.WriteString(`" alt="`)
371 parsedDescription.WriteString(esc(d.Author.Username))
372 parsedDescription.WriteString(" - ")
373 parsedDescription.WriteString(esc(d.Title))
374 parsedDescription.WriteString(`" title="`)
375 parsedDescription.WriteString(esc(d.Author.Username))
376 parsedDescription.WriteString(" - ")
377 parsedDescription.WriteString(esc(d.Title))
378 parsedDescription.WriteString(`"></a>`)
379 }
380 case "unstyled":
381 if l := len(Styles); l != 0 {
382 for n, r := range Styles {
383 var tag string
384 if x.Type == "header-two" {
385 tag = "h2"
386 }
387
388 parsedDescription.WriteString(esc(x.Text[:r.From]))
389 if len(urls) != 0 && len(x.EntityRanges) != 0 {
390 ra := &x.EntityRanges[0]
391
392 parsedDescription.WriteString(`<a target="_blank" href="`)
393 parsedDescription.WriteString(esc(urls[ra.Key]))
394 parsedDescription.WriteString(`">`)
395 parsedDescription.WriteString(r.Txt)
396 parsedDescription.WriteString(`</a>`)
397 } else if l > n+1 {
398 parsedDescription.WriteString(r.Txt)
399 }
400 parsedDescription.WriteString(TagBuilder(tag, esc(x.Text[r.To:])))
401 }
402 } else {
403 parsedDescription.WriteString(esc(x.Text))
404 }
405 }
406 parsedDescription.WriteString("<br>")
407 }
408 } else if dl != 0 {
409 for tt := html.NewTokenizer(strings.NewReader(dscr.Html.Markup)); ; {
410 switch tt.Next() {
411 case html.ErrorToken:
412 return parsedDescription.String()
413 case html.CommentToken, html.DoctypeToken:
414 // No renderable content; skip.
415 case html.StartTagToken, html.EndTagToken, html.SelfClosingTagToken:
416 token := tt.Token()
417 switch token.Data {
418 case "a":
419 for _, a := range token.Attr {
420 if a.Key == "href" {
421 url := DeleteTrackingFromURL(a.Val)
422 parsedDescription.WriteString(`<a target="_blank" href="`)
423 parsedDescription.WriteString(esc(url))
424 parsedDescription.WriteString(`">`)
425 parsedDescription.WriteString(esc(GetValueOfTag(tt)))
426 parsedDescription.WriteString("</a> ")
427 }
428 }
429 case "img":
430 var uri, title string
431 for b, a := range token.Attr {
432 switch a.Key {
433 case "src":
434 if len(a.Val) > 9 && a.Val[8:9] == "e" {
435 uri = URLBuilder(host, "media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
436 }
437 case "title":
438 title = a.Val
439 }
440 if title != "" {
441 for x := -1; x < b; x++ {
442 parsedDescription.WriteString(`<img src="`)
443 parsedDescription.WriteString(esc(uri))
444 parsedDescription.WriteString(`" title="`)
445 parsedDescription.WriteString(esc(title))
446 parsedDescription.WriteString(`">`)
447 }
448 }
449 }
450 case "br", "li", "ul", "p", "b":
451 // The bare tag, not token.String(): that would carry over
452 // whatever attributes DeviantArt's markup put on it.
453 if token.Type == html.EndTagToken {
454 parsedDescription.WriteString("</")
455 } else {
456 parsedDescription.WriteString("<")
457 }
458 parsedDescription.WriteString(token.Data)
459 parsedDescription.WriteString(">")
460 case "div":
461 parsedDescription.WriteString("<p> ")
462 }
463 case html.TextToken:
464 parsedDescription.WriteString(esc(string(tt.Text())))
465 }
466 }
467 }
468
469 return parsedDescription.String()
470}