Commit `5adf3557d1`

5adf3557d1d92868b8d48bdd05b2d018158916f3

parent: c22afa0a45

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-12 02:19 UTC

Render the document description format

DeviantArt's editor now stores descriptions and comments as a document
tree ({"version":1,"document":...}) rather than Draft.js blocks, and
the parser, expecting blocks, rendered every one of them empty. A new
renderer walks the tree: paragraphs, headings, lists, quotes, code,
rules and breaks; text with bold, italic, underline, strike, code and
link marks (outgoing-link tracking stripped); official and custom
emotes through the emote route; embedded artworks as proxied
thumbnails linking to the post, subject to the instance's NSFW and
hide-ai rules; GIF embeds as links; mentions as profile links. Unknown
containers render their children, unknown leaves nothing. Every string
is escaped. Draft.js and plain HTML still go to their own parsers.

Checked against 92 live descriptions and comments fetched from the
public instance's egress.

Closes #3
Closes #6

Layout: unified · split

app/document.go added +241

@@ -0,0 +1,241 @@
	1	package app
	2
	3	import (
	4	"encoding/json"
	5	"slices"
	6	"strconv"
	7	"strings"
	8
	9	"github.com/krazywarez/devianter"
	10	)
	11
	12	// docNode is one node of the document format DeviantArt's current editor
	13	// stores: a tree of typed nodes, text leaves carrying marks. Only the fields
	14	// the renderer reads are declared; attrs is decoded per node type.
	15	type docNode struct {
	16	Type string `json:"type"`
	17	Text string `json:"text"`
	18	Attrs json.RawMessage `json:"attrs"`
	19	Marks []docMark `json:"marks"`
	20	Content []docNode `json:"content"`
	21	}
	22
	23	type docMark struct {
	24	Type string `json:"type"`
	25	Attrs struct {
	26	Href string `json:"href"`
	27	} `json:"attrs"`
	28	}
	29
	30	// parseDocument reports whether markup is a document-format description and
	31	// returns its root. A JSON payload without a "document" key, such as the
	32	// older Draft.js form, is not one.
	33	func parseDocument(markup string) (docNode, bool) {
	34	if len(markup) == 0 \|\| markup[0] != '{' {
	35	return docNode{}, false
	36	}
	37	var d struct {
	38	Document *docNode `json:"document"`
	39	}
	40	if json.Unmarshal([]byte(markup), &d) != nil \|\| d.Document == nil {
	41	return docNode{}, false
	42	}
	43	return *d.Document, true
	44	}
	45
	46	// deleteTrackingFromURL strips DeviantArt's outgoing-link redirector, so a
	47	// link goes where it says rather than through deviantart.com first.
	48	func deleteTrackingFromURL(u string) string {
	49	return strings.TrimPrefix(u, "https://www.deviantart.com/users/outgoing?")
	50	}
	51
	52	// renderDocument renders a document-format description as HTML. Every string
	53	// from the payload is escaped here, since the result is handed to the
	54	// template as trusted HTML. Unknown container nodes render their children;
	55	// unknown leaves render nothing.
	56	func renderDocument(host string, root docNode) string {
	57	var b strings.Builder
	58	renderDocNode(&b, host, root)
	59	return b.String()
	60	}
	61
	62	func renderDocNode(b *strings.Builder, host string, n docNode) {
	63	children := func() {
	64	for _, c := range n.Content {
	65	renderDocNode(b, host, c)
	66	}
	67	}
	68	wrap := func(opening, closing string) {
	69	b.WriteString(opening)
	70	children()
	71	b.WriteString(closing)
	72	}
	73
	74	switch n.Type {
	75	case "text":
	76	renderDocText(b, host, n)
	77	case "paragraph":
	78	wrap("<p>", "</p>")
	79	case "heading":
	80	var a struct {
	81	Level int `json:"level"`
	82	}
	83	_ = json.Unmarshal(n.Attrs, &a)
	84	// The page's own title is its h1; keep description headings below it.
	85	level := min(max(a.Level+1, 2), 4)
	86	tag := "h" + strconv.Itoa(level)
	87	wrap("<"+tag+">", "</"+tag+">")
	88	case "bulletList":
	89	wrap("<ul>", "</ul>")
	90	case "orderedList":
	91	wrap("<ol>", "</ol>")
	92	case "listItem":
	93	wrap("<li>", "</li>")
	94	case "blockquote":
	95	wrap("<blockquote>", "</blockquote>")
	96	case "codeBlock":
	97	wrap("<pre>", "</pre>")
	98	case "hardBreak":
	99	b.WriteString("<br>")
	100	case "horizontalRule":
	101	b.WriteString("<hr>")
	102	case "da-emote":
	103	renderDocEmote(b, host, n.Attrs)
	104	case "da-deviation", "da-deviation-thumb":
	105	renderDocDeviation(b, host, n.Attrs)
	106	case "da-gif":
	107	var a struct {
	108	URL string `json:"url"`
	109	}
	110	_ = json.Unmarshal(n.Attrs, &a)
	111	if a.URL != "" {
	112	b.WriteString(`<a target="_blank" href="`)
	113	b.WriteString(esc(a.URL))
	114	b.WriteString(`">[GIF]</a>`)
	115	}
	116	case "da-mention":
	117	var a struct {
	118	Username string `json:"username"`
	119	User struct {
	120	Username string `json:"username"`
	121	} `json:"user"`
	122	}
	123	_ = json.Unmarshal(n.Attrs, &a)
	124	name := a.Username
	125	if name == "" {
	126	name = a.User.Username
	127	}
	128	if name != "" {
	129	b.WriteString(`<a href="`)
	130	b.WriteString(esc(URLBuilder(host, "group_user", "?type=about&q=", name)))
	131	b.WriteString(`">@`)
	132	b.WriteString(esc(name))
	133	b.WriteString("</a>")
	134	}
	135	default:
	136	children()
	137	}
	138	}
	139
	140	// docMarkTags maps the inline marks to the tags that render them. textStyle
	141	// carries colour and font choices, which the instance's own stylesheet
	142	// decides, so it is not listed.
	143	var docMarkTags = map[string]string{
	144	"bold": "b", "strong": "b",
	145	"italic": "i", "em": "i",
	146	"underline": "u",
	147	"strike": "s", "strikethrough": "s",
	148	"code": "code",
	149	}
	150
	151	func renderDocText(b *strings.Builder, _ string, n docNode) {
	152	var opening strings.Builder
	153	var closing []string // pushed in mark order, emitted in reverse
	154	for _, m := range n.Marks {
	155	if m.Type == "link" && m.Attrs.Href != "" {
	156	opening.WriteString(`<a target="_blank" href="`)
	157	opening.WriteString(esc(deleteTrackingFromURL(m.Attrs.Href)))
	158	opening.WriteString(`">`)
	159	closing = append(closing, "</a>")
	160	continue
	161	}
	162	if tag, ok := docMarkTags[m.Type]; ok {
	163	opening.WriteString("<" + tag + ">")
	164	closing = append(closing, "</"+tag+">")
	165	}
	166	}
	167	b.WriteString(opening.String())
	168	b.WriteString(esc(n.Text))
	169	for _, c := range slices.Backward(closing) {
	170	b.WriteString(c)
	171	}
	172	}
	173
	174	// renderDocEmote renders an emoticon through the instance's emote route. An
	175	// official emote names its image; when it does not, the emote code without
	176	// its colons is the best guess at the file name.
	177	func renderDocEmote(b *strings.Builder, host string, attrs json.RawMessage) {
	178	var a struct {
	179	Code string `json:"data-emote"`
	180	Src string `json:"src"`
	181	Title string `json:"title"`
	182	}
	183	_ = json.Unmarshal(attrs, &a)
	184	src := emoticonURL(host, a.Src)
	185	if src == "" {
	186	if name := strings.Trim(a.Code, ":"); name != "" {
	187	src = URLBuilder(host, "media", "emojitar", name, "?type=e")
	188	}
	189	}
	190	if src == "" {
	191	return
	192	}
	193	label := a.Title
	194	if label == "" {
	195	label = a.Code
	196	}
	197	b.WriteString(`<img src="`)
	198	b.WriteString(esc(src))
	199	b.WriteString(`" alt="`)
	200	b.WriteString(esc(label))
	201	b.WriteString(`" title="`)
	202	b.WriteString(esc(label))
	203	b.WriteString(`">`)
	204	}
	205
	206	// renderDocDeviation renders an embedded artwork as its thumbnail linking to
	207	// the post on this instance, or as a text link when it has no media.
	208	func renderDocDeviation(b *strings.Builder, host string, attrs json.RawMessage) {
	209	var a struct {
	210	Deviation devianter.Deviation `json:"deviation"`
	211	}
	212	if json.Unmarshal(attrs, &a) != nil {
	213	return
	214	}
	215	d := &a.Deviation
	216	if !VisibleDeviation(d) {
	217	return
	218	}
	219	link := ConvertDeviantArtURLToSkunkyArt(host, d.Url)
	220	label := esc(d.Author.Username + " - " + d.Title)
	221	img := ParseMedia(host, d.Media, 320)
	222	if link != "" {
	223	b.WriteString(`<a href="`)
	224	b.WriteString(esc(link))
	225	b.WriteString(`">`)
	226	}
	227	if img != "" {
	228	b.WriteString(`<img width="50%" src="`)
	229	b.WriteString(esc(img))
	230	b.WriteString(`" alt="`)
	231	b.WriteString(label)
	232	b.WriteString(`" title="`)
	233	b.WriteString(label)
	234	b.WriteString(`">`)
	235	} else {
	236	b.WriteString(label)
	237	}
	238	if link != "" {
	239	b.WriteString("</a>")
	240	}
	241	}

app/document_test.go added +122

@@ -0,0 +1,122 @@
	1	package app
	2
	3	import (
	4	"strings"
	5	"testing"
	6
	7	"github.com/krazywarez/devianter"
	8	)
	9
	10	// docText wraps a document body the way DeviantArt stores it.
	11	func docText(body string) devianter.Text {
	12	var t devianter.Text
	13	t.Html.Markup = `{"version":1,"document":{"type":"doc","content":[` + body + `]},"features":[]}`
	14	return t
	15	}
	16
	17	func withDocConfig(t *testing.T) {
	18	t.Helper()
	19	proxy, uri, nsfw := CFG.Proxy, CFG.URI, CFG.Nsfw
	20	CFG.Proxy, CFG.URI, CFG.Nsfw = true, "/", true
	21	t.Cleanup(func() { CFG.Proxy, CFG.URI, CFG.Nsfw = proxy, uri, nsfw })
	22	}
	23
	24	// TestDocumentParagraphsMarksAndBreaks uses the shape seen on live
	25	// descriptions: paragraphs of text with link, underline and italic marks.
	26	func TestDocumentParagraphsMarksAndBreaks(t *testing.T) {
	27	withDocConfig(t)
	28	out := ParseDescription("http://localhost", docText(`
	29	{"type":"paragraph","attrs":{"textAlign":"center"},"content":[
	30	{"type":"text","text":"\| "},
	31	{"type":"text","marks":[{"type":"link","attrs":{"href":"https://www.deviantart.com/users/outgoing?https://www.patreon.com/x","target":"_blank"}},{"type":"underline"}],"text":"PATREON"},
	32	{"type":"hardBreak"},
	33	{"type":"text","marks":[{"type":"italic"}],"text":"Finished <YCH> for "},
	34	{"type":"text","marks":[{"type":"textStyle"}],"text":"plain"}
	35	]}`))
	36	for _, want := range []string{
	37	`<p>\| <a target="_blank" href="https://www.patreon.com/x"><u>PATREON</u></a><br><i>Finished <YCH> for </i>plain</p>`,
	38	} {
	39	if !strings.Contains(out, want) {
	40	t.Errorf("output lacks %q:\n%s", want, out)
	41	}
	42	}
	43	}
	44
	45	func TestDocumentBlocksAndLists(t *testing.T) {
	46	withDocConfig(t)
	47	out := ParseDescription("http://localhost", docText(`
	48	{"type":"heading","attrs":{"level":1},"content":[{"type":"text","text":"Title"}]},
	49	{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold"}],"text":"one"}]}]}]},
	50	{"type":"orderedList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"text","text":"two"}]}]}]},
	51	{"type":"blockquote","content":[{"type":"paragraph","content":[{"type":"text","text":"quoted"}]}]},
	52	{"type":"codeBlock","content":[{"type":"text","text":"x < y"}]},
	53	{"type":"horizontalRule"},
	54	{"type":"mystery","content":[{"type":"text","text":"still shown"}]},
	55	{"type":"mysteryLeaf","attrs":{"x":1}}`))
	56	for _, want := range []string{"<h2>Title</h2>", "<ul><li><p><b>one</b></p></li></ul>", "<ol><li><p>two</p></li></ol>", "<blockquote><p>quoted</p></blockquote>", "<pre>x < y</pre>", "<hr>", "still shown"} {
	57	if !strings.Contains(out, want) {
	58	t.Errorf("output lacks %q:\n%s", want, out)
	59	}
	60	}
	61	}
	62
	63	// TestDocumentEmotes is the new-format half of #6: an official emote with a
	64	// source image maps to the emote route by file name, one without a source
	65	// falls back to its code.
	66	func TestDocumentEmotes(t *testing.T) {
	67	withDocConfig(t)
	68	out := ParseDescription("http://localhost", docText(`
	69	{"type":"paragraph","content":[
	70	{"type":"da-emote","attrs":{"data-emote":":star:","data-type":"official","src":"https://e.deviantart.net/emoticons/s/star_full.gif","width":17,"height":16,"url":null,"title":null}},
	71	{"type":"da-emote","attrs":{"data-emote":":love:","data-type":"official","src":null,"width":null,"height":null,"url":null,"title":null}},
	72	{"type":"da-emote","attrs":{"data-emote":"","data-type":"custom","src":null}}
	73	]}`))
	74	for _, want := range []string{`src="http://localhost/media/emojitar/star_full?type=e" alt=":star:"`, `src="http://localhost/media/emojitar/love?type=e" alt=":love:"`} {
	75	if !strings.Contains(out, want) {
	76	t.Errorf("output lacks %q:\n%s", want, out)
	77	}
	78	}
	79	if strings.Count(out, "<img") != 2 {
	80	t.Errorf("want 2 images (the nameless emote renders nothing):\n%s", out)
	81	}
	82	}
	83
	84	func TestDocumentEmbeddedDeviationAndGif(t *testing.T) {
	85	withDocConfig(t)
	86	out := ParseDescription("http://localhost", docText(`
	87	{"type":"paragraph","content":[
	88	{"type":"da-deviation-thumb","attrs":{"cropping":"fill","deviation":{"deviationId":1276762071,"url":"https://www.deviantart.com/ashiori-chan/art/CLOSED-YCH-Portrait-auction-1276762071","title":"[CLOSED] YCH","author":{"username":"AShiori-chan"},"media":{"baseUri":"https://images-wixmp-abc.wixmp.com/f/u/x.png","prettyName":"x_by_y","token":["tok.en.sig"],"types":[{"t":"fullview","h":1920,"w":1280}]}}}},
	89	{"type":"da-gif","attrs":{"url":"https://media4.giphy.com/media/x/giphy.mp4","width":480,"height":362}}
	90	]}`))
	91	for _, want := range []string{`<a href="http://localhost/post/ashiori-chan/CLOSED-YCH-Portrait-auction-1276762071">`, `src="http://localhost/media/file/abc/`, `alt="AShiori-chan - [CLOSED] YCH"`, `<a target="_blank" href="https://media4.giphy.com/media/x/giphy.mp4">[GIF]</a>`} {
	92	if !strings.Contains(out, want) {
	93	t.Errorf("output lacks %q:\n%s", want, out)
	94	}
	95	}
	96	}
	97
	98	func TestDocumentHidesEmbeddedNSFWWhenDisabled(t *testing.T) {
	99	withDocConfig(t)
	100	CFG.Nsfw = false
	101	out := ParseDescription("http://localhost", docText(`
	102	{"type":"paragraph","content":[{"type":"da-deviation","attrs":{"deviation":{"deviationId":1,"isMature":true,"url":"https://www.deviantart.com/a/art/b-1","title":"secret","author":{"username":"a"}}}}]}`))
	103	if strings.Contains(out, "secret") {
	104	t.Errorf("mature embed shown with nsfw off:\n%s", out)
	105	}
	106	}
	107
	108	// TestDraftJSAndHTMLStillDispatch pins that the two older formats still reach
	109	// their own parsers.
	110	func TestDraftJSAndHTMLStillDispatch(t *testing.T) {
	111	withDocConfig(t)
	112	var draft devianter.Text
	113	draft.Html.Markup = `{"blocks":[{"text":"draft text","type":"unstyled","inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}`
	114	if out := ParseDescription("http://localhost", draft); !strings.Contains(out, "draft text") {
	115	t.Errorf("Draft.js description not rendered:\n%s", out)
	116	}
	117	var legacy devianter.Text
	118	legacy.Html.Markup = `plain <b>html</b>`
	119	if out := ParseDescription("http://localhost", legacy); !strings.Contains(out, "<b>html</b>") {
	120	t.Errorf("HTML description not rendered:\n%s", out)
	121	}
	122	}

app/parsers.go +9 −7

 // rewriting embedded links and artwork references to point at this instance.
 // host is the request's scheme and host, as taken by URLBuilder.
 //
-// TODO: rewrite this whole mess.
+// Three formats arrive: the current editor's document JSON (a "document"
+// key), handled by renderDocument; the older Draft.js JSON ("blocks" and
+// "entityMap"), handled below and kept for the posts that still carry it; and
+// plain HTML markup from before either.
 func ParseDescription(host string, dscr devianter.Text) string {
+	if doc, ok := parseDocument(dscr.Html.Markup); ok {
+		return renderDocument(host, doc)
+	}
 	var parsedDescription strings.Builder
 	TagBuilder := func(content string, tags ...string) string {
 		l := len(tags)
 		}
 		return content
 	}
-	DeleteTrackingFromURL := func(url string) string {
+	DeleteTrackingFromURL := deleteTrackingFromURL
-		if len(url) > 42 && url[:42] == "https://www.deviantart.com/users/outgoing?" {
-			url = url[42:]
-		}
-		return url
-	}
 	if description, dl := dscr.Html.Markup, len(dscr.Html.Markup); dl != 0 &&
 		description[0] == '{' &&

Commit 5adf3557d1

Commit `5adf3557d1`