Commit `5adf3557d1`

5adf3557d1d92868b8d48bdd05b2d018158916f3

parent: c22afa0a45

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-12 02:19 UTC

Render the document description format

DeviantArt's editor now stores descriptions and comments as a document
tree ({"version":1,"document":...}) rather than Draft.js blocks, and
the parser, expecting blocks, rendered every one of them empty. A new
renderer walks the tree: paragraphs, headings, lists, quotes, code,
rules and breaks; text with bold, italic, underline, strike, code and
link marks (outgoing-link tracking stripped); official and custom
emotes through the emote route; embedded artworks as proxied
thumbnails linking to the post, subject to the instance's NSFW and
hide-ai rules; GIF embeds as links; mentions as profile links. Unknown
containers render their children, unknown leaves nothing. Every string
is escaped. Draft.js and plain HTML still go to their own parsers.

Checked against 92 live descriptions and comments fetched from the
public instance's egress.

Closes #3
Closes #6

Layout: unified · split

app/document.go added +241

		@@ -0,0 +1,241 @@
	1	package app
	2
	3	import (
	4	"encoding/json"
	5	"slices"
	6	"strconv"
	7	"strings"
	8
	9	"github.com/krazywarez/devianter"
	10	)
	11
	12	// docNode is one node of the document format DeviantArt's current editor
	13	// stores: a tree of typed nodes, text leaves carrying marks. Only the fields
	14	// the renderer reads are declared; attrs is decoded per node type.
	15	type docNode struct {
	16	Type string `json:"type"`
	17	Text string `json:"text"`
	18	Attrs json.RawMessage `json:"attrs"`
	19	Marks []docMark `json:"marks"`
	20	Content []docNode `json:"content"`
	21	}
	22
	23	type docMark struct {
	24	Type string `json:"type"`
	25	Attrs struct {
	26	Href string `json:"href"`
	27	} `json:"attrs"`
	28	}
	29
	30	// parseDocument reports whether markup is a document-format description and
	31	// returns its root. A JSON payload without a "document" key, such as the
	32	// older Draft.js form, is not one.
	33	func parseDocument(markup string) (docNode, bool) {
	34	if len(markup) == 0 \|\| markup[0] != '{' {
	35	return docNode{}, false
	36	}
	37	var d struct {
	38	Document *docNode `json:"document"`
	39	}
	40	if json.Unmarshal([]byte(markup), &d) != nil \|\| d.Document == nil {
	41	return docNode{}, false
	42	}
	43	return *d.Document, true
	44	}
	45
	46	// deleteTrackingFromURL strips DeviantArt's outgoing-link redirector, so a
	47	// link goes where it says rather than through deviantart.com first.
	48	func deleteTrackingFromURL(u string) string {
	49	return strings.TrimPrefix(u, "https://www.deviantart.com/users/outgoing?")
	50	}
	51
	52	// renderDocument renders a document-format description as HTML. Every string
	53	// from the payload is escaped here, since the result is handed to the
	54	// template as trusted HTML. Unknown container nodes render their children;
	55	// unknown leaves render nothing.
	56	func renderDocument(host string, root docNode) string {
	57	var b strings.Builder
	58	renderDocNode(&b, host, root)
	59	return b.String()
	60	}
	61
	62	func renderDocNode(b *strings.Builder, host string, n docNode) {
	63	children := func() {
	64	for _, c := range n.Content {
	65	renderDocNode(b, host, c)
	66	}
	67	}
	68	wrap := func(opening, closing string) {
	69	b.WriteString(opening)
	70	children()
	71	b.WriteString(closing)
	72	}
	73
	74	switch n.Type {
	75	case "text":
	76	renderDocText(b, host, n)
	77	case "paragraph":
	78	wrap("<p>", "</p>")
	79	case "heading":
	80	var a struct {
	81	Level int `json:"level"`
	82	}
	83	_ = json.Unmarshal(n.Attrs, &a)
	84	// The page's own title is its h1; keep description headings below it.
	85	level := min(max(a.Level+1, 2), 4)
	86	tag := "h" + strconv.Itoa(level)
	87	wrap("<"+tag+">", "</"+tag+">")
	88	case "bulletList":
	89	wrap("<ul>", "</ul>")
	90	case "orderedList":
	91	wrap("<ol>", "</ol>")
	92	case "listItem":
	93	wrap("<li>", "</li>")
	94	case "blockquote":
	95	wrap("<blockquote>", "</blockquote>")
	96	case "codeBlock":
	97	wrap("<pre>", "</pre>")
	98	case "hardBreak":
	99	b.WriteString("<br>")
	100	case "horizontalRule":
	101	b.WriteString("<hr>")
	102	case "da-emote":
	103	renderDocEmote(b, host, n.Attrs)
	104	case "da-deviation", "da-deviation-thumb":
	105	renderDocDeviation(b, host, n.Attrs)
	106	case "da-gif":
	107	var a struct {
	108	URL string `json:"url"`
	109	}
	110	_ = json.Unmarshal(n.Attrs, &a)
	111	if a.URL != "" {
	112	b.WriteString(`<a target="_blank" href="`)
	113	b.WriteString(esc(a.URL))
	114	b.WriteString(`">[GIF]</a>`)
	115	}
	116	case "da-mention":
	117	var a struct {
	118	Username string `json:"username"`
	119	User struct {
	120	Username string `json:"username"`
	121	} `json:"user"`
	122	}
	123	_ = json.Unmarshal(n.Attrs, &a)
	124	name := a.Username
	125	if name == "" {
	126	name = a.User.Username
	127	}
	128	if name != "" {
	129	b.WriteString(`<a href="`)
	130	b.WriteString(esc(URLBuilder(host, "group_user", "?type=about&q=", name)))
	131	b.WriteString(`">@`)
	132	b.WriteString(esc(name))
	133	b.WriteString("</a>")
	134	}
	135	default:
	136	children()
	137	}
	138	}
	139
	140	// docMarkTags maps the inline marks to the tags that render them. textStyle
	141	// carries colour and font choices, which the instance's own stylesheet
	142	// decides, so it is not listed.
	143	var docMarkTags = map[string]string{
	144	"bold": "b", "strong": "b",
	145	"italic": "i", "em": "i",
	146	"underline": "u",
	147	"strike": "s", "strikethrough": "s",
	148	"code": "code",
	149	}
	150
	151	func renderDocText(b *strings.Builder, _ string, n docNode) {
	152	var opening strings.Builder
	153	var closing []string // pushed in mark order, emitted in reverse
	154	for _, m := range n.Marks {
	155	if m.Type == "link" && m.Attrs.Href != "" {
	156	opening.WriteString(`<a target="_blank" href="`)
	157	opening.WriteString(esc(deleteTrackingFromURL(m.Attrs.Href)))
	158	opening.WriteString(`">`)
	159	closing = append(closing, "</a>")
	160	continue
	161	}
	162	if tag, ok := docMarkTags[m.Type]; ok {
	163	opening.WriteString("<" + tag + ">")
	164	closing = append(closing, "</"+tag+">")
	165	}
	166	}
	167	b.WriteString(opening.String())
	168	b.WriteString(esc(n.Text))
	169	for _, c := range slices.Backward(closing) {
	170	b.WriteString(c)
	171	}
	172	}
	173
	174	// renderDocEmote renders an emoticon through the instance's emote route. An
	175	// official emote names its image; when it does not, the emote code without
	176	// its colons is the best guess at the file name.
	177	func renderDocEmote(b *strings.Builder, host string, attrs json.RawMessage) {
	178	var a struct {
	179	Code string `json:"data-emote"`
	180	Src string `json:"src"`
	181	Title string `json:"title"`
	182	}
	183	_ = json.Unmarshal(attrs, &a)
	184	src := emoticonURL(host, a.Src)
	185	if src == "" {
	186	if name := strings.Trim(a.Code, ":"); name != "" {
	187	src = URLBuilder(host, "media", "emojitar", name, "?type=e")
	188	}
	189	}
	190	if src == "" {
	191	return
	192	}
	193	label := a.Title
	194	if label == "" {
	195	label = a.Code
	196	}
	197	b.WriteString(`<img src="`)
	198	b.WriteString(esc(src))
	199	b.WriteString(`" alt="`)
	200	b.WriteString(esc(label))
	201	b.WriteString(`" title="`)
	202	b.WriteString(esc(label))
	203	b.WriteString(`">`)
	204	}
	205
	206	// renderDocDeviation renders an embedded artwork as its thumbnail linking to
	207	// the post on this instance, or as a text link when it has no media.
	208	func renderDocDeviation(b *strings.Builder, host string, attrs json.RawMessage) {
	209	var a struct {
	210	Deviation devianter.Deviation `json:"deviation"`
	211	}
	212	if json.Unmarshal(attrs, &a) != nil {
	213	return
	214	}
	215	d := &a.Deviation
	216	if !VisibleDeviation(d) {
	217	return
	218	}
	219	link := ConvertDeviantArtURLToSkunkyArt(host, d.Url)
	220	label := esc(d.Author.Username + " - " + d.Title)
	221	img := ParseMedia(host, d.Media, 320)
	222	if link != "" {
	223	b.WriteString(`<a href="`)
	224	b.WriteString(esc(link))
	225	b.WriteString(`">`)
	226	}
	227	if img != "" {
	228	b.WriteString(`<img width="50%" src="`)
	229	b.WriteString(esc(img))
	230	b.WriteString(`" alt="`)
	231	b.WriteString(label)
	232	b.WriteString(`" title="`)
	233	b.WriteString(label)
	234	b.WriteString(`">`)
	235	} else {
	236	b.WriteString(label)
	237	}
	238	if link != "" {
	239	b.WriteString("</a>")
	240	}
	241	}

app/document_test.go added +122

		@@ -0,0 +1,122 @@
	1	package app
	2
	3	import (
	4	"strings"
	5	"testing"
	6
	7	"github.com/krazywarez/devianter"
	8	)
	9
	10	// docText wraps a document body the way DeviantArt stores it.
	11	func docText(body string) devianter.Text {
	12	var t devianter.Text
	13	t.Html.Markup = `{"version":1,"document":{"type":"doc","content":[` + body + `]},"features":[]}`
	14	return t
	15	}
	16
	17	func withDocConfig(t *testing.T) {
	18	t.Helper()
	19	proxy, uri, nsfw := CFG.Proxy, CFG.URI, CFG.Nsfw
	20	CFG.Proxy, CFG.URI, CFG.Nsfw = true, "/", true
	21	t.Cleanup(func() { CFG.Proxy, CFG.URI, CFG.Nsfw = proxy, uri, nsfw })
	22	}
	23
	24	// TestDocumentParagraphsMarksAndBreaks uses the shape seen on live
	25	// descriptions: paragraphs of text with link, underline and italic marks.
	26	func TestDocumentParagraphsMarksAndBreaks(t *testing.T) {
	27	withDocConfig(t)
	28	out := ParseDescription("http://localhost", docText(`
	29	{"type":"paragraph","attrs":{"textAlign":"center"},"content":[
	30	{"type":"text","text":"\| "},
	31	{"type":"text","marks":[{"type":"link","attrs":{"href":"https://www.deviantart.com/users/outgoing?https://www.patreon.com/x","target":"_blank"}},{"type":"underline"}],"text":"PATREON"},
	32	{"type":"hardBreak"},
	33	{"type":"text","marks":[{"type":"italic"}],"text":"Finished <YCH> for "},
	34	{"type":"text","marks":[{"type":"textStyle"}],"text":"plain"}
	35	]}`))
	36	for _, want := range []string{
	37	`<p>\| <a target="_blank" href="https://www.patreon.com/x"><u>PATREON</u></a><br><i>Finished <YCH> for </i>plain</p>`,
	38	} {
	39	if !strings.Contains(out, want) {
	40	t.Errorf("output lacks %q:\n%s", want, out)
	41	}
	42	}
	43	}
	44
	45	func TestDocumentBlocksAndLists(t *testing.T) {
	46	withDocConfig(t)
	47	out := ParseDescription("http://localhost", docText(`
	48	{"type":"heading","attrs":{"level":1},"content":[{"type":"text","text":"Title"}]},
	49	{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold"}],"text":"one"}]}]}]},
	50	{"type":"orderedList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"text","text":"two"}]}]}]},
	51	{"type":"blockquote","content":[{"type":"paragraph","content":[{"type":"text","text":"quoted"}]}]},
	52	{"type":"codeBlock","content":[{"type":"text","text":"x < y"}]},
	53	{"type":"horizontalRule"},
	54	{"type":"mystery","content":[{"type":"text","text":"still shown"}]},
	55	{"type":"mysteryLeaf","attrs":{"x":1}}`))
	56	for _, want := range []string{"<h2>Title</h2>", "<ul><li><p><b>one</b></p></li></ul>", "<ol><li><p>two</p></li></ol>", "<blockquote><p>quoted</p></blockquote>", "<pre>x < y</pre>", "<hr>", "still shown"} {
	57	if !strings.Contains(out, want) {
	58	t.Errorf("output lacks %q:\n%s", want, out)
	59	}
	60	}
	61	}
	62
	63	// TestDocumentEmotes is the new-format half of #6: an official emote with a
	64	// source image maps to the emote route by file name, one without a source
	65	// falls back to its code.
	66	func TestDocumentEmotes(t *testing.T) {
	67	withDocConfig(t)
	68	out := ParseDescription("http://localhost", docText(`
	69	{"type":"paragraph","content":[
	70	{"type":"da-emote","attrs":{"data-emote":":star:","data-type":"official","src":"https://e.deviantart.net/emoticons/s/star_full.gif","width":17,"height":16,"url":null,"title":null}},
	71	{"type":"da-emote","attrs":{"data-emote":":love:","data-type":"official","src":null,"width":null,"height":null,"url":null,"title":null}},
	72	{"type":"da-emote","attrs":{"data-emote":"","data-type":"custom","src":null}}
	73	]}`))
	74	for _, want := range []string{`src="http://localhost/media/emojitar/star_full?type=e" alt=":star:"`, `src="http://localhost/media/emojitar/love?type=e" alt=":love:"`} {
	75	if !strings.Contains(out, want) {
	76	t.Errorf("output lacks %q:\n%s", want, out)
	77	}
	78	}
	79	if strings.Count(out, "<img") != 2 {
	80	t.Errorf("want 2 images (the nameless emote renders nothing):\n%s", out)
	81	}
	82	}
	83
	84	func TestDocumentEmbeddedDeviationAndGif(t *testing.T) {
	85	withDocConfig(t)
	86	out := ParseDescription("http://localhost", docText(`
	87	{"type":"paragraph","content":[
	88	{"type":"da-deviation-thumb","attrs":{"cropping":"fill","deviation":{"deviationId":1276762071,"url":"https://www.deviantart.com/ashiori-chan/art/CLOSED-YCH-Portrait-auction-1276762071","title":"[CLOSED] YCH","author":{"username":"AShiori-chan"},"media":{"baseUri":"https://images-wixmp-abc.wixmp.com/f/u/x.png","prettyName":"x_by_y","token":["tok.en.sig"],"types":[{"t":"fullview","h":1920,"w":1280}]}}}},
	89	{"type":"da-gif","attrs":{"url":"https://media4.giphy.com/media/x/giphy.mp4","width":480,"height":362}}
	90	]}`))
	91	for _, want := range []string{`<a href="http://localhost/post/ashiori-chan/CLOSED-YCH-Portrait-auction-1276762071">`, `src="http://localhost/media/file/abc/`, `alt="AShiori-chan - [CLOSED] YCH"`, `<a target="_blank" href="https://media4.giphy.com/media/x/giphy.mp4">[GIF]</a>`} {
	92	if !strings.Contains(out, want) {
	93	t.Errorf("output lacks %q:\n%s", want, out)
	94	}
	95	}
	96	}
	97
	98	func TestDocumentHidesEmbeddedNSFWWhenDisabled(t *testing.T) {
	99	withDocConfig(t)
	100	CFG.Nsfw = false
	101	out := ParseDescription("http://localhost", docText(`
	102	{"type":"paragraph","content":[{"type":"da-deviation","attrs":{"deviation":{"deviationId":1,"isMature":true,"url":"https://www.deviantart.com/a/art/b-1","title":"secret","author":{"username":"a"}}}}]}`))
	103	if strings.Contains(out, "secret") {
	104	t.Errorf("mature embed shown with nsfw off:\n%s", out)
	105	}
	106	}
	107
	108	// TestDraftJSAndHTMLStillDispatch pins that the two older formats still reach
	109	// their own parsers.
	110	func TestDraftJSAndHTMLStillDispatch(t *testing.T) {
	111	withDocConfig(t)
	112	var draft devianter.Text
	113	draft.Html.Markup = `{"blocks":[{"text":"draft text","type":"unstyled","inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}`
	114	if out := ParseDescription("http://localhost", draft); !strings.Contains(out, "draft text") {
	115	t.Errorf("Draft.js description not rendered:\n%s", out)
	116	}
	117	var legacy devianter.Text
	118	legacy.Html.Markup = `plain <b>html</b>`
	119	if out := ParseDescription("http://localhost", legacy); !strings.Contains(out, "<b>html</b>") {
	120	t.Errorf("HTML description not rendered:\n%s", out)
	121	}
	122	}

app/parsers.go +9 −7

		@@ -276,8 +276,15 @@ type text struct {
276	276	// rewriting embedded links and artwork references to point at this instance.
277	277	// host is the request's scheme and host, as taken by URLBuilder.
278	278	//
279		// TODO: rewrite this whole mess.
	279	// Three formats arrive: the current editor's document JSON (a "document"
	280	// key), handled by renderDocument; the older Draft.js JSON ("blocks" and
	281	// "entityMap"), handled below and kept for the posts that still carry it; and
	282	// plain HTML markup from before either.
280	283	func ParseDescription(host string, dscr devianter.Text) string {
	284	if doc, ok := parseDocument(dscr.Html.Markup); ok {
	285	return renderDocument(host, doc)
	286	}
	287
281	288	var parsedDescription strings.Builder
282	289	TagBuilder := func(content string, tags ...string) string {
283	290	l := len(tags)
		@@ -296,12 +303,7 @@ func ParseDescription(host string, dscr devianter.Text) string {
296	303	}
297	304	return content
298	305	}
299		DeleteTrackingFromURL := func(url string) string {
300		if len(url) > 42 && url[:42] == "https://www.deviantart.com/users/outgoing?" {
301		url = url[42:]
302		}
303		return url
304		}
	306	DeleteTrackingFromURL := deleteTrackingFromURL
305	307
306	308	if description, dl := dscr.Html.Markup, len(dscr.Html.Markup); dl != 0 &&
307	309	description[0] == '{' &&

Commit 5adf3557d1

Commit `5adf3557d1`