Commit `9c6ad663cf`

9c6ad663cf5901669aaed59cb16482e0f716a313

parent: 971bdb3e80

Unregistered key

cmc <hello@cleberg.net> · 2026-07-15 23:01 UTC
committer: <noreply@github.com>

fix: extract text from tiptap bodies, and read the right description field (#4)

Checking the library against the live API disproved what its comments
claimed. Comment and Description were handing back raw JSON blobs, and
the parser this package has always carried could not have prevented it.

DeviantArt no longer stores rich text as Draft.js. It uses tiptap:

  Draft.js, what we parse:  {"blocks":[{"text":"..."}]}
  tiptap, what DA sends:    {"version":1,"document":{"type":"doc",...}}

Across 34 live comments: 33 tiptap, 0 Draft.js, and 33 whose .Comment was
the raw JSON. flattenMarkup now walks the tiptap tree — text nodes joined
per block, hardBreak as a newline, entities decoded — and keeps the
Draft.js path for old bodies that still carry it. Nodes with no text of
their own (images, galleries, emotes) drop out, as they always did.

The description had a second, larger bug behind that one. It lives in
Extended.DescriptionText, not TextContent, which GetDeviation read: of 24
deviations sampled, 23 populated the former and 1 the latter. So
Post.Description was empty for nearly every deviation, and the txt[1]
guard removed in 3dad0ea was never what stood in the way. Prefer
Extended.DescriptionText and fall back, since either may be the one set.

Verified against the live API rather than by reading: comments still raw
JSON went 33/34 to 0/34, and descriptions that were "" now return prose.

Also corrects the doc comments that asserted the Draft.js story on
pkg.go.dev, and notes that flattening drops images, links, and emotes.

Layout: unified · split

comments.go +86 −22

		@@ -2,6 +2,7 @@ package devianter
2	2
3	3	import (
4	4	"encoding/json"
	5	"html"
5	6	"net/url"
6	7	"strconv"
7	8	"strings"
		@@ -25,6 +26,10 @@ type Thread struct {
25	26
26	27	// Comment is the comment's plain text, which [GetComments] extracts from
27	28	// TextContent. Prefer it; TextContent is the unprocessed original.
	29	//
	30	// Text is all it holds: a comment is a rich document, and its images,
	31	// emotes, mentions, and link targets are dropped in the flattening. Read
	32	// TextContent for those.
28	33	Comment string
29	34
30	35	TextContent Text
		@@ -71,46 +76,105 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment
71	76	cursor = cmmts.Cursor
72	77
73	78	for i := 0; i < len(cmmts.Thread); i++ {
74		cmmts.Thread[i].Comment = flattenComment(cmmts.Thread[i].TextContent.Html.Markup)
	79	cmmts.Thread[i].Comment = flattenMarkup(cmmts.Thread[i].TextContent.Html.Markup)
75	80	}
76	81	}
77	82
78	83	return
79	84	}
80	85
81		// flattenComment renders a body of user-written markup as plain text, be it a
82		// comment or a deviation's description. Bodies are JSON inside JSON: newer ones
83		// are a Draft.js document encoded into the markup string, older ones are plain
84		// HTML, which passes through unchanged. Markup that does not parse, and empty
85		// markup, also pass through.
	86	// flattenMarkup renders a body of user-written markup as plain text, be it a
	87	// comment or a deviation's description. Bodies are JSON inside JSON, and
	88	// DeviantArt still serves all three formats it has used over the years:
86	89	//
87		// A Draft.js document is a list of blocks, which are block-level elements
88		// (paragraphs, list items); they are joined with newlines, one block per line.
89		func flattenComment(m string) string {
	90	// - tiptap, current: {"version":1,"document":{"type":"doc","content":[...]}}
	91	// - Draft.js, legacy: {"blocks":[{"text":"..."}]}
	92	// - plain HTML, oldest, which passes through unchanged
	93	//
	94	// Block-level elements (paragraphs, headings) are joined with newlines, one per
	95	// line, and a hard break inside one becomes a newline too. HTML entities in the
	96	// text are decoded, so a body reads as ’ on the wire but an apostrophe
	97	// here. Markup matching no known format passes through unchanged rather than
	98	// being replaced by an empty string.
	99	func flattenMarkup(m string) string {
90	100	l := len(m)
91	101	if l == 0 \|\| m[0] != '{' \|\| m[l-1] != '}' {
92	102	return m
93	103	}
94	104
	105	if text, ok := flattenTiptap(m); ok {
	106	return text
	107	}
	108	if text, ok := flattenDraftJS(m); ok {
	109	return text
	110	}
	111	return m
	112	}
	113
	114	// tiptapNode is one node of a tiptap (ProseMirror) document tree.
	115	//
	116	// The document's "version" field is deliberately not modelled: DeviantArt sends
	117	// it as a number on some bodies and a string on others, so any typed field for
	118	// it fails to unmarshal on half of them.
	119	type tiptapNode struct {
	120	Type string `json:"type"`
	121	Text string `json:"text"`
	122	Content []tiptapNode `json:"content"`
	123	}
	124
	125	// flattenTiptap renders a tiptap document, reporting false if the markup is not
	126	// one.
	127	func flattenTiptap(m string) (string, bool) {
	128	var doc struct {
	129	Document tiptapNode `json:"document"`
	130	}
	131	if json.Unmarshal([]byte(m), &doc) != nil \|\| doc.Document.Type != "doc" {
	132	return "", false
	133	}
	134
	135	lines := make([]string, 0, len(doc.Document.Content))
	136	for _, block := range doc.Document.Content {
	137	var b strings.Builder
	138	writeTiptapText(block, &b)
	139	lines = append(lines, b.String())
	140	}
	141
	142	return html.UnescapeString(strings.Join(lines, "\n")), true
	143	}
	144
	145	// writeTiptapText collects the text of a node and everything nested inside it.
	146	// Nodes carrying no text of their own — images, galleries, emotes — contribute
	147	// nothing.
	148	func writeTiptapText(n tiptapNode, b *strings.Builder) {
	149	switch n.Type {
	150	case "text":
	151	b.WriteString(n.Text)
	152	return
	153	case "hardBreak":
	154	b.WriteString("\n")
	155	return
	156	}
	157	for _, c := range n.Content {
	158	writeTiptapText(c, b)
	159	}
	160	}
	161
	162	// flattenDraftJS renders a legacy Draft.js document, reporting false if the
	163	// markup is not one.
	164	func flattenDraftJS(m string) (string, bool) {
95	165	var content struct {
96	166	Blocks []struct {
97	167	Text string
98	168	}
99	169	}
100
101		e := json.Unmarshal([]byte(m), &content)
102		try(e)
103
104		if len(content.Blocks) == 0 {
105		return m
	170	if json.Unmarshal([]byte(m), &content) != nil \|\| len(content.Blocks) == 0 {
	171	return "", false
106	172	}
107	173
108		var b strings.Builder
109		for i, a := range content.Blocks {
110		if i > 0 {
111		b.WriteString("\n")
112		}
113		b.WriteString(a.Text)
	174	lines := make([]string, 0, len(content.Blocks))
	175	for _, blk := range content.Blocks {
	176	lines = append(lines, blk.Text)
114	177	}
115		return b.String()
	178
	179	return html.UnescapeString(strings.Join(lines, "\n")), true
116	180	}

comments_test.go +117 −22

		@@ -2,50 +2,145 @@ package devianter
2	2
3	3	import "testing"
4	4
5		// Regression: flattenComment's shape check used to read m[0] and m[len(m)-1]
6		// without a length check, so a comment with an empty markup body panicked with
7		// index out of range and killed the caller's process.
8		func TestFlattenCommentEmptyMarkup(t *testing.T) {
9		if got := flattenComment(""); got != "" {
	5	// Regression: the shape check used to read m[0] and m[len(m)-1] without a length
	6	// check, so a comment with an empty markup body panicked with index out of range
	7	// and killed the caller's process.
	8	func TestFlattenMarkupEmptyMarkup(t *testing.T) {
	9	if got := flattenMarkup(""); got != "" {
10	10	t.Errorf("want an empty comment for empty markup, got %q", got)
11	11	}
12	12	}
13	13
14		func TestFlattenComment(t *testing.T) {
15		// A newer, Draft.js-encoded body is flattened to its text.
16		draft := `{"blocks":[{"text":"hello there"}]}`
17		if got := flattenComment(draft); got != "hello there" {
18		t.Errorf("want the Draft.js block text, got %q", got)
	14	// tiptap is what DeviantArt actually serves today; the fixtures below are shaped
	15	// like responses captured from the live API.
	16	func TestFlattenMarkupTiptap(t *testing.T) {
	17	one := `{"version":1,"document":{"type":"doc","content":[` +
	18	`{"type":"paragraph","attrs":{"textAlign":"left"},"content":[` +
	19	`{"type":"text","text":"Nice artwork!"}]}]}}`
	20	if got := flattenMarkup(one); got != "Nice artwork!" {
	21	t.Errorf("want the paragraph's text, got %q", got)
19	22	}
20	23
21		// An older, plain-HTML body passes through untouched.
22		html := "<b>hello</b> there"
23		if got := flattenComment(html); got != html {
24		t.Errorf("want plain HTML passed through, got %q", got)
	24	// Regression: DeviantArt sends "version" as a number on some bodies and a
	25	// string on others. Modelling it as either type fails to unmarshal half of
	26	// them, so it must not be modelled at all.
	27	stringVersion := `{"version":"1","document":{"type":"doc","content":[` +
	28	`{"type":"paragraph","content":[{"type":"text","text":"hello"}]}]}}`
	29	if got := flattenMarkup(stringVersion); got != "hello" {
	30	t.Errorf(`want a string "version" handled the same as a numeric one, got %q`, got)
	31	}
	32
	33	// Each block is its own line.
	34	two := `{"version":1,"document":{"type":"doc","content":[` +
	35	`{"type":"paragraph","content":[{"type":"text","text":"first"}]},` +
	36	`{"type":"paragraph","content":[{"type":"text","text":"second"}]}]}}`
	37	if got, want := flattenMarkup(two), "first\nsecond"; got != want {
	38	t.Errorf("want one line per block:\n got %q\nwant %q", got, want)
	39	}
	40
	41	// An empty paragraph is a blank line, not something to skip.
	42	blank := `{"version":1,"document":{"type":"doc","content":[` +
	43	`{"type":"paragraph","content":[{"type":"text","text":"a"}]},` +
	44	`{"type":"paragraph"},` +
	45	`{"type":"paragraph","content":[{"type":"text","text":"b"}]}]}}`
	46	if got, want := flattenMarkup(blank), "a\n\nb"; got != want {
	47	t.Errorf("want an empty paragraph preserved as a blank line:\n got %q\nwant %q", got, want)
	48	}
	49
	50	// A hard break is a newline within its block.
	51	brk := `{"version":1,"document":{"type":"doc","content":[` +
	52	`{"type":"paragraph","content":[{"type":"text","text":"up"},` +
	53	`{"type":"hardBreak"},{"type":"text","text":"down"}]}]}}`
	54	if got, want := flattenMarkup(brk), "up\ndown"; got != want {
	55	t.Errorf("want a hardBreak as a newline:\n got %q\nwant %q", got, want)
	56	}
	57
	58	// Marked-up runs are separate text nodes and must be concatenated, not
	59	// separated.
	60	marks := `{"version":1,"document":{"type":"doc","content":[` +
	61	`{"type":"paragraph","content":[` +
	62	`{"type":"text","text":"plain "},` +
	63	`{"type":"text","marks":[{"type":"bold"}],"text":"bold"},` +
	64	`{"type":"text","text":" tail"}]}]}}`
	65	if got, want := flattenMarkup(marks), "plain bold tail"; got != want {
	66	t.Errorf("want marked runs concatenated:\n got %q\nwant %q", got, want)
	67	}
	68
	69	// Text carrying a link mark still surfaces; the href does not.
	70	link := `{"version":1,"document":{"type":"doc","content":[` +
	71	`{"type":"paragraph","content":[{"type":"text","marks":[` +
	72	`{"type":"link","attrs":{"href":"https://example.com"}}],"text":"click here"}]}]}}`
	73	if got, want := flattenMarkup(link), "click here"; got != want {
	74	t.Errorf("want the link text without the href:\n got %q\nwant %q", got, want)
	75	}
	76
	77	// Headings are blocks like any other.
	78	heading := `{"version":1,"document":{"type":"doc","content":[` +
	79	`{"type":"heading","attrs":{"level":2},"content":[{"type":"text","text":"Title"}]},` +
	80	`{"type":"paragraph","content":[{"type":"text","text":"body"}]}]}}`
	81	if got, want := flattenMarkup(heading), "Title\nbody"; got != want {
	82	t.Errorf("want a heading flattened as a block:\n got %q\nwant %q", got, want)
	83	}
	84
	85	// Bodies carry HTML entities on the wire; plain text should not.
	86	entity := `{"version":1,"document":{"type":"doc","content":[` +
	87	`{"type":"paragraph","content":[{"type":"text","text":"I’d love & more"}]}]}}`
	88	if got, want := flattenMarkup(entity), "I’d love & more"; got != want {
	89	t.Errorf("want HTML entities decoded:\n got %q\nwant %q", got, want)
	90	}
	91
	92	// A node carrying no text of its own contributes nothing.
	93	emote := `{"version":1,"document":{"type":"doc","content":[` +
	94	`{"type":"paragraph","content":[{"type":"text","text":"hi "},` +
	95	`{"type":"da-emote","attrs":{"name":":happy:"}}]}]}}`
	96	if got, want := flattenMarkup(emote), "hi "; got != want {
	97	t.Errorf("want a textless node to contribute nothing:\n got %q\nwant %q", got, want)
	98	}
	99	}
	100
	101	// Draft.js is legacy but still served on old bodies, so the path stays.
	102	func TestFlattenMarkupDraftJS(t *testing.T) {
	103	draft := `{"blocks":[{"text":"hello there"}]}`
	104	if got := flattenMarkup(draft); got != "hello there" {
	105	t.Errorf("want the Draft.js block text, got %q", got)
25	106	}
26	107
27	108	// Regression: the block loop used to assign rather than accumulate, so every
28	109	// block but the last was silently dropped and a multi-paragraph comment came
29	110	// back as its closing line only.
30	111	multi := `{"blocks":[{"text":"first"},{"text":"second"},{"text":"third"}]}`
31		if got, want := flattenComment(multi), "first\nsecond\nthird"; got != want {
	112	if got, want := flattenMarkup(multi), "first\nsecond\nthird"; got != want {
32	113	t.Errorf("want every block, one per line:\n got %q\nwant %q", got, want)
33	114	}
34	115
35		// An empty block is a blank line in the comment, not something to skip.
	116	// An empty block is a blank line, not something to skip.
36	117	blank := `{"blocks":[{"text":"first"},{"text":""},{"text":"third"}]}`
37		if got, want := flattenComment(blank), "first\n\nthird"; got != want {
	118	if got, want := flattenMarkup(blank), "first\n\nthird"; got != want {
38	119	t.Errorf("want an empty block preserved as a blank line:\n got %q\nwant %q", got, want)
39	120	}
	121	}
	122
	123	func TestFlattenMarkupPassthrough(t *testing.T) {
	124	// An older, plain-HTML body passes through untouched.
	125	html := "<b>hello</b> there"
	126	if got := flattenMarkup(html); got != html {
	127	t.Errorf("want plain HTML passed through, got %q", got)
	128	}
	129
	130	// Brace-shaped markup in no known format falls back to itself rather than to
	131	// an empty string.
	132	if got := flattenMarkup("{}"); got != "{}" {
	133	t.Errorf("want the original markup when nothing parses, got %q", got)
	134	}
40	135
41		// Brace-shaped markup that isn't a Draft.js document falls back to itself
42		// rather than to an empty string.
43		if got := flattenComment("{}"); got != "{}" {
44		t.Errorf("want the original markup when there are no blocks, got %q", got)
	136	// Well-formed JSON that is neither format is still not silently eaten.
	137	other := `{"something":"else"}`
	138	if got := flattenMarkup(other); got != other {
	139	t.Errorf("want unrecognised JSON passed through, got %q", got)
45	140	}
46	141
47	142	// A single brace satisfies neither end of the shape check.
48		if got := flattenComment("{"); got != "{" {
	143	if got := flattenMarkup("{"); got != "{" {
49	144	t.Errorf("want a lone brace passed through, got %q", got)
50	145	}
51	146	}

deviantion.go +19 −6

		@@ -79,9 +79,14 @@ type Media struct {
79	79	}
80	80
81	81	// Text is a block of user-written text — a description, a comment, a group's
82		// about page. Markup holds either HTML or a JSON-encoded Draft.js document,
83		// distinguished by Type; the functions that return a Text generally extract the
84		// plain text into a neighbouring field, which is easier to use.
	82	// about page.
	83	//
	84	// Markup is a rich document rather than a string of prose, in whichever format
	85	// DeviantArt stored it: tiptap JSON on anything recent (Type is "tiptap"),
	86	// Draft.js JSON on older bodies, or plain HTML on the oldest. The functions
	87	// returning a Text generally flatten it to plain text in a neighbouring field,
	88	// which is what most callers want; read Markup itself for the formatting,
	89	// images, and links that flattening discards.
85	90	type Text struct {
86	91	Excerpt string
87	92	Html struct {
		@@ -91,8 +96,9 @@ type Text struct {
91	96
92	97	// Post is a deviation together with its comment metadata, as returned by
93	98	// [GetDeviation]. IMG and Description are conveniences that GetDeviation derives
94		// from the Deviation, so callers need not assemble a URL or decode Draft.js
95		// markup themselves.
	99	// from the Deviation, so callers need not assemble a URL or flatten a rich-text
	100	// document themselves. Description is empty for the many deviations that have
	101	// none.
96	102	//
97	103	// Comments holds only a total and a cursor. To retrieve the comments, pass them
98	104	// to [GetComments] with type 1.
		@@ -175,7 +181,14 @@ func GetDeviation(id string, user string) (st Post, err Error) {
175	181
176	182	st.IMG, _ = UrlFromMedia(st.Deviation.Media)
177	183
178		st.Description = flattenComment(st.Deviation.TextContent.Html.Markup)
	184	// The description lives in Extended.DescriptionText on the great majority of
	185	// deviations; TextContent carries it on only a small minority. Prefer the
	186	// former and fall back, since either may be the populated one.
	187	desc := st.Deviation.Extended.DescriptionText.Html.Markup
	188	if desc == "" {
	189	desc = st.Deviation.TextContent.Html.Markup
	190	}
	191	st.Description = flattenMarkup(desc)
179	192
180	193	return
181	194	}

Commit 9c6ad663cf

Commit `9c6ad663cf`