Commit `9c6ad663cf`

9c6ad663cf5901669aaed59cb16482e0f716a313

parent: 971bdb3e80

Unregistered key

cmc <hello@cleberg.net> · 2026-07-15 23:01 UTC
committer: <noreply@github.com>

fix: extract text from tiptap bodies, and read the right description field (#4)

Checking the library against the live API disproved what its comments
claimed. Comment and Description were handing back raw JSON blobs, and
the parser this package has always carried could not have prevented it.

DeviantArt no longer stores rich text as Draft.js. It uses tiptap:

  Draft.js, what we parse:  {"blocks":[{"text":"..."}]}
  tiptap, what DA sends:    {"version":1,"document":{"type":"doc",...}}

Across 34 live comments: 33 tiptap, 0 Draft.js, and 33 whose .Comment was
the raw JSON. flattenMarkup now walks the tiptap tree — text nodes joined
per block, hardBreak as a newline, entities decoded — and keeps the
Draft.js path for old bodies that still carry it. Nodes with no text of
their own (images, galleries, emotes) drop out, as they always did.

The description had a second, larger bug behind that one. It lives in
Extended.DescriptionText, not TextContent, which GetDeviation read: of 24
deviations sampled, 23 populated the former and 1 the latter. So
Post.Description was empty for nearly every deviation, and the txt[1]
guard removed in 3dad0ea was never what stood in the way. Prefer
Extended.DescriptionText and fall back, since either may be the one set.

Verified against the live API rather than by reading: comments still raw
JSON went 33/34 to 0/34, and descriptions that were "" now return prose.

Also corrects the doc comments that asserted the Draft.js story on
pkg.go.dev, and notes that flattening drops images, links, and emotes.

Layout: unified · split

comments.go +86 −22

	@@ -2,6 +2,7 @@ package devianter
2		2
3	import (	3	import (
4	"encoding/json"	4	"encoding/json"
		5	"html"
5	"net/url"	6	"net/url"
6	"strconv"	7	"strconv"
7	"strings"	8	"strings"
	@@ -25,6 +26,10 @@ type Thread struct {
25		26
26	// Comment is the comment's plain text, which [GetComments] extracts from	27	// Comment is the comment's plain text, which [GetComments] extracts from
27	// TextContent. Prefer it; TextContent is the unprocessed original.	28	// TextContent. Prefer it; TextContent is the unprocessed original.
		29	//
		30	// Text is all it holds: a comment is a rich document, and its images,
		31	// emotes, mentions, and link targets are dropped in the flattening. Read
		32	// TextContent for those.
28	Comment string	33	Comment string
29		34
30	TextContent Text	35	TextContent Text
	@@ -71,46 +76,105 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment
71	cursor = cmmts.Cursor	76	cursor = cmmts.Cursor
72		77
73	for i := 0; i < len(cmmts.Thread); i++ {	78	for i := 0; i < len(cmmts.Thread); i++ {
74	cmmts.Thread[i].Comment = flattenComment(cmmts.Thread[i].TextContent.Html.Markup)	79	cmmts.Thread[i].Comment = flattenMarkup(cmmts.Thread[i].TextContent.Html.Markup)
75	}	80	}
76	}	81	}
77		82
78	return	83	return
79	}	84	}
80		85
81	// flattenComment renders a body of user-written markup as plain text, be it a	86	// flattenMarkup renders a body of user-written markup as plain text, be it a
82	// comment or a deviation's description. Bodies are JSON inside JSON: newer ones	87	// comment or a deviation's description. Bodies are JSON inside JSON, and
83	// are a Draft.js document encoded into the markup string, older ones are plain	88	// DeviantArt still serves all three formats it has used over the years:
84	// HTML, which passes through unchanged. Markup that does not parse, and empty
85	// markup, also pass through.
86	//	89	//
87	// A Draft.js document is a list of blocks, which are block-level elements	90	// - tiptap, current: {"version":1,"document":{"type":"doc","content":[...]}}
88	// (paragraphs, list items); they are joined with newlines, one block per line.	91	// - Draft.js, legacy: {"blocks":[{"text":"..."}]}
89	func flattenComment(m string) string {	92	// - plain HTML, oldest, which passes through unchanged
		93	//
		94	// Block-level elements (paragraphs, headings) are joined with newlines, one per
		95	// line, and a hard break inside one becomes a newline too. HTML entities in the
		96	// text are decoded, so a body reads as ’ on the wire but an apostrophe
		97	// here. Markup matching no known format passes through unchanged rather than
		98	// being replaced by an empty string.
		99	func flattenMarkup(m string) string {
90	l := len(m)	100	l := len(m)
91	if l == 0 \|\| m[0] != '{' \|\| m[l-1] != '}' {	101	if l == 0 \|\| m[0] != '{' \|\| m[l-1] != '}' {
92	return m	102	return m
93	}	103	}
94		104
		105	if text, ok := flattenTiptap(m); ok {
		106	return text
		107	}
		108	if text, ok := flattenDraftJS(m); ok {
		109	return text
		110	}
		111	return m
		112	}
		113
		114	// tiptapNode is one node of a tiptap (ProseMirror) document tree.
		115	//
		116	// The document's "version" field is deliberately not modelled: DeviantArt sends
		117	// it as a number on some bodies and a string on others, so any typed field for
		118	// it fails to unmarshal on half of them.
		119	type tiptapNode struct {
		120	Type string `json:"type"`
		121	Text string `json:"text"`
		122	Content []tiptapNode `json:"content"`
		123	}
		124
		125	// flattenTiptap renders a tiptap document, reporting false if the markup is not
		126	// one.
		127	func flattenTiptap(m string) (string, bool) {
		128	var doc struct {
		129	Document tiptapNode `json:"document"`
		130	}
		131	if json.Unmarshal([]byte(m), &doc) != nil \|\| doc.Document.Type != "doc" {
		132	return "", false
		133	}
		134
		135	lines := make([]string, 0, len(doc.Document.Content))
		136	for _, block := range doc.Document.Content {
		137	var b strings.Builder
		138	writeTiptapText(block, &b)
		139	lines = append(lines, b.String())
		140	}
		141
		142	return html.UnescapeString(strings.Join(lines, "\n")), true
		143	}
		144
		145	// writeTiptapText collects the text of a node and everything nested inside it.
		146	// Nodes carrying no text of their own — images, galleries, emotes — contribute
		147	// nothing.
		148	func writeTiptapText(n tiptapNode, b *strings.Builder) {
		149	switch n.Type {
		150	case "text":
		151	b.WriteString(n.Text)
		152	return
		153	case "hardBreak":
		154	b.WriteString("\n")
		155	return
		156	}
		157	for _, c := range n.Content {
		158	writeTiptapText(c, b)
		159	}
		160	}
		161
		162	// flattenDraftJS renders a legacy Draft.js document, reporting false if the
		163	// markup is not one.
		164	func flattenDraftJS(m string) (string, bool) {
95	var content struct {	165	var content struct {
96	Blocks []struct {	166	Blocks []struct {
97	Text string	167	Text string
98	}	168	}
99	}	169	}
100		170	if json.Unmarshal([]byte(m), &content) != nil \|\| len(content.Blocks) == 0 {
101	e := json.Unmarshal([]byte(m), &content)	171	return "", false
102	try(e)
103
104	if len(content.Blocks) == 0 {
105	return m
106	}	172	}
107		173
108	var b strings.Builder	174	lines := make([]string, 0, len(content.Blocks))
109	for i, a := range content.Blocks {	175	for _, blk := range content.Blocks {
110	if i > 0 {	176	lines = append(lines, blk.Text)
111	b.WriteString("\n")
112	}
113	b.WriteString(a.Text)
114	}	177	}
115	return b.String()	178
		179	return html.UnescapeString(strings.Join(lines, "\n")), true
116	}	180	}

comments_test.go +117 −22

	@@ -2,50 +2,145 @@ package devianter
2		2
3	import "testing"	3	import "testing"
4		4
5	// Regression: flattenComment's shape check used to read m[0] and m[len(m)-1]	5	// Regression: the shape check used to read m[0] and m[len(m)-1] without a length
6	// without a length check, so a comment with an empty markup body panicked with	6	// check, so a comment with an empty markup body panicked with index out of range
7	// index out of range and killed the caller's process.	7	// and killed the caller's process.
8	func TestFlattenCommentEmptyMarkup(t *testing.T) {	8	func TestFlattenMarkupEmptyMarkup(t *testing.T) {
9	if got := flattenComment(""); got != "" {	9	if got := flattenMarkup(""); got != "" {
10	t.Errorf("want an empty comment for empty markup, got %q", got)	10	t.Errorf("want an empty comment for empty markup, got %q", got)
11	}	11	}
12	}	12	}
13		13
14	func TestFlattenComment(t *testing.T) {	14	// tiptap is what DeviantArt actually serves today; the fixtures below are shaped
15	// A newer, Draft.js-encoded body is flattened to its text.	15	// like responses captured from the live API.
16	draft := `{"blocks":[{"text":"hello there"}]}`	16	func TestFlattenMarkupTiptap(t *testing.T) {
17	if got := flattenComment(draft); got != "hello there" {	17	one := `{"version":1,"document":{"type":"doc","content":[` +
18	t.Errorf("want the Draft.js block text, got %q", got)	18	`{"type":"paragraph","attrs":{"textAlign":"left"},"content":[` +
		19	`{"type":"text","text":"Nice artwork!"}]}]}}`
		20	if got := flattenMarkup(one); got != "Nice artwork!" {
		21	t.Errorf("want the paragraph's text, got %q", got)
19	}	22	}
20		23
21	// An older, plain-HTML body passes through untouched.	24	// Regression: DeviantArt sends "version" as a number on some bodies and a
22	html := "<b>hello</b> there"	25	// string on others. Modelling it as either type fails to unmarshal half of
23	if got := flattenComment(html); got != html {	26	// them, so it must not be modelled at all.
24	t.Errorf("want plain HTML passed through, got %q", got)	27	stringVersion := `{"version":"1","document":{"type":"doc","content":[` +
		28	`{"type":"paragraph","content":[{"type":"text","text":"hello"}]}]}}`
		29	if got := flattenMarkup(stringVersion); got != "hello" {
		30	t.Errorf(`want a string "version" handled the same as a numeric one, got %q`, got)
		31	}
		32
		33	// Each block is its own line.
		34	two := `{"version":1,"document":{"type":"doc","content":[` +
		35	`{"type":"paragraph","content":[{"type":"text","text":"first"}]},` +
		36	`{"type":"paragraph","content":[{"type":"text","text":"second"}]}]}}`
		37	if got, want := flattenMarkup(two), "first\nsecond"; got != want {
		38	t.Errorf("want one line per block:\n got %q\nwant %q", got, want)
		39	}
		40
		41	// An empty paragraph is a blank line, not something to skip.
		42	blank := `{"version":1,"document":{"type":"doc","content":[` +
		43	`{"type":"paragraph","content":[{"type":"text","text":"a"}]},` +
		44	`{"type":"paragraph"},` +
		45	`{"type":"paragraph","content":[{"type":"text","text":"b"}]}]}}`
		46	if got, want := flattenMarkup(blank), "a\n\nb"; got != want {
		47	t.Errorf("want an empty paragraph preserved as a blank line:\n got %q\nwant %q", got, want)
		48	}
		49
		50	// A hard break is a newline within its block.
		51	brk := `{"version":1,"document":{"type":"doc","content":[` +
		52	`{"type":"paragraph","content":[{"type":"text","text":"up"},` +
		53	`{"type":"hardBreak"},{"type":"text","text":"down"}]}]}}`
		54	if got, want := flattenMarkup(brk), "up\ndown"; got != want {
		55	t.Errorf("want a hardBreak as a newline:\n got %q\nwant %q", got, want)
		56	}
		57
		58	// Marked-up runs are separate text nodes and must be concatenated, not
		59	// separated.
		60	marks := `{"version":1,"document":{"type":"doc","content":[` +
		61	`{"type":"paragraph","content":[` +
		62	`{"type":"text","text":"plain "},` +
		63	`{"type":"text","marks":[{"type":"bold"}],"text":"bold"},` +
		64	`{"type":"text","text":" tail"}]}]}}`
		65	if got, want := flattenMarkup(marks), "plain bold tail"; got != want {
		66	t.Errorf("want marked runs concatenated:\n got %q\nwant %q", got, want)
		67	}
		68
		69	// Text carrying a link mark still surfaces; the href does not.
		70	link := `{"version":1,"document":{"type":"doc","content":[` +
		71	`{"type":"paragraph","content":[{"type":"text","marks":[` +
		72	`{"type":"link","attrs":{"href":"https://example.com"}}],"text":"click here"}]}]}}`
		73	if got, want := flattenMarkup(link), "click here"; got != want {
		74	t.Errorf("want the link text without the href:\n got %q\nwant %q", got, want)
		75	}
		76
		77	// Headings are blocks like any other.
		78	heading := `{"version":1,"document":{"type":"doc","content":[` +
		79	`{"type":"heading","attrs":{"level":2},"content":[{"type":"text","text":"Title"}]},` +
		80	`{"type":"paragraph","content":[{"type":"text","text":"body"}]}]}}`
		81	if got, want := flattenMarkup(heading), "Title\nbody"; got != want {
		82	t.Errorf("want a heading flattened as a block:\n got %q\nwant %q", got, want)
		83	}
		84
		85	// Bodies carry HTML entities on the wire; plain text should not.
		86	entity := `{"version":1,"document":{"type":"doc","content":[` +
		87	`{"type":"paragraph","content":[{"type":"text","text":"I’d love & more"}]}]}}`
		88	if got, want := flattenMarkup(entity), "I’d love & more"; got != want {
		89	t.Errorf("want HTML entities decoded:\n got %q\nwant %q", got, want)
		90	}
		91
		92	// A node carrying no text of its own contributes nothing.
		93	emote := `{"version":1,"document":{"type":"doc","content":[` +
		94	`{"type":"paragraph","content":[{"type":"text","text":"hi "},` +
		95	`{"type":"da-emote","attrs":{"name":":happy:"}}]}]}}`
		96	if got, want := flattenMarkup(emote), "hi "; got != want {
		97	t.Errorf("want a textless node to contribute nothing:\n got %q\nwant %q", got, want)
		98	}
		99	}
		100
		101	// Draft.js is legacy but still served on old bodies, so the path stays.
		102	func TestFlattenMarkupDraftJS(t *testing.T) {
		103	draft := `{"blocks":[{"text":"hello there"}]}`
		104	if got := flattenMarkup(draft); got != "hello there" {
		105	t.Errorf("want the Draft.js block text, got %q", got)
25	}	106	}
26		107
27	// Regression: the block loop used to assign rather than accumulate, so every	108	// Regression: the block loop used to assign rather than accumulate, so every
28	// block but the last was silently dropped and a multi-paragraph comment came	109	// block but the last was silently dropped and a multi-paragraph comment came
29	// back as its closing line only.	110	// back as its closing line only.
30	multi := `{"blocks":[{"text":"first"},{"text":"second"},{"text":"third"}]}`	111	multi := `{"blocks":[{"text":"first"},{"text":"second"},{"text":"third"}]}`
31	if got, want := flattenComment(multi), "first\nsecond\nthird"; got != want {	112	if got, want := flattenMarkup(multi), "first\nsecond\nthird"; got != want {
32	t.Errorf("want every block, one per line:\n got %q\nwant %q", got, want)	113	t.Errorf("want every block, one per line:\n got %q\nwant %q", got, want)
33	}	114	}
34		115
35	// An empty block is a blank line in the comment, not something to skip.	116	// An empty block is a blank line, not something to skip.
36	blank := `{"blocks":[{"text":"first"},{"text":""},{"text":"third"}]}`	117	blank := `{"blocks":[{"text":"first"},{"text":""},{"text":"third"}]}`
37	if got, want := flattenComment(blank), "first\n\nthird"; got != want {	118	if got, want := flattenMarkup(blank), "first\n\nthird"; got != want {
38	t.Errorf("want an empty block preserved as a blank line:\n got %q\nwant %q", got, want)	119	t.Errorf("want an empty block preserved as a blank line:\n got %q\nwant %q", got, want)
39	}	120	}
		121	}
		122
		123	func TestFlattenMarkupPassthrough(t *testing.T) {
		124	// An older, plain-HTML body passes through untouched.
		125	html := "<b>hello</b> there"
		126	if got := flattenMarkup(html); got != html {
		127	t.Errorf("want plain HTML passed through, got %q", got)
		128	}
		129
		130	// Brace-shaped markup in no known format falls back to itself rather than to
		131	// an empty string.
		132	if got := flattenMarkup("{}"); got != "{}" {
		133	t.Errorf("want the original markup when nothing parses, got %q", got)
		134	}
40		135
41	// Brace-shaped markup that isn't a Draft.js document falls back to itself	136	// Well-formed JSON that is neither format is still not silently eaten.
42	// rather than to an empty string.	137	other := `{"something":"else"}`
43	if got := flattenComment("{}"); got != "{}" {	138	if got := flattenMarkup(other); got != other {
44	t.Errorf("want the original markup when there are no blocks, got %q", got)	139	t.Errorf("want unrecognised JSON passed through, got %q", got)
45	}	140	}
46		141
47	// A single brace satisfies neither end of the shape check.	142	// A single brace satisfies neither end of the shape check.
48	if got := flattenComment("{"); got != "{" {	143	if got := flattenMarkup("{"); got != "{" {
49	t.Errorf("want a lone brace passed through, got %q", got)	144	t.Errorf("want a lone brace passed through, got %q", got)
50	}	145	}
51	}	146	}

deviantion.go +19 −6

 }
 // Text is a block of user-written text — a description, a comment, a group's
-// about page. Markup holds either HTML or a JSON-encoded Draft.js document,
+// about page.
-// distinguished by Type; the functions that return a Text generally extract the
+//
-// plain text into a neighbouring field, which is easier to use.
+// Markup is a rich document rather than a string of prose, in whichever format
+// DeviantArt stored it: tiptap JSON on anything recent (Type is "tiptap"),
+// Draft.js JSON on older bodies, or plain HTML on the oldest. The functions
+// returning a Text generally flatten it to plain text in a neighbouring field,
+// which is what most callers want; read Markup itself for the formatting,
+// images, and links that flattening discards.
 type Text struct {
 	Excerpt string
 	Html    struct {
 // Post is a deviation together with its comment metadata, as returned by
 // [GetDeviation]. IMG and Description are conveniences that GetDeviation derives
-// from the Deviation, so callers need not assemble a URL or decode Draft.js
+// from the Deviation, so callers need not assemble a URL or flatten a rich-text
-// markup themselves.
+// document themselves. Description is empty for the many deviations that have
+// none.
 //
 // Comments holds only a total and a cursor. To retrieve the comments, pass them
 // to [GetComments] with type 1.
 	st.IMG, _ = UrlFromMedia(st.Deviation.Media)
-	st.Description = flattenComment(st.Deviation.TextContent.Html.Markup)
+	// The description lives in Extended.DescriptionText on the great majority of
+	// deviations; TextContent carries it on only a small minority. Prefer the
+	// former and fall back, since either may be the populated one.
+	desc := st.Deviation.Extended.DescriptionText.Html.Markup
+	if desc == "" {
+		desc = st.Deviation.TextContent.Html.Markup
+	}
+	st.Description = flattenMarkup(desc)
 	return
 }

Commit 9c6ad663cf

Commit `9c6ad663cf`