Commit `c22afa0a45`

c22afa0a45eb0ec1461610149a41e3465cba5988

parent: 21a4095a22

Verified · cmc ci/build: success ci/lint: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-12 02:15 UTC

Parse media and post URLs instead of slicing them by offset

wixmpMedia and proxiedMediaURL replace the fixed-offset slicing in
ParseMedia and sendMedia; ConvertDeviantArtURLToSkunkyArt parses the
link and also accepts the author.deviantart.com/art form. Legacy HTML
descriptions map an emoticon to the emote route by the image's base
name instead of by offsets that fit one URL shape, and carry alt text.
GetValueOfTag loses its one-iteration loop. Stale TODOs about JSON
caching and filters are gone.

Ref #1
Ref #6

Layout: unified · split

app/api.go +3 −10

		@@ -3,7 +3,6 @@ package app
3	3	import (
4	4	"encoding/json"
5	5	"math/rand"
6		"net/url"
7	6	"strings"
8	7
9	8	"github.com/krazywarez/devianter"
		@@ -59,17 +58,13 @@ func (a API) sendMedia(d *devianter.Deviation) {
59	58	return
60	59	}
61	60
62		// Parsed, not sliced: the signing token has to reach wixmp as a query
63		// parameter. Passing the raw tail as the path put "?token=..." inside
64		// the path, which wixmp answers with 401.
65		u, err := url.Parse(mediaURL)
66		if err != nil {
	61	subdomain, path, token, ok := wixmpMedia(mediaURL)
	62	if !ok {
67	63	a.Error("bad media url", 502)
68	64	return
69	65	}
70		subdomain := strings.TrimSuffix(strings.TrimPrefix(u.Host, "images-wixmp-"), ".wixmp.com")
71	66	a.main.Writer.Header().Del("Content-Type")
72		a.main.downloadAndSendMedia(subdomain, strings.TrimPrefix(u.Path, "/"), u.Query().Get("token"))
	67	a.main.downloadAndSendMedia(subdomain, path, token)
73	68	}
74	69
75	70	// fetchDailyDeviations is devianter.GetDailyDeviations behind a variable so
		@@ -80,8 +75,6 @@ var fetchDailyDeviations = devianter.GetDailyDeviations
80	75	// deviations. That page is one upstream call the API cache answers for its
81	76	// TTL, where the previous random searches were up to three uncacheable calls
82	77	// per hit and a cheap way for a bot to burn the instance's upstream budget.
83		//
84		// TODO: add filters.
85	78	func (a API) Random() {
86	79	dd, daErr := fetchDailyDeviations(0)
87	80	if daErr.RAW != nil {

app/cache.go −2

		@@ -1,7 +1,5 @@
1	1	package app
2	2
3		// TODO: implement JSON caching and clean up the code.
4
5	3	import (
6	4	"crypto/sha1" //nolint:gosec // G505: SHA-1 is a cache-key hash here, not a security primitive
7	5	"encoding/base64"

app/parsers.go +31 −13

		@@ -2,6 +2,8 @@ package app
2	2
3	3	import (
4	4	"encoding/json"
	5	"net/url"
	6	"path"
5	7	"strconv"
6	8	"strings"
7	9	"time"
		@@ -244,6 +246,22 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con
244	246
245	247	/* DESCRIPTION/COMMENT PARSER */
246	248
	249	// emoticonURL maps an e.deviantart.net emoticon image URL to this instance's
	250	// emote route, by the file's base name without its extension, which is what
	251	// devianter.AEmedia takes. Anything else yields "".
	252	func emoticonURL(host, raw string) string {
	253	u, err := url.Parse(raw)
	254	if err != nil \|\| u.Host != "e.deviantart.net" {
	255	return ""
	256	}
	257	name := path.Base(u.Path)
	258	name = strings.TrimSuffix(name, path.Ext(name))
	259	if name == "" \|\| name == "." \|\| name == "/" {
	260	return ""
	261	}
	262	return URLBuilder(host, "media", "emojitar", name, "?type=e")
	263	}
	264
247	265	// text is one styled run within a description: the rendered HTML, the raw source
248	266	// it came from, and the offsets it spans in the original block.
249	267	type text struct {
		@@ -427,25 +445,25 @@ func ParseDescription(host string, dscr devianter.Text) string {
427	445	}
428	446	}
429	447	case "img":
	448	// Only DeviantArt's emoticons are carried over, served
	449	// through this instance; any other image is dropped.
430	450	var uri, title string
431		for b, a := range token.Attr {
	451	for _, a := range token.Attr {
432	452	switch a.Key {
433	453	case "src":
434		if len(a.Val) > 9 && a.Val[8:9] == "e" {
435		uri = URLBuilder(host, "media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
436		}
	454	uri = emoticonURL(host, a.Val)
437	455	case "title":
438	456	title = a.Val
439	457	}
440		if title != "" {
441		for x := -1; x < b; x++ {
442		parsedDescription.WriteString(`<img src="`)
443		parsedDescription.WriteString(esc(uri))
444		parsedDescription.WriteString(`" title="`)
445		parsedDescription.WriteString(esc(title))
446		parsedDescription.WriteString(`">`)
447		}
448		}
	458	}
	459	if uri != "" {
	460	parsedDescription.WriteString(`<img src="`)
	461	parsedDescription.WriteString(esc(uri))
	462	parsedDescription.WriteString(`" alt="`)
	463	parsedDescription.WriteString(esc(title))
	464	parsedDescription.WriteString(`" title="`)
	465	parsedDescription.WriteString(esc(title))
	466	parsedDescription.WriteString(`">`)
449	467	}
450	468	case "br", "li", "ul", "p", "b":
451	469	// The bare tag, not token.String(): that would carry over

app/urls_test.go added +84

		@@ -0,0 +1,84 @@
	1	package app
	2
	3	import (
	4	"strings"
	5	"testing"
	6
	7	"github.com/krazywarez/devianter"
	8	)
	9
	10	func TestProxiedMediaURLCarriesTokenAndFilenameAsQuery(t *testing.T) {
	11	uri := CFG.URI
	12	CFG.URI = "/"
	13	defer func() { CFG.URI = uri }()
	14
	15	raw := "https://images-wixmp-abc.wixmp.com/f/u/x.png/v1/fit/w_640,h_364/x_by_y.png?token=tok.en.sig"
	16	got := proxiedMediaURL("http://localhost", raw, "x_by_y.png")
	17	want := "http://localhost/media/file/abc/f/u/x.png/v1/fit/w_640,h_364/x_by_y.png?filename=x_by_y.png&token=tok.en.sig"
	18	if got != want {
	19	t.Errorf("got %s\nwant %s", got, want)
	20	}
	21	if proxiedMediaURL("http://localhost", "https://example.com/x.png", "x.png") != "" {
	22	t.Error("a non-wixmp URL was proxied")
	23	}
	24	}
	25
	26	func TestParseMediaMatchesTheProxyRoute(t *testing.T) {
	27	proxy, uri := CFG.Proxy, CFG.URI
	28	CFG.Proxy, CFG.URI = true, "/"
	29	defer func() { CFG.Proxy, CFG.URI = proxy, uri }()
	30
	31	d := fullviewDeviation()
	32	d.Media.Token = []string{"tok.en.sig"}
	33	got := ParseMedia("http://localhost", d.Media)
	34	if !strings.HasPrefix(got, "http://localhost/media/file/abc/") \|\| !strings.Contains(got, "token=tok.en.sig") \|\| !strings.Contains(got, "filename=") {
	35	t.Errorf("proxied media URL is %q", got)
	36	}
	37
	38	CFG.Proxy = false
	39	if got := ParseMedia("http://localhost", d.Media); !strings.HasPrefix(got, "https://images-wixmp-abc.wixmp.com/") {
	40	t.Errorf("with proxying off got %q, want the wixmp URL", got)
	41	}
	42	}
	43
	44	func TestConvertDeviantArtURLToSkunkyArt(t *testing.T) {
	45	uri := CFG.URI
	46	CFG.URI = "/"
	47	defer func() { CFG.URI = uri }()
	48
	49	cases := map[string]string{
	50	"https://www.deviantart.com/alice/art/Title-123": "http://localhost/post/alice/Title-123",
	51	"https://alice.deviantart.com/art/Title-123": "http://localhost/post/alice/Title-123",
	52	"https://www.deviantart.com/stash/01t1te6losnc": "",
	53	"https://sta.sh/01t1te6losnc": "",
	54	"https://www.deviantart.com/alice": "",
	55	"https://example.com/alice/art/Title-123": "",
	56	"https://www.deviantart.com/alice/art/Title-123?comment": "http://localhost/post/alice/Title-123",
	57	}
	58	for in, want := range cases {
	59	if got := ConvertDeviantArtURLToSkunkyArt("http://localhost", in); got != want {
	60	t.Errorf("%s: got %q, want %q", in, got, want)
	61	}
	62	}
	63	}
	64
	65	// TestLegacyEmoticonIsServedThroughTheInstance is the regression test for #6's
	66	// HTML half: the emoticon name used to be cut out of the URL by fixed offsets,
	67	// which only fit one URL shape.
	68	func TestLegacyEmoticonIsServedThroughTheInstance(t *testing.T) {
	69	uri := CFG.URI
	70	CFG.URI = "/"
	71	defer func() { CFG.URI = uri }()
	72
	73	var d devianter.Text
	74	d.Html.Markup = `hi <img src="https://e.deviantart.net/emoticons/s/smile.gif" title=":) (Smile)"> and <img src="https://e.deviantart.net/emoticons/letters/l/love.gif" title="Love"> not <img src="https://example.com/x.png">`
	75	out := ParseDescription("http://localhost", d)
	76	for _, want := range []string{`src="http://localhost/media/emojitar/smile?type=e"`, `src="http://localhost/media/emojitar/love?type=e"`, `alt=":) (Smile)"`} {
	77	if !strings.Contains(out, want) {
	78	t.Errorf("output lacks %q:\n%s", want, out)
	79	}
	80	}
	81	if strings.Contains(out, "example.com") {
	82	t.Errorf("foreign image carried over:\n%s", out)
	83	}
	84	}

app/util.go +54 −27

		@@ -332,39 +332,69 @@ func Download(urlString string) (d Downloaded) {
332	332
333	333	/* PARSING HELPERS */
334	334
	335	// wixmpMedia splits a wixmp CDN URL into the pieces the media proxy takes:
	336	// the variable hostname label, the path without its leading slash, and the
	337	// signing token. ok is false for anything that is not wixmp media.
	338	func wixmpMedia(raw string) (subdomain, path, token string, ok bool) {
	339	u, err := url.Parse(raw)
	340	if err != nil \|\| !strings.HasPrefix(u.Host, "images-wixmp-") \|\| !strings.HasSuffix(u.Host, ".wixmp.com") {
	341	return "", "", "", false
	342	}
	343	subdomain = strings.TrimSuffix(strings.TrimPrefix(u.Host, "images-wixmp-"), ".wixmp.com")
	344	return subdomain, strings.TrimPrefix(u.Path, "/"), u.Query().Get("token"), true
	345	}
	346
	347	// proxiedMediaURL is the instance URL that serves raw through the media proxy,
	348	// with the token and a download filename as query parameters, or "" when raw is
	349	// not wixmp media. host is the request's scheme and host, as taken by URLBuilder.
	350	func proxiedMediaURL(host, raw, filename string) string {
	351	subdomain, path, token, ok := wixmpMedia(raw)
	352	if !ok {
	353	return ""
	354	}
	355	q := url.Values{}
	356	if token != "" {
	357	q.Set("token", token)
	358	}
	359	if filename != "" {
	360	q.Set("filename", filename)
	361	}
	362	return URLBuilder(host, "media", "file", subdomain, path) + "?" + q.Encode()
	363	}
	364
335	365	// ParseMedia returns the URL to serve for media: a link back through this
336	366	// instance's media proxy when proxying is on, or DeviantArt's own URL when it is
337	367	// off. An optional thumb width selects a thumbnail instead of the full image.
338	368	// host is the request's scheme and host, as taken by URLBuilder.
339	369	func ParseMedia(host string, media devianter.Media, thumb ...int) string {
340	370	mediaURL, filename := devianter.UrlFromMedia(media, thumb...)
341		if len(mediaURL) != 0 && CFG.Proxy {
342		mediaURL = mediaURL[21:]
343		dot := strings.Index(mediaURL, ".")
344		if filename == "" {
345		filename = "image.gif"
346		}
347		return URLBuilder(host, "media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename)
348		} else if !CFG.Proxy {
	371	if mediaURL == "" \|\| !CFG.Proxy {
349	372	return mediaURL
350	373	}
351		return ""
	374	if filename == "" {
	375	filename = "image.gif"
	376	}
	377	return proxiedMediaURL(host, mediaURL, filename)
352	378	}
353	379
354		// ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link into the
355		// equivalent link on this instance. It returns an empty string for URLs it does
356		// not handle, including sta.sh links. host is the request's scheme and host, as
	380	// ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link, in the
	381	// www.deviantart.com/<author>/art/<name> or <author>.deviantart.com/art/<name>
	382	// form, into the equivalent link on this instance. It returns "" for anything
	383	// else, including sta.sh links. host is the request's scheme and host, as
357	384	// taken by URLBuilder.
358		func ConvertDeviantArtURLToSkunkyArt(host, url string) (output string) {
359		if len(url) > 32 && url[27:32] != "stash" {
360		url = url[27:]
361		firstshash := strings.Index(url, "/")
362		lastshash := firstshash + strings.Index(url[firstshash+1:], "/")
363		if lastshash != -1 {
364		output = URLBuilder(host, "post", url[:firstshash], url[lastshash+2:])
365		}
	385	func ConvertDeviantArtURLToSkunkyArt(host, raw string) string {
	386	u, err := url.Parse(raw)
	387	if err != nil \|\| !strings.HasSuffix(u.Host, "deviantart.com") {
	388	return ""
366	389	}
367		return
	390	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
	391	switch {
	392	case len(parts) == 3 && parts[1] == "art" && parts[0] != "stash":
	393	return URLBuilder(host, "post", parts[0], parts[2])
	394	case len(parts) == 2 && parts[0] == "art" && u.Host != "www.deviantart.com":
	395	return URLBuilder(host, "post", strings.TrimSuffix(u.Host, ".deviantart.com"), parts[1])
	396	}
	397	return ""
368	398	}
369	399
370	400	// BuildUserPlate renders the small avatar-and-username block linking to a user's
		@@ -386,13 +416,10 @@ func BuildUserPlate(host, name string) string {
386	416	// GetValueOfTag returns the text of the tokenizer's next token, or an empty
387	417	// string if that token is not text.
388	418	func GetValueOfTag(t *html.Tokenizer) string {
389		for tt := t.Next(); ; {
390		if tt == html.TextToken {
391		return string(t.Text())
392		} else {
393		return ""
394		}
	419	if t.Next() == html.TextToken {
	420	return string(t.Text())
395	421	}
	422	return ""
396	423	}
397	424
398	425	// DeviationList describes the pagination state of a list of artworks: how many

Commit c22afa0a45

Commit `c22afa0a45`