Parse media and post URLs instead of slicing them by offset !33

merged merged by cmc on 2026-09-12 02:18 UTC · krz/skunky-art:chore/url-helpers into main

5 files changed, +172 −52

Layout: unified · split

app/api.go +3 −10

 import (
 	"encoding/json"
 	"math/rand"
-	"net/url"
 	"strings"
 	"github.com/krazywarez/devianter"
 		return
 	}
-	// Parsed, not sliced: the signing token has to reach wixmp as a query
+	subdomain, path, token, ok := wixmpMedia(mediaURL)
-	// parameter. Passing the raw tail as the path put "?token=..." inside
+	if !ok {
-	// the path, which wixmp answers with 401.
-	u, err := url.Parse(mediaURL)
-	if err != nil {
 		a.Error("bad media url", 502)
 		return
 	}
-	subdomain := strings.TrimSuffix(strings.TrimPrefix(u.Host, "images-wixmp-"), ".wixmp.com")
 	a.main.Writer.Header().Del("Content-Type")
-	a.main.downloadAndSendMedia(subdomain, strings.TrimPrefix(u.Path, "/"), u.Query().Get("token"))
+	a.main.downloadAndSendMedia(subdomain, path, token)
 }
 // fetchDailyDeviations is devianter.GetDailyDeviations behind a variable so
 // deviations. That page is one upstream call the API cache answers for its
 // TTL, where the previous random searches were up to three uncacheable calls
 // per hit and a cheap way for a bot to burn the instance's upstream budget.
-//
-// TODO: add filters.
 func (a API) Random() {
 	dd, daErr := fetchDailyDeviations(0)
 	if daErr.RAW != nil {

app/cache.go −2

	@@ -1,7 +1,5 @@
1	package app	1	package app
2		2
3	// TODO: implement JSON caching and clean up the code.
4
5	import (	3	import (
6	"crypto/sha1" //nolint:gosec // G505: SHA-1 is a cache-key hash here, not a security primitive	4	"crypto/sha1" //nolint:gosec // G505: SHA-1 is a cache-key hash here, not a security primitive
7	"encoding/base64"	5	"encoding/base64"

app/parsers.go +31 −13

 import (
 	"encoding/json"
+	"net/url"
+	"path"
 	"strconv"
 	"strings"
 	"time"
 /* DESCRIPTION/COMMENT PARSER */
+// emoticonURL maps an e.deviantart.net emoticon image URL to this instance's
+// emote route, by the file's base name without its extension, which is what
+// devianter.AEmedia takes. Anything else yields "".
+func emoticonURL(host, raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil || u.Host != "e.deviantart.net" {
+		return ""
+	}
+	name := path.Base(u.Path)
+	name = strings.TrimSuffix(name, path.Ext(name))
+	if name == "" || name == "." || name == "/" {
+		return ""
+	}
+	return URLBuilder(host, "media", "emojitar", name, "?type=e")
+}
 // text is one styled run within a description: the rendered HTML, the raw source
 // it came from, and the offsets it spans in the original block.
 type text struct {
 						}
 					}
 				case "img":
+					// Only DeviantArt's emoticons are carried over, served
+					// through this instance; any other image is dropped.
 					var uri, title string
-					for b, a := range token.Attr {
+					for _, a := range token.Attr {
 						switch a.Key {
 						case "src":
-							if len(a.Val) > 9 && a.Val[8:9] == "e" {
+							uri = emoticonURL(host, a.Val)
-								uri = URLBuilder(host, "media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
-							}
 						case "title":
 							title = a.Val
 						}
-						if title != "" {
+					}
-							for x := -1; x < b; x++ {
+					if uri != "" {
-								parsedDescription.WriteString(`<img src="`)
+						parsedDescription.WriteString(`<img src="`)
-								parsedDescription.WriteString(esc(uri))
+						parsedDescription.WriteString(esc(uri))
-								parsedDescription.WriteString(`" title="`)
+						parsedDescription.WriteString(`" alt="`)
-								parsedDescription.WriteString(esc(title))
+						parsedDescription.WriteString(esc(title))
-								parsedDescription.WriteString(`">`)
+						parsedDescription.WriteString(`" title="`)
-							}
+						parsedDescription.WriteString(esc(title))
-						}
+						parsedDescription.WriteString(`">`)
 					}
 				case "br", "li", "ul", "p", "b":
 					// The bare tag, not token.String(): that would carry over

app/urls_test.go added +84

@@ -0,0 +1,84 @@
	1	package app
	2
	3	import (
	4	"strings"
	5	"testing"
	6
	7	"github.com/krazywarez/devianter"
	8	)
	9
	10	func TestProxiedMediaURLCarriesTokenAndFilenameAsQuery(t *testing.T) {
	11	uri := CFG.URI
	12	CFG.URI = "/"
	13	defer func() { CFG.URI = uri }()
	14
	15	raw := "https://images-wixmp-abc.wixmp.com/f/u/x.png/v1/fit/w_640,h_364/x_by_y.png?token=tok.en.sig"
	16	got := proxiedMediaURL("http://localhost", raw, "x_by_y.png")
	17	want := "http://localhost/media/file/abc/f/u/x.png/v1/fit/w_640,h_364/x_by_y.png?filename=x_by_y.png&token=tok.en.sig"
	18	if got != want {
	19	t.Errorf("got %s\nwant %s", got, want)
	20	}
	21	if proxiedMediaURL("http://localhost", "https://example.com/x.png", "x.png") != "" {
	22	t.Error("a non-wixmp URL was proxied")
	23	}
	24	}
	25
	26	func TestParseMediaMatchesTheProxyRoute(t *testing.T) {
	27	proxy, uri := CFG.Proxy, CFG.URI
	28	CFG.Proxy, CFG.URI = true, "/"
	29	defer func() { CFG.Proxy, CFG.URI = proxy, uri }()
	30
	31	d := fullviewDeviation()
	32	d.Media.Token = []string{"tok.en.sig"}
	33	got := ParseMedia("http://localhost", d.Media)
	34	if !strings.HasPrefix(got, "http://localhost/media/file/abc/") \|\| !strings.Contains(got, "token=tok.en.sig") \|\| !strings.Contains(got, "filename=") {
	35	t.Errorf("proxied media URL is %q", got)
	36	}
	37
	38	CFG.Proxy = false
	39	if got := ParseMedia("http://localhost", d.Media); !strings.HasPrefix(got, "https://images-wixmp-abc.wixmp.com/") {
	40	t.Errorf("with proxying off got %q, want the wixmp URL", got)
	41	}
	42	}
	43
	44	func TestConvertDeviantArtURLToSkunkyArt(t *testing.T) {
	45	uri := CFG.URI
	46	CFG.URI = "/"
	47	defer func() { CFG.URI = uri }()
	48
	49	cases := map[string]string{
	50	"https://www.deviantart.com/alice/art/Title-123": "http://localhost/post/alice/Title-123",
	51	"https://alice.deviantart.com/art/Title-123": "http://localhost/post/alice/Title-123",
	52	"https://www.deviantart.com/stash/01t1te6losnc": "",
	53	"https://sta.sh/01t1te6losnc": "",
	54	"https://www.deviantart.com/alice": "",
	55	"https://example.com/alice/art/Title-123": "",
	56	"https://www.deviantart.com/alice/art/Title-123?comment": "http://localhost/post/alice/Title-123",
	57	}
	58	for in, want := range cases {
	59	if got := ConvertDeviantArtURLToSkunkyArt("http://localhost", in); got != want {
	60	t.Errorf("%s: got %q, want %q", in, got, want)
	61	}
	62	}
	63	}
	64
	65	// TestLegacyEmoticonIsServedThroughTheInstance is the regression test for #6's
	66	// HTML half: the emoticon name used to be cut out of the URL by fixed offsets,
	67	// which only fit one URL shape.
	68	func TestLegacyEmoticonIsServedThroughTheInstance(t *testing.T) {
	69	uri := CFG.URI
	70	CFG.URI = "/"
	71	defer func() { CFG.URI = uri }()
	72
	73	var d devianter.Text
	74	d.Html.Markup = `hi <img src="https://e.deviantart.net/emoticons/s/smile.gif" title=":) (Smile)"> and <img src="https://e.deviantart.net/emoticons/letters/l/love.gif" title="Love"> not <img src="https://example.com/x.png">`
	75	out := ParseDescription("http://localhost", d)
	76	for _, want := range []string{`src="http://localhost/media/emojitar/smile?type=e"`, `src="http://localhost/media/emojitar/love?type=e"`, `alt=":) (Smile)"`} {
	77	if !strings.Contains(out, want) {
	78	t.Errorf("output lacks %q:\n%s", want, out)
	79	}
	80	}
	81	if strings.Contains(out, "example.com") {
	82	t.Errorf("foreign image carried over:\n%s", out)
	83	}
	84	}

app/util.go +54 −27

 /* PARSING HELPERS */
+// wixmpMedia splits a wixmp CDN URL into the pieces the media proxy takes:
+// the variable hostname label, the path without its leading slash, and the
+// signing token. ok is false for anything that is not wixmp media.
+func wixmpMedia(raw string) (subdomain, path, token string, ok bool) {
+	u, err := url.Parse(raw)
+	if err != nil || !strings.HasPrefix(u.Host, "images-wixmp-") || !strings.HasSuffix(u.Host, ".wixmp.com") {
+		return "", "", "", false
+	}
+	subdomain = strings.TrimSuffix(strings.TrimPrefix(u.Host, "images-wixmp-"), ".wixmp.com")
+	return subdomain, strings.TrimPrefix(u.Path, "/"), u.Query().Get("token"), true
+}
+// proxiedMediaURL is the instance URL that serves raw through the media proxy,
+// with the token and a download filename as query parameters, or "" when raw is
+// not wixmp media. host is the request's scheme and host, as taken by URLBuilder.
+func proxiedMediaURL(host, raw, filename string) string {
+	subdomain, path, token, ok := wixmpMedia(raw)
+	if !ok {
+		return ""
+	}
+	q := url.Values{}
+	if token != "" {
+		q.Set("token", token)
+	}
+	if filename != "" {
+		q.Set("filename", filename)
+	}
+	return URLBuilder(host, "media", "file", subdomain, path) + "?" + q.Encode()
+}
 // ParseMedia returns the URL to serve for media: a link back through this
 // instance's media proxy when proxying is on, or DeviantArt's own URL when it is
 // off. An optional thumb width selects a thumbnail instead of the full image.
 // host is the request's scheme and host, as taken by URLBuilder.
 func ParseMedia(host string, media devianter.Media, thumb ...int) string {
 	mediaURL, filename := devianter.UrlFromMedia(media, thumb...)
-	if len(mediaURL) != 0 && CFG.Proxy {
+	if mediaURL == "" || !CFG.Proxy {
-		mediaURL = mediaURL[21:]
-		dot := strings.Index(mediaURL, ".")
-		if filename == "" {
-			filename = "image.gif"
-		}
-		return URLBuilder(host, "media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename)
-	} else if !CFG.Proxy {
 		return mediaURL
 	}
-	return ""
+	if filename == "" {
+		filename = "image.gif"
+	}
+	return proxiedMediaURL(host, mediaURL, filename)
 }
-// ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link into the
+// ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link, in the
-// equivalent link on this instance. It returns an empty string for URLs it does
+// www.deviantart.com/<author>/art/<name> or <author>.deviantart.com/art/<name>
-// not handle, including sta.sh links. host is the request's scheme and host, as
+// form, into the equivalent link on this instance. It returns "" for anything
+// else, including sta.sh links. host is the request's scheme and host, as
 // taken by URLBuilder.
-func ConvertDeviantArtURLToSkunkyArt(host, url string) (output string) {
+func ConvertDeviantArtURLToSkunkyArt(host, raw string) string {
-	if len(url) > 32 && url[27:32] != "stash" {
+	u, err := url.Parse(raw)
-		url = url[27:]
+	if err != nil || !strings.HasSuffix(u.Host, "deviantart.com") {
-		firstshash := strings.Index(url, "/")
+		return ""
-		lastshash := firstshash + strings.Index(url[firstshash+1:], "/")
-		if lastshash != -1 {
-			output = URLBuilder(host, "post", url[:firstshash], url[lastshash+2:])
-		}
 	}
-	return
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	switch {
+	case len(parts) == 3 && parts[1] == "art" && parts[0] != "stash":
+		return URLBuilder(host, "post", parts[0], parts[2])
+	case len(parts) == 2 && parts[0] == "art" && u.Host != "www.deviantart.com":
+		return URLBuilder(host, "post", strings.TrimSuffix(u.Host, ".deviantart.com"), parts[1])
+	}
+	return ""
 }
 // BuildUserPlate renders the small avatar-and-username block linking to a user's
 // GetValueOfTag returns the text of the tokenizer's next token, or an empty
 // string if that token is not text.
 func GetValueOfTag(t *html.Tokenizer) string {
-	for tt := t.Next(); ; {
+	if t.Next() == html.TextToken {
-		if tt == html.TextToken {
+		return string(t.Text())
-			return string(t.Text())
-		} else {
-			return ""
-		}
 	}
+	return ""
 }
 // DeviationList describes the pagination state of a list of artworks: how many