Commit `83d9cd0b7a`

83d9cd0b7a3e0ec6beb5c889102211567a6db03f

parent: 8d08f343c9

Verified · cmc

cmc <hello@cleberg.net> · 2026-07-15 08:17 UTC

fix: serve media again by unsetting download-proxy and scoping Host per request

Two independent faults made every image fail while pages still rendered.

config.example.json shipped download-proxy=http://127.0.0.1:8080. Only media
fetches go through that proxy — pages reach DeviantArt via devianter on the
default transport — so when nothing listens there, images 502 and the rest of
the page looks fine. In a scratch container 127.0.0.1 is the container itself,
so the default could never work under Docker. Unset it and document that it must
stay empty unless an operator really runs a proxy.

Host was a package global reassigned by every request, so a concurrent request
could overwrite it mid-render and emit URLs on another origin's host and port.
The instance's own default-src 'self' CSP then blocked those images. Thread the
request's host through skunkyart instead, and take it as an explicit argument in
URLBuilder, ParseMedia, ParseDescription, BuildUserPlate and
ConvertDeviantArtURLToSkunkyArt. Feeds keep their absolute URLs.

Also start RefreshInstances after ExecuteConfig rather than before it: the
goroutine read CFG while json.Unmarshal was writing it (a race the detector
flags), and its fetch escaped both the throttle and the configured User-Agent.

Verified: 300 concurrent requests with distinct Host headers now round-trip
their own host (was 1 leak per 300), go test -race is clean, and
cache+proxy both enabled serves 200 image/jpeg cold and from cache.

Layout: unified · split

SETUP.md +10 −1

	@@ -17,8 +17,17 @@ Time units:
17	* `max-size` — Maximum file size in megabytes	17	* `max-size` — Maximum file size in megabytes
18	* `update-interval` — Automatic rotation interval	18	* `update-interval` — Automatic rotation interval
19	* `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag.	19	* `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag.
20	* `download-proxy` — Proxy address for downloading files.	20	* `download-proxy` — Outbound proxy used when fetching media from DeviantArt's
		21	CDN. Leave empty (`""`) unless you actually run a proxy: if this points at
		22	something that isn't listening, every image 502s while pages still render,
		23	because only media fetches go through it. Inside a container `127.0.0.1` is
		24	the container itself, so a host-side proxy must be addressed by service name
		25	or host IP, not loopback.
21	* `user-agent` — String, which SkunkyArt uses as UA	26	* `user-agent` — String, which SkunkyArt uses as UA
		27	* `proxy` — Serve media through this instance instead of linking straight to
		28	DeviantArt's CDN. Required by `cache`; when off, clients fetch images from
		29	wixmp directly.
		30	* `nsfw` — Show mature content.
22		31
23	# Setting up reverse proxy	32	# Setting up reverse proxy
24	Pretty much business as usual, except for the [`X-Forwarded-Proto`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto) header setting.	33	Pretty much business as usual, except for the [`X-Forwarded-Proto`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto) header setting.

app/parsers.go +14 −13

 		cmmts.WriteString(`"><p id="`)
 		cmmts.WriteString(strconv.Itoa(x.ID))
 		cmmts.WriteString(`"><img src="`)
-		cmmts.WriteString(URLBuilder("media", "emojitar", x.User.Username, "?type=a"))
+		cmmts.WriteString(URLBuilder(s.Host, "media", "emojitar", x.User.Username, "?type=a"))
 		cmmts.WriteString(`" width="30px" height="30px"><a href="`)
-		cmmts.WriteString(URLBuilder("group_user", "?q=", x.User.Username, "&type=a"))
+		cmmts.WriteString(URLBuilder(s.Host, "group_user", "?q=", x.User.Username, "&type=a"))
 		cmmts.WriteString(`"><b`)
 		cmmts.WriteString(` class="`)
 		if x.User.Banned {
 		cmmts.WriteString(x.Posted.UTC().String())
 		cmmts.WriteString("]<p>")
-		cmmts.WriteString(ParseDescription(x.TextContent))
+		cmmts.WriteString(ParseDescription(s.Host, x.TextContent))
 		cmmts.WriteString("<p>👍: ")
 		cmmts.WriteString(strconv.Itoa(x.Likes))
 		cmmts.WriteString(" ⏩: ")
 	for i, l := 0, len(devs); i < l; i++ {
 		data := &devs[i]
-		if preview, fullview := ParseMedia(data.Media, 320), ParseMedia(data.Media); !data.NSFW || CFG.Nsfw {
+		if preview, fullview := ParseMedia(s.Host, data.Media, 320), ParseMedia(s.Host, data.Media); !data.NSFW || CFG.Nsfw {
 			if allowAtom && s.Atom {
 				s.Writer.Header().Add("Content-Type", "application/atom+xml")
 				id := strconv.Itoa(data.ID)
 				listContent.WriteString(`</name></author><title>`)
 				listContent.WriteString(data.Title)
 				listContent.WriteString(`</title><link rel="alternate" type="text/html" href="`)
-				listContent.WriteString(URLBuilder("post", data.Author.Username, "atom-"+id))
+				listContent.WriteString(URLBuilder(s.Host, "post", data.Author.Username, "atom-"+id))
 				listContent.WriteString(`"/><id>`)
 				listContent.WriteString(id)
 				listContent.WriteString(`</id><published>`)
 				listContent.WriteString(`</media:title><media:thumbinal url="`)
 				listContent.WriteString(preview)
 				listContent.WriteString(`"/></media:group><content type="xhtml"><div xmlns="http://www.w3.org/1999/xhtml"><a href="`)
-				listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(data.Url))
+				listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url))
 				listContent.WriteString(`"><img src="`)
 				listContent.WriteString(fullview)
 				listContent.WriteString(`"/></a><p>`)
-				listContent.WriteString(ParseDescription(data.TextContent))
+				listContent.WriteString(ParseDescription(s.Host, data.TextContent))
 				listContent.WriteString(`</p></div></content></entry>`)
 			} else {
 				listContent.WriteString(`<div class="block">`)
 					listContent.WriteString(`<h1>[ TEXT ]</h1>`)
 				}
 				listContent.WriteString(`<br><a href="`)
-				listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(data.Url))
+				listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url))
 				listContent.WriteString(`">`)
 				listContent.WriteString(data.Author.Username)
 				listContent.WriteString(" - ")
 		list.WriteString(`</title>`)
 		list.WriteString(`<link rel="alternate" href="`)
-		list.WriteString(Host)
+		list.WriteString(s.Host)
 		list.WriteString(`"/>`)
 		list.WriteString(listContent.String())
 // ParseDescription renders a DeviantArt description into HTML, handling both the
 // Draft.js-style JSON payload and the plain HTML markup DeviantArt returns, and
 // rewriting embedded links and artwork references to point at this instance.
+// host is the request's scheme and host, as taken by URLBuilder.
 //
 // TODO: rewrite this whole mess.
-func ParseDescription(dscr devianter.Text) string {
+func ParseDescription(host string, dscr devianter.Text) string {
 	var parsedDescription strings.Builder
 	TagBuilder := func(content string, tags ...string) string {
 		l := len(tags)
 				if len(x.EntityRanges) != 0 {
 					d := entities[x.EntityRanges[0].Key]
 					parsedDescription.WriteString(`<a href="`)
-					parsedDescription.WriteString(ConvertDeviantArtURLToSkunkyArt(d.Url))
+					parsedDescription.WriteString(ConvertDeviantArtURLToSkunkyArt(host, d.Url))
 					parsedDescription.WriteString(`"><img width="50%" src="`)
-					parsedDescription.WriteString(ParseMedia(d.Media))
+					parsedDescription.WriteString(ParseMedia(host, d.Media))
 					parsedDescription.WriteString(`" title="`)
 					parsedDescription.WriteString(d.Author.Username)
 					parsedDescription.WriteString(" - ")
 						switch a.Key {
 						case "src":
 							if len(a.Val) > 9 && a.Val[8:9] == "e" {
-								uri = URLBuilder("media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
+								uri = URLBuilder(host, "media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
 							}
 						case "title":
 							title = a.Val

app/router.go +9 −7

	@@ -10,10 +10,6 @@ import (
10	"time"	10	"time"
11	)	11	)
12		12
13	// Host is the scheme and host that generated links are built from. It is set per
14	// request from the Host header and X-Forwarded-Proto.
15	var Host string
16
17	// Router registers the single catch-all handler that dispatches every path, then	13	// Router registers the single catch-all handler that dispatches every path, then
18	// serves until the process exits. It does not return on success.	14	// serves until the process exits. It does not return on success.
19	func Router() {	15	func Router() {
	@@ -68,12 +64,18 @@ func Router() {
68	// the function that drives everything	64	// the function that drives everything
69	handle := func(w http.ResponseWriter, r *http.Request) {	65	handle := func(w http.ResponseWriter, r *http.Request) {
70	path := parsepath(r.URL.Path)	66	path := parsepath(r.URL.Path)
71	Host = "http://" + r.Host	67
		68	// Per-request, not a package global: requests arrive concurrently on
		69	// different hosts and ports (bots hitting a proxy's alternate ports, for
		70	// one), and a shared global lets one request's host leak into another's
		71	// rendered URLs. Those URLs then point at a different origin, which this
		72	// handler's own default-src 'self' CSP blocks.
		73	host := "http://" + r.Host
72	if h := r.Header["X-Forwarded-Proto"]; len(h) != 0 && h[0] == "https" {	74	if h := r.Header["X-Forwarded-Proto"]; len(h) != 0 && h[0] == "https" {
73	Host = "https://" + r.Host	75	host = "https://" + r.Host
74	}	76	}
75		77
76	var skunky = skunkyart{Version: Release.Version}	78	var skunky = skunkyart{Version: Release.Version, Host: host}
77	skunky._pth = r.URL.Path	79	skunky._pth = r.URL.Path
78		80
79	skunky.Args = r.URL.Query()	81	skunky.Args = r.URL.Query()

app/util.go +25 −16

 	Type rune
 	Atom bool
+	// Host is the scheme and host this request arrived on, e.g.
+	// "https://art.example.com". It is per-request rather than global because
+	// concurrent requests can arrive on different hosts and ports.
+	Host string
 	BasePath, Endpoint string
 	Query, QueryRaw    string
 	wr(s.Writer, buf.String())
 }
-// URLBuilder joins strs into an absolute instance URL, prefixing the current
+// URLBuilder joins strs into an absolute instance URL, prefixing host and the
-// Host and configured URI and inserting slashes between path segments but not
+// configured URI and inserting slashes between path segments but not before
-// before query separators.
+// query separators. host is the request's own scheme and host: passing the
-func URLBuilder(strs ...string) string {
+// wrong one emits links to another origin, which the instance's own
+// Content-Security-Policy then blocks.
+func URLBuilder(host string, strs ...string) string {
 	var str strings.Builder
 	l := len(strs)
-	str.WriteString(Host)
+	str.WriteString(host)
 	str.WriteString(CFG.URI)
 	for n, x := range strs {
 		str.WriteString(x)
 	var msg strings.Builder
 	msg.WriteString(`<html><link rel="stylesheet" href="`)
-	msg.WriteString(URLBuilder("stylesheet"))
+	msg.WriteString(URLBuilder(s.Host, "stylesheet"))
 	msg.WriteString(`" /><h3>DeviantArt error — '`)
 	msg.WriteString(dAerr.Error)
 	msg.WriteString("'</h3></html>")
 	var msg strings.Builder
 	msg.WriteString(`<html><link rel="stylesheet" href="`)
-	msg.WriteString(URLBuilder("stylesheet"))
+	msg.WriteString(URLBuilder(s.Host, "stylesheet"))
 	msg.WriteString(`" /><h1>`)
 	msg.WriteString(strconv.Itoa(status))
 	msg.WriteString(" - ")
 // ParseMedia returns the URL to serve for media: a link back through this
 // instance's media proxy when proxying is on, or DeviantArt's own URL when it is
 // off. An optional thumb width selects a thumbnail instead of the full image.
-func ParseMedia(media devianter.Media, thumb ...int) string {
+// host is the request's scheme and host, as taken by URLBuilder.
+func ParseMedia(host string, media devianter.Media, thumb ...int) string {
 	mediaURL, filename := devianter.UrlFromMedia(media, thumb...)
 	if len(mediaURL) != 0 && CFG.Proxy {
 		mediaURL = mediaURL[21:]
 		if filename == "" {
 			filename = "image.gif"
 		}
-		return URLBuilder("media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename)
+		return URLBuilder(host, "media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename)
 	} else if !CFG.Proxy {
 		return mediaURL
 	}
 // ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link into the
 // equivalent link on this instance. It returns an empty string for URLs it does
-// not handle, including sta.sh links.
+// not handle, including sta.sh links. host is the request's scheme and host, as
-func ConvertDeviantArtURLToSkunkyArt(url string) (output string) {
+// taken by URLBuilder.
+func ConvertDeviantArtURLToSkunkyArt(host, url string) (output string) {
 	if len(url) > 32 && url[27:32] != "stash" {
 		url = url[27:]
 		firstshash := strings.Index(url, "/")
 		lastshash := firstshash + strings.Index(url[firstshash+1:], "/")
 		if lastshash != -1 {
-			output = URLBuilder("post", url[:firstshash], url[lastshash+2:])
+			output = URLBuilder(host, "post", url[:firstshash], url[lastshash+2:])
 		}
 	}
 	return
 }
 // BuildUserPlate renders the small avatar-and-username block linking to a user's
-// about page.
+// about page. host is the request's scheme and host, as taken by URLBuilder.
-func BuildUserPlate(name string) string {
+func BuildUserPlate(host, name string) string {
 	var htm strings.Builder
 	htm.WriteString(`<div class="user-plate"><img src="`)
-	htm.WriteString(URLBuilder("media", "emojitar", name, "?type=a"))
+	htm.WriteString(URLBuilder(host, "media", "emojitar", name, "?type=a"))
 	htm.WriteString(`"><a href="`)
-	htm.WriteString(URLBuilder("group_user", "?type=about&q=", name))
+	htm.WriteString(URLBuilder(host, "group_user", "?type=about&q=", name))
 	htm.WriteString(`">`)
 	htm.WriteString(name)
 	htm.WriteString(`</a></div>`)

app/wrapper.go +13 −13

	@@ -42,12 +42,12 @@ func (s skunkyart) GRUser() {
42	var about = &x.ModuleData.GroupAbout	42	var about = &x.ModuleData.GroupAbout
43	group.Group = true	43	group.Group = true
44	group.CreationDate = x.ModuleData.GroupAbout.FoundatedAt.UTC().String()	44	group.CreationDate = x.ModuleData.GroupAbout.FoundatedAt.UTC().String()
45	group.About.DescriptionFormatted = ParseDescription(about.Description)	45	group.About.DescriptionFormatted = ParseDescription(s.Host, about.Description)
46	} else if false {	46	} else if false {
47	group.About.A = x.ModuleData.About	47	group.About.A = x.ModuleData.About
48	var about = &group.About.A	48	var about = &group.About.A
49	group.CreationDate = time.Unix(time.Now().Unix()-x.ModuleData.About.RegDate, 0).UTC().String()	49	group.CreationDate = time.Unix(time.Now().Unix()-x.ModuleData.About.RegDate, 0).UTC().String()
50	group.About.DescriptionFormatted = ParseDescription(about.Description)	50	group.About.DescriptionFormatted = ParseDescription(s.Host, about.Description)
51		51
52	for _, val := range x.ModuleData.About.SocialLinks {	52	for _, val := range x.ModuleData.About.SocialLinks {
53	var social strings.Builder	53	var social strings.Builder
	@@ -72,12 +72,12 @@ func (s skunkyart) GRUser() {
72		72
73	case "cover_deviation":	73	case "cover_deviation":
74	group.About.BGMeta = x.ModuleData.CoverDeviation.Deviation	74	group.About.BGMeta = x.ModuleData.CoverDeviation.Deviation
75	group.About.BGMeta.Url = ConvertDeviantArtURLToSkunkyArt(group.About.BGMeta.Url)	75	group.About.BGMeta.Url = ConvertDeviantArtURLToSkunkyArt(s.Host, group.About.BGMeta.Url)
76	group.About.BG = ParseMedia(group.About.BGMeta.Media)	76	group.About.BG = ParseMedia(s.Host, group.About.BGMeta.Media)
77	case "group_admins":	77	case "group_admins":
78	var htm strings.Builder	78	var htm strings.Builder
79	for _, z := range x.ModuleData.GroupAdmins.Results {	79	for _, z := range x.ModuleData.GroupAdmins.Results {
80	htm.WriteString(BuildUserPlate(z.User.Username))	80	htm.WriteString(BuildUserPlate(s.Host, z.User.Username))
81	}	81	}
82	group.Admins += htm.String()	82	group.Admins += htm.String()
83	}	83	}
	@@ -124,9 +124,9 @@ func (s skunkyart) GRUser() {
124		124
125	if !x.Thumb.NSFW \|\| CFG.Nsfw {	125	if !x.Thumb.NSFW \|\| CFG.Nsfw {
126	folders.WriteString(`<a href="`)	126	folders.WriteString(`<a href="`)
127	folders.WriteString(ConvertDeviantArtURLToSkunkyArt(x.Thumb.Url))	127	folders.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, x.Thumb.Url))
128	folders.WriteString(`"><img loading="lazy" src="`)	128	folders.WriteString(`"><img loading="lazy" src="`)
129	folders.WriteString(ParseMedia(x.Thumb.Media))	129	folders.WriteString(ParseMedia(s.Host, x.Thumb.Media))
130	folders.WriteString(`" title="`)	130	folders.WriteString(`" title="`)
131	folders.WriteString(x.Thumb.Title)	131	folders.WriteString(x.Thumb.Title)
132	folders.WriteString(`"></a>`)	132	folders.WriteString(`"></a>`)
	@@ -191,7 +191,7 @@ func (s skunkyart) Deviation(author, postname string) {
191	if post.Post.Deviation.NSFW && !CFG.Nsfw {	191	if post.Post.Deviation.NSFW && !CFG.Nsfw {
192	s.Writer.WriteHeader(403)	192	s.Writer.WriteHeader(403)
193	wr(s.Writer, `<html><link rel="stylesheet" href="`+	193	wr(s.Writer, `<html><link rel="stylesheet" href="`+
194	URLBuilder("stylesheet")+	194	URLBuilder(s.Host, "stylesheet")+
195	`" /><h1>NSFW content are disabled on this instance.</h1></html>`)	195	`" /><h1>NSFW content are disabled on this instance.</h1></html>`)
196	return	196	return
197	}	197	}
	@@ -201,9 +201,9 @@ func (s skunkyart) Deviation(author, postname string) {
201	}	201	}
202		202
203	if post.Post.Deviation.TextContent.Excerpt != "" {	203	if post.Post.Deviation.TextContent.Excerpt != "" {
204	post.Post.Description = ParseDescription(post.Post.Deviation.TextContent)	204	post.Post.Description = ParseDescription(s.Host, post.Post.Deviation.TextContent)
205	} else {	205	} else {
206	post.Post.Description = ParseDescription(post.Post.Deviation.Extended.DescriptionText)	206	post.Post.Description = ParseDescription(s.Host, post.Post.Deviation.Extended.DescriptionText)
207	}	207	}
208		208
209	for _, x := range post.Post.Deviation.Extended.RelatedContent {	209	for _, x := range post.Post.Deviation.Extended.RelatedContent {
	@@ -216,7 +216,7 @@ func (s skunkyart) Deviation(author, postname string) {
216	for _, x := range post.Post.Deviation.Extended.Tags {	216	for _, x := range post.Post.Deviation.Extended.Tags {
217	var tag strings.Builder	217	var tag strings.Builder
218	tag.WriteString(` <a href="`)	218	tag.WriteString(` <a href="`)
219	tag.WriteString(URLBuilder("search", "?q=", x.Name, "&type=tag"))	219	tag.WriteString(URLBuilder(s.Host, "search", "?q=", x.Name, "&type=tag"))
220	tag.WriteString(`">#`)	220	tag.WriteString(`">#`)
221	tag.WriteString(x.Name)	221	tag.WriteString(x.Name)
222	tag.WriteString("</a>")	222	tag.WriteString("</a>")
	@@ -226,7 +226,7 @@ func (s skunkyart) Deviation(author, postname string) {
226		226
227	post.Comments = s.ParseComments(devianter.GetComments(id, post.Post.Comments.Cursor, s.Page, 1))	227	post.Comments = s.ParseComments(devianter.GetComments(id, post.Post.Comments.Cursor, s.Page, 1))
228	post.StringTime = post.Post.Deviation.PublishedTime.UTC().String()	228	post.StringTime = post.Post.Deviation.PublishedTime.UTC().String()
229	post.Post.IMG = ParseMedia(post.Post.Deviation.Media)	229	post.Post.IMG = ParseMedia(s.Host, post.Post.Deviation.Media)
230		230
231	s.ExecuteTemplate("deviantion.htm", "html", &s)	231	s.ExecuteTemplate("deviantion.htm", "html", &s)
232	}	232	}
	@@ -312,7 +312,7 @@ func (s skunkyart) Search() {
312	if l := len(usernames); l != 0 {	312	if l := len(usernames); l != 0 {
313	ss.List += `<div class="content plates">`	313	ss.List += `<div class="content plates">`
314	for x := range len(usernames) {	314	for x := range len(usernames) {
315	ss.List += BuildUserPlate(usernames[x])	315	ss.List += BuildUserPlate(s.Host, usernames[x])
316	}	316	}
317	ss.List += `</div>`	317	ss.List += `</div>`
318	ss.List += s.NavBase(DeviationList{	318	ss.List += s.NavBase(DeviationList{

config.example.json +1 −1

	@@ -10,7 +10,7 @@
10	"update-interval": 5	10	"update-interval": 5
11	},	11	},
12	"static-path": "static",	12	"static-path": "static",
13	"download-proxy": "http://127.0.0.1:8080",	13	"download-proxy": "",
14	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",	14	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
15	"proxy": true,	15	"proxy": true,
16	"nsfw": false	16	"nsfw": false

main.go +5 −1

 func main() {
 	app.Release.Version = "1.3.2"
 	app.Release.Description = "Two API endpoints and template embedding into binary"
-	go app.RefreshInstances()
 	app.ExecuteCommandLineArguments()
 	app.ExecuteConfig()
 	// can't exhaust the process or get our egress IP banned by CloudFront/WAF.
 	app.InstallDAThrottle()
+	// Only once the config is loaded and the throttle installed: this fetches over
+	// the network, so starting it earlier both raced ExecuteConfig's writes to CFG
+	// and let the request escape the throttle and the configured User-Agent.
+	go app.RefreshInstances()
 	go func() {
 		for {
 			err := devianter.UpdateCSRF()

Commit 83d9cd0b7a

Commit `83d9cd0b7a`