Commit `83d9cd0b7a`

83d9cd0b7a3e0ec6beb5c889102211567a6db03f

parent: 8d08f343c9

Verified · cmc

cmc <hello@cleberg.net> · 2026-07-15 08:17 UTC

fix: serve media again by unsetting download-proxy and scoping Host per request

Two independent faults made every image fail while pages still rendered.

config.example.json shipped download-proxy=http://127.0.0.1:8080. Only media
fetches go through that proxy — pages reach DeviantArt via devianter on the
default transport — so when nothing listens there, images 502 and the rest of
the page looks fine. In a scratch container 127.0.0.1 is the container itself,
so the default could never work under Docker. Unset it and document that it must
stay empty unless an operator really runs a proxy.

Host was a package global reassigned by every request, so a concurrent request
could overwrite it mid-render and emit URLs on another origin's host and port.
The instance's own default-src 'self' CSP then blocked those images. Thread the
request's host through skunkyart instead, and take it as an explicit argument in
URLBuilder, ParseMedia, ParseDescription, BuildUserPlate and
ConvertDeviantArtURLToSkunkyArt. Feeds keep their absolute URLs.

Also start RefreshInstances after ExecuteConfig rather than before it: the
goroutine read CFG while json.Unmarshal was writing it (a race the detector
flags), and its fetch escaped both the throttle and the configured User-Agent.

Verified: 300 concurrent requests with distinct Host headers now round-trip
their own host (was 1 leak per 300), go test -race is clean, and
cache+proxy both enabled serves 200 image/jpeg cold and from cache.

Layout: unified · split

SETUP.md +10 −1

		@@ -17,8 +17,17 @@ Time units:
17	17	* `max-size` — Maximum file size in megabytes
18	18	* `update-interval` — Automatic rotation interval
19	19	* `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag.
20		* `download-proxy` — Proxy address for downloading files.
	20	* `download-proxy` — Outbound proxy used when fetching media from DeviantArt's
	21	CDN. Leave empty (`""`) unless you actually run a proxy: if this points at
	22	something that isn't listening, every image 502s while pages still render,
	23	because only media fetches go through it. Inside a container `127.0.0.1` is
	24	the container itself, so a host-side proxy must be addressed by service name
	25	or host IP, not loopback.
21	26	* `user-agent` — String, which SkunkyArt uses as UA
	27	* `proxy` — Serve media through this instance instead of linking straight to
	28	DeviantArt's CDN. Required by `cache`; when off, clients fetch images from
	29	wixmp directly.
	30	* `nsfw` — Show mature content.
22	31
23	32	# Setting up reverse proxy
24	33	Pretty much business as usual, except for the [`X-Forwarded-Proto`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto) header setting.

app/parsers.go +14 −13

		@@ -32,9 +32,9 @@ func (s skunkyart) ParseComments(c devianter.Comments, daError devianter.Error)
32	32	cmmts.WriteString(`"><p id="`)
33	33	cmmts.WriteString(strconv.Itoa(x.ID))
34	34	cmmts.WriteString(`"><img src="`)
35		cmmts.WriteString(URLBuilder("media", "emojitar", x.User.Username, "?type=a"))
	35	cmmts.WriteString(URLBuilder(s.Host, "media", "emojitar", x.User.Username, "?type=a"))
36	36	cmmts.WriteString(`" width="30px" height="30px"><a href="`)
37		cmmts.WriteString(URLBuilder("group_user", "?q=", x.User.Username, "&type=a"))
	37	cmmts.WriteString(URLBuilder(s.Host, "group_user", "?q=", x.User.Username, "&type=a"))
38	38	cmmts.WriteString(`"><b`)
39	39	cmmts.WriteString(` class="`)
40	40	if x.User.Banned {
		@@ -64,7 +64,7 @@ func (s skunkyart) ParseComments(c devianter.Comments, daError devianter.Error)
64	64	cmmts.WriteString(x.Posted.UTC().String())
65	65	cmmts.WriteString("]<p>")
66	66
67		cmmts.WriteString(ParseDescription(x.TextContent))
	67	cmmts.WriteString(ParseDescription(s.Host, x.TextContent))
68	68	cmmts.WriteString("<p>👍: ")
69	69	cmmts.WriteString(strconv.Itoa(x.Likes))
70	70	cmmts.WriteString(" ⏩: ")
		@@ -92,7 +92,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con
92	92
93	93	for i, l := 0, len(devs); i < l; i++ {
94	94	data := &devs[i]
95		if preview, fullview := ParseMedia(data.Media, 320), ParseMedia(data.Media); !data.NSFW \|\| CFG.Nsfw {
	95	if preview, fullview := ParseMedia(s.Host, data.Media, 320), ParseMedia(s.Host, data.Media); !data.NSFW \|\| CFG.Nsfw {
96	96	if allowAtom && s.Atom {
97	97	s.Writer.Header().Add("Content-Type", "application/atom+xml")
98	98	id := strconv.Itoa(data.ID)
		@@ -101,7 +101,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con
101	101	listContent.WriteString(`</name></author><title>`)
102	102	listContent.WriteString(data.Title)
103	103	listContent.WriteString(`</title><link rel="alternate" type="text/html" href="`)
104		listContent.WriteString(URLBuilder("post", data.Author.Username, "atom-"+id))
	104	listContent.WriteString(URLBuilder(s.Host, "post", data.Author.Username, "atom-"+id))
105	105	listContent.WriteString(`"/><id>`)
106	106	listContent.WriteString(id)
107	107	listContent.WriteString(`</id><published>`)
		@@ -112,11 +112,11 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con
112	112	listContent.WriteString(`</media:title><media:thumbinal url="`)
113	113	listContent.WriteString(preview)
114	114	listContent.WriteString(`"/></media:group><content type="xhtml"><div xmlns="http://www.w3.org/1999/xhtml"><a href="`)
115		listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(data.Url))
	115	listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url))
116	116	listContent.WriteString(`"><img src="`)
117	117	listContent.WriteString(fullview)
118	118	listContent.WriteString(`"/></a><p>`)
119		listContent.WriteString(ParseDescription(data.TextContent))
	119	listContent.WriteString(ParseDescription(s.Host, data.TextContent))
120	120	listContent.WriteString(`</p></div></content></entry>`)
121	121	} else {
122	122	listContent.WriteString(`<div class="block">`)
		@@ -130,7 +130,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con
130	130	listContent.WriteString(`<h1>[ TEXT ]</h1>`)
131	131	}
132	132	listContent.WriteString(`<br><a href="`)
133		listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(data.Url))
	133	listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url))
134	134	listContent.WriteString(`">`)
135	135	listContent.WriteString(data.Author.Username)
136	136	listContent.WriteString(" - ")
		@@ -166,7 +166,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con
166	166	list.WriteString(`</title>`)
167	167
168	168	list.WriteString(`<link rel="alternate" href="`)
169		list.WriteString(Host)
	169	list.WriteString(s.Host)
170	170	list.WriteString(`"/>`)
171	171
172	172	list.WriteString(listContent.String())
		@@ -201,9 +201,10 @@ type text struct {
201	201	// ParseDescription renders a DeviantArt description into HTML, handling both the
202	202	// Draft.js-style JSON payload and the plain HTML markup DeviantArt returns, and
203	203	// rewriting embedded links and artwork references to point at this instance.
	204	// host is the request's scheme and host, as taken by URLBuilder.
204	205	//
205	206	// TODO: rewrite this whole mess.
206		func ParseDescription(dscr devianter.Text) string {
	207	func ParseDescription(host string, dscr devianter.Text) string {
207	208	var parsedDescription strings.Builder
208	209	TagBuilder := func(content string, tags ...string) string {
209	210	l := len(tags)
		@@ -308,9 +309,9 @@ func ParseDescription(dscr devianter.Text) string {
308	309	if len(x.EntityRanges) != 0 {
309	310	d := entities[x.EntityRanges[0].Key]
310	311	parsedDescription.WriteString(`<a href="`)
311		parsedDescription.WriteString(ConvertDeviantArtURLToSkunkyArt(d.Url))
	312	parsedDescription.WriteString(ConvertDeviantArtURLToSkunkyArt(host, d.Url))
312	313	parsedDescription.WriteString(`"><img width="50%" src="`)
313		parsedDescription.WriteString(ParseMedia(d.Media))
	314	parsedDescription.WriteString(ParseMedia(host, d.Media))
314	315	parsedDescription.WriteString(`" title="`)
315	316	parsedDescription.WriteString(d.Author.Username)
316	317	parsedDescription.WriteString(" - ")
		@@ -372,7 +373,7 @@ func ParseDescription(dscr devianter.Text) string {
372	373	switch a.Key {
373	374	case "src":
374	375	if len(a.Val) > 9 && a.Val[8:9] == "e" {
375		uri = URLBuilder("media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
	376	uri = URLBuilder(host, "media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e")
376	377	}
377	378	case "title":
378	379	title = a.Val

app/router.go +9 −7

		@@ -10,10 +10,6 @@ import (
10	10	"time"
11	11	)
12	12
13		// Host is the scheme and host that generated links are built from. It is set per
14		// request from the Host header and X-Forwarded-Proto.
15		var Host string
16
17	13	// Router registers the single catch-all handler that dispatches every path, then
18	14	// serves until the process exits. It does not return on success.
19	15	func Router() {
		@@ -68,12 +64,18 @@ func Router() {
68	64	// the function that drives everything
69	65	handle := func(w http.ResponseWriter, r *http.Request) {
70	66	path := parsepath(r.URL.Path)
71		Host = "http://" + r.Host
	67
	68	// Per-request, not a package global: requests arrive concurrently on
	69	// different hosts and ports (bots hitting a proxy's alternate ports, for
	70	// one), and a shared global lets one request's host leak into another's
	71	// rendered URLs. Those URLs then point at a different origin, which this
	72	// handler's own default-src 'self' CSP blocks.
	73	host := "http://" + r.Host
72	74	if h := r.Header["X-Forwarded-Proto"]; len(h) != 0 && h[0] == "https" {
73		Host = "https://" + r.Host
	75	host = "https://" + r.Host
74	76	}
75	77
76		var skunky = skunkyart{Version: Release.Version}
	78	var skunky = skunkyart{Version: Release.Version, Host: host}
77	79	skunky._pth = r.URL.Path
78	80
79	81	skunky.Args = r.URL.Query()

app/util.go +25 −16

		@@ -84,6 +84,11 @@ type skunkyart struct {
84	84	Type rune
85	85	Atom bool
86	86
	87	// Host is the scheme and host this request arrived on, e.g.
	88	// "https://art.example.com". It is per-request rather than global because
	89	// concurrent requests can arrive on different hosts and ports.
	90	Host string
	91
87	92	BasePath, Endpoint string
88	93	Query, QueryRaw string
89	94
		@@ -147,13 +152,15 @@ func (s skunkyart) ExecuteTemplate(file, dir string, data any) {
147	152	wr(s.Writer, buf.String())
148	153	}
149	154
150		// URLBuilder joins strs into an absolute instance URL, prefixing the current
151		// Host and configured URI and inserting slashes between path segments but not
152		// before query separators.
153		func URLBuilder(strs ...string) string {
	155	// URLBuilder joins strs into an absolute instance URL, prefixing host and the
	156	// configured URI and inserting slashes between path segments but not before
	157	// query separators. host is the request's own scheme and host: passing the
	158	// wrong one emits links to another origin, which the instance's own
	159	// Content-Security-Policy then blocks.
	160	func URLBuilder(host string, strs ...string) string {
154	161	var str strings.Builder
155	162	l := len(strs)
156		str.WriteString(Host)
	163	str.WriteString(host)
157	164	str.WriteString(CFG.URI)
158	165	for n, x := range strs {
159	166	str.WriteString(x)
		@@ -170,7 +177,7 @@ func (s skunkyart) Error(dAerr devianter.Error) {
170	177
171	178	var msg strings.Builder
172	179	msg.WriteString(`<html><link rel="stylesheet" href="`)
173		msg.WriteString(URLBuilder("stylesheet"))
	180	msg.WriteString(URLBuilder(s.Host, "stylesheet"))
174	181	msg.WriteString(`" /><h3>DeviantArt error — '`)
175	182	msg.WriteString(dAerr.Error)
176	183	msg.WriteString("'</h3></html>")
		@@ -189,7 +196,7 @@ func (s skunkyart) ReturnHTTPError(status int) {
189	196
190	197	var msg strings.Builder
191	198	msg.WriteString(`<html><link rel="stylesheet" href="`)
192		msg.WriteString(URLBuilder("stylesheet"))
	199	msg.WriteString(URLBuilder(s.Host, "stylesheet"))
193	200	msg.WriteString(`" /><h1>`)
194	201	msg.WriteString(strconv.Itoa(status))
195	202	msg.WriteString(" - ")
		@@ -264,7 +271,8 @@ func Download(urlString string) (d Downloaded) {
264	271	// ParseMedia returns the URL to serve for media: a link back through this
265	272	// instance's media proxy when proxying is on, or DeviantArt's own URL when it is
266	273	// off. An optional thumb width selects a thumbnail instead of the full image.
267		func ParseMedia(media devianter.Media, thumb ...int) string {
	274	// host is the request's scheme and host, as taken by URLBuilder.
	275	func ParseMedia(host string, media devianter.Media, thumb ...int) string {
268	276	mediaURL, filename := devianter.UrlFromMedia(media, thumb...)
269	277	if len(mediaURL) != 0 && CFG.Proxy {
270	278	mediaURL = mediaURL[21:]
		@@ -272,7 +280,7 @@ func ParseMedia(media devianter.Media, thumb ...int) string {
272	280	if filename == "" {
273	281	filename = "image.gif"
274	282	}
275		return URLBuilder("media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename)
	283	return URLBuilder(host, "media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename)
276	284	} else if !CFG.Proxy {
277	285	return mediaURL
278	286	}
		@@ -281,27 +289,28 @@ func ParseMedia(media devianter.Media, thumb ...int) string {
281	289
282	290	// ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link into the
283	291	// equivalent link on this instance. It returns an empty string for URLs it does
284		// not handle, including sta.sh links.
285		func ConvertDeviantArtURLToSkunkyArt(url string) (output string) {
	292	// not handle, including sta.sh links. host is the request's scheme and host, as
	293	// taken by URLBuilder.
	294	func ConvertDeviantArtURLToSkunkyArt(host, url string) (output string) {
286	295	if len(url) > 32 && url[27:32] != "stash" {
287	296	url = url[27:]
288	297	firstshash := strings.Index(url, "/")
289	298	lastshash := firstshash + strings.Index(url[firstshash+1:], "/")
290	299	if lastshash != -1 {
291		output = URLBuilder("post", url[:firstshash], url[lastshash+2:])
	300	output = URLBuilder(host, "post", url[:firstshash], url[lastshash+2:])
292	301	}
293	302	}
294	303	return
295	304	}
296	305
297	306	// BuildUserPlate renders the small avatar-and-username block linking to a user's
298		// about page.
299		func BuildUserPlate(name string) string {
	307	// about page. host is the request's scheme and host, as taken by URLBuilder.
	308	func BuildUserPlate(host, name string) string {
300	309	var htm strings.Builder
301	310	htm.WriteString(`<div class="user-plate"><img src="`)
302		htm.WriteString(URLBuilder("media", "emojitar", name, "?type=a"))
	311	htm.WriteString(URLBuilder(host, "media", "emojitar", name, "?type=a"))
303	312	htm.WriteString(`"><a href="`)
304		htm.WriteString(URLBuilder("group_user", "?type=about&q=", name))
	313	htm.WriteString(URLBuilder(host, "group_user", "?type=about&q=", name))
305	314	htm.WriteString(`">`)
306	315	htm.WriteString(name)
307	316	htm.WriteString(`</a></div>`)

app/wrapper.go +13 −13

		@@ -42,12 +42,12 @@ func (s skunkyart) GRUser() {
42	42	var about = &x.ModuleData.GroupAbout
43	43	group.Group = true
44	44	group.CreationDate = x.ModuleData.GroupAbout.FoundatedAt.UTC().String()
45		group.About.DescriptionFormatted = ParseDescription(about.Description)
	45	group.About.DescriptionFormatted = ParseDescription(s.Host, about.Description)
46	46	} else if false {
47	47	group.About.A = x.ModuleData.About
48	48	var about = &group.About.A
49	49	group.CreationDate = time.Unix(time.Now().Unix()-x.ModuleData.About.RegDate, 0).UTC().String()
50		group.About.DescriptionFormatted = ParseDescription(about.Description)
	50	group.About.DescriptionFormatted = ParseDescription(s.Host, about.Description)
51	51
52	52	for _, val := range x.ModuleData.About.SocialLinks {
53	53	var social strings.Builder
		@@ -72,12 +72,12 @@ func (s skunkyart) GRUser() {
72	72
73	73	case "cover_deviation":
74	74	group.About.BGMeta = x.ModuleData.CoverDeviation.Deviation
75		group.About.BGMeta.Url = ConvertDeviantArtURLToSkunkyArt(group.About.BGMeta.Url)
76		group.About.BG = ParseMedia(group.About.BGMeta.Media)
	75	group.About.BGMeta.Url = ConvertDeviantArtURLToSkunkyArt(s.Host, group.About.BGMeta.Url)
	76	group.About.BG = ParseMedia(s.Host, group.About.BGMeta.Media)
77	77	case "group_admins":
78	78	var htm strings.Builder
79	79	for _, z := range x.ModuleData.GroupAdmins.Results {
80		htm.WriteString(BuildUserPlate(z.User.Username))
	80	htm.WriteString(BuildUserPlate(s.Host, z.User.Username))
81	81	}
82	82	group.Admins += htm.String()
83	83	}
		@@ -124,9 +124,9 @@ func (s skunkyart) GRUser() {
124	124
125	125	if !x.Thumb.NSFW \|\| CFG.Nsfw {
126	126	folders.WriteString(`<a href="`)
127		folders.WriteString(ConvertDeviantArtURLToSkunkyArt(x.Thumb.Url))
	127	folders.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, x.Thumb.Url))
128	128	folders.WriteString(`"><img loading="lazy" src="`)
129		folders.WriteString(ParseMedia(x.Thumb.Media))
	129	folders.WriteString(ParseMedia(s.Host, x.Thumb.Media))
130	130	folders.WriteString(`" title="`)
131	131	folders.WriteString(x.Thumb.Title)
132	132	folders.WriteString(`"></a>`)
		@@ -191,7 +191,7 @@ func (s skunkyart) Deviation(author, postname string) {
191	191	if post.Post.Deviation.NSFW && !CFG.Nsfw {
192	192	s.Writer.WriteHeader(403)
193	193	wr(s.Writer, `<html><link rel="stylesheet" href="`+
194		URLBuilder("stylesheet")+
	194	URLBuilder(s.Host, "stylesheet")+
195	195	`" /><h1>NSFW content are disabled on this instance.</h1></html>`)
196	196	return
197	197	}
		@@ -201,9 +201,9 @@ func (s skunkyart) Deviation(author, postname string) {
201	201	}
202	202
203	203	if post.Post.Deviation.TextContent.Excerpt != "" {
204		post.Post.Description = ParseDescription(post.Post.Deviation.TextContent)
	204	post.Post.Description = ParseDescription(s.Host, post.Post.Deviation.TextContent)
205	205	} else {
206		post.Post.Description = ParseDescription(post.Post.Deviation.Extended.DescriptionText)
	206	post.Post.Description = ParseDescription(s.Host, post.Post.Deviation.Extended.DescriptionText)
207	207	}
208	208
209	209	for _, x := range post.Post.Deviation.Extended.RelatedContent {
		@@ -216,7 +216,7 @@ func (s skunkyart) Deviation(author, postname string) {
216	216	for _, x := range post.Post.Deviation.Extended.Tags {
217	217	var tag strings.Builder
218	218	tag.WriteString(` <a href="`)
219		tag.WriteString(URLBuilder("search", "?q=", x.Name, "&type=tag"))
	219	tag.WriteString(URLBuilder(s.Host, "search", "?q=", x.Name, "&type=tag"))
220	220	tag.WriteString(`">#`)
221	221	tag.WriteString(x.Name)
222	222	tag.WriteString("</a>")
		@@ -226,7 +226,7 @@ func (s skunkyart) Deviation(author, postname string) {
226	226
227	227	post.Comments = s.ParseComments(devianter.GetComments(id, post.Post.Comments.Cursor, s.Page, 1))
228	228	post.StringTime = post.Post.Deviation.PublishedTime.UTC().String()
229		post.Post.IMG = ParseMedia(post.Post.Deviation.Media)
	229	post.Post.IMG = ParseMedia(s.Host, post.Post.Deviation.Media)
230	230
231	231	s.ExecuteTemplate("deviantion.htm", "html", &s)
232	232	}
		@@ -312,7 +312,7 @@ func (s skunkyart) Search() {
312	312	if l := len(usernames); l != 0 {
313	313	ss.List += `<div class="content plates">`
314	314	for x := range len(usernames) {
315		ss.List += BuildUserPlate(usernames[x])
	315	ss.List += BuildUserPlate(s.Host, usernames[x])
316	316	}
317	317	ss.List += `</div>`
318	318	ss.List += s.NavBase(DeviationList{

config.example.json +1 −1

		@@ -10,7 +10,7 @@
10	10	"update-interval": 5
11	11	},
12	12	"static-path": "static",
13		"download-proxy": "http://127.0.0.1:8080",
	13	"download-proxy": "",
14	14	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
15	15	"proxy": true,
16	16	"nsfw": false

main.go +5 −1

		@@ -11,7 +11,6 @@ import (
11	11	func main() {
12	12	app.Release.Version = "1.3.2"
13	13	app.Release.Description = "Two API endpoints and template embedding into binary"
14		go app.RefreshInstances()
15	14
16	15	app.ExecuteCommandLineArguments()
17	16	app.ExecuteConfig()
		@@ -21,6 +20,11 @@ func main() {
21	20	// can't exhaust the process or get our egress IP banned by CloudFront/WAF.
22	21	app.InstallDAThrottle()
23	22
	23	// Only once the config is loaded and the throttle installed: this fetches over
	24	// the network, so starting it earlier both raced ExecuteConfig's writes to CFG
	25	// and let the request escape the throttle and the configured User-Agent.
	26	go app.RefreshInstances()
	27
24	28	go func() {
25	29	for {
26	30	err := devianter.UpdateCSRF()

Commit 83d9cd0b7a

Commit `83d9cd0b7a`