Commit `1044b66ee7`

1044b66ee7180d28ade26c8174f3c5e2e1430b16

parent: b3c99749f1

Verified · cmc

cmc <hello@cleberg.net> · 2026-07-14 23:32 UTC

fix: resolve nil-deref crash, CSRF panic, add timeouts + clear CDN-block errors; repoint module to zerolabsco

References: https://github.com/zerolabsco/skunky-art/issues/2

Layout: unified · split

deviantion.go +1 −1

		@@ -142,7 +142,7 @@ func GetDeviation(id string, user string) (st Post, err Error) {
142	142
143	143	// базовая обработка описания
144	144	txt := st.Deviation.TextContent.Html.Markup
145		if len(txt) > 0 && txt[1] == '{' {
	145	if len(txt) > 1 && txt[1] == '{' {
146	146	var description struct {
147	147	Blocks []struct {
148	148	Text string

go.mod +1 −1

		@@ -1,3 +1,3 @@
1		module git.macaw.me/skunky/devianter
	1	module github.com/zerolabsco/devianter
2	2
3	3	go 1.18

util.go +93 −8

		@@ -5,7 +5,9 @@ import (
5	5	"errors"
6	6	"io"
7	7	"net/http"
	8	"strconv"
8	9	"strings"
	10	"time"
9	11	)
10	12
11	13	// функция для высера ошибки в stderr
		@@ -31,7 +33,13 @@ type Error struct {
31	33	func APIError(inputError error) (err Error) {
32	34	if inputError != nil {
33	35	err.RAW = []byte(inputError.Error())
34		try(json.Unmarshal(err.RAW, &err))
	36	// DA's API errors are JSON. Anything else (CDN block pages, transport
	37	// failures) is surfaced as-is rather than spamming a JSON parse error —
	38	// this is what used to print `invalid character '<'` on every page.
	39	if json.Unmarshal(err.RAW, &err) != nil {
	40	err.Reason = "request_failed"
	41	err.Error = inputError.Error()
	42	}
35	43	}
36	44	return
37	45	}
		@@ -43,18 +51,30 @@ type reqrt struct {
43	51	Status int
44	52	Cookies []*http.Cookie
45	53	Headers http.Header
	54	// Err is set when the request never completed (transport error). Status is 0.
	55	Err error
46	56	}
47	57
48	58	// функция для совершения запроса
49	59	var UserAgent string
50	60
	61	// Timeout bounds a single request end-to-end (dial, response, body read).
	62	// Without it, a hung connection blocks its caller forever.
	63	var Timeout = 30 * time.Second
	64
51	65	func request(uri string, other ...string) reqrt {
52	66	var r reqrt
53	67
54	68	// создаём новый запрос
55		cli := &http.Client{}
	69	// Transport is deliberately left nil so http.DefaultTransport applies: that
	70	// keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit).
	71	cli := &http.Client{Timeout: Timeout}
56	72	req, e := http.NewRequest("GET", uri, nil)
57		try(e)
	73	if e != nil {
	74	try(e)
	75	r.Err = e
	76	return r
	77	}
58	78
59	79	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
60	80
		@@ -67,11 +87,20 @@ func request(uri string, other ...string) reqrt {
67	87	}
68	88
69	89	resp, e := cli.Do(req)
70		try(e)
	90	if e != nil {
	91	// resp is nil on error: returning here avoids dereferencing it, which
	92	// used to panic and (from UpdateCSRF's goroutine) kill the process.
	93	try(e)
	94	r.Err = e
	95	return r
	96	}
71	97	defer resp.Body.Close()
72	98
73	99	body, e := io.ReadAll(resp.Body)
74		try(e)
	100	if e != nil {
	101	try(e)
	102	r.Err = e
	103	}
75	104
76	105	// заполняем структуру
77	106	r.Body = string(body)
		@@ -82,11 +111,44 @@ func request(uri string, other ...string) reqrt {
82	111	return r
83	112	}
84	113
	114	// looksLikeJSON reports whether a response is actually JSON, so an HTML page from
	115	// a CDN/edge never reaches json.Unmarshal.
	116	func looksLikeJSON(r reqrt) bool {
	117	if ct := r.Headers.Get("Content-Type"); ct != "" && !strings.Contains(ct, "json") {
	118	return false
	119	}
	120	b := strings.TrimSpace(r.Body)
	121	return len(b) > 0 && (b[0] == '{' \|\| b[0] == '[')
	122	}
	123
	124	// describe renders a failed response as a readable message, instead of the opaque
	125	// `invalid character '<'` you get from json.Unmarshal on an HTML error page.
	126	func describe(r reqrt) string {
	127	body := strings.TrimSpace(r.Body)
	128	if looksLikeJSON(r) {
	129	return body // DA's own JSON error; callers unmarshal it into Error
	130	}
	131
	132	msg := "devianter: HTTP " + strconv.Itoa(r.Status) + " non-JSON response from DeviantArt"
	133	if strings.Contains(body, "Generated by cloudfront") \|\| strings.Contains(body, "Request blocked") {
	134	msg += ": blocked by CloudFront/WAF — this egress IP is likely banned"
	135	}
	136	if len(body) > 200 {
	137	body = body[:200] + "..."
	138	}
	139	return msg + " — " + body
	140	}
	141
85	142	/* PUPPY aka DeviantArt API */
86	143	// получение или обновление токена
87	144	var cookie string
88	145	var token string
89	146
	147	const (
	148	csrfPrefix = "window.__CSRF_TOKEN__ = '"
	149	xhrMarker = "window.__XHR_LOCAL__"
	150	)
	151
90	152	func UpdateCSRF() error {
91	153	if cookie == "" {
92	154	req := request("https://www.deviantart.com/_puppy")
		@@ -97,10 +159,25 @@ func UpdateCSRF() error {
97	159	}
98	160
99	161	req := request("https://www.deviantart.com", cookie)
	162	if req.Err != nil {
	163	return req.Err
	164	}
100	165	if req.Status != 200 {
101		return errors.New(req.Body)
	166	return errors.New(describe(req))
102	167	}
103		token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3]
	168
	169	// Bounds-check the markers. On a block/challenge page they are absent, and the
	170	// old arithmetic sliced Body[24:-4] — a panic that killed the whole process.
	171	start, end := strings.Index(req.Body, csrfPrefix), strings.Index(req.Body, xhrMarker)
	172	if start < 0 \|\| end < 0 {
	173	return errors.New("devianter: CSRF token not found in homepage (blocked, challenged, or markup changed)")
	174	}
	175	start += len(csrfPrefix)
	176	end -= 3
	177	if end <= start \|\| end > len(req.Body) {
	178	return errors.New("devianter: CSRF token markers out of order (markup changed)")
	179	}
	180	token = req.Body[start:end]
104	181
105	182	return nil
106	183	}
		@@ -114,10 +191,18 @@ func puppy(data string) (string, error) {
114	191	url.WriteString("&da_minor_version=20230710")
115	192
116	193	body := request(url.String(), cookie)
	194	if body.Err != nil {
	195	return "", body.Err
	196	}
117	197
118	198	// если код ответа не 200, возвращается ошибка
119	199	if body.Status != 200 {
120		return "", errors.New(body.Body)
	200	return "", errors.New(describe(body))
	201	}
	202
	203	// A 200 that isn't JSON means an edge/CDN page slipped through.
	204	if !looksLikeJSON(body) {
	205	return "", errors.New(describe(body))
121	206	}
122	207
123	208	return body.Body, nil

util_test.go added +84

		@@ -0,0 +1,84 @@
	1	package devianter
	2
	3	import (
	4	"net/http"
	5	"strings"
	6	"testing"
	7	)
	8
	9	// Regression: request() used to call try(e) and then dereference resp (nil on a
	10	// transport error), panicking. From UpdateCSRF's goroutine that panic was
	11	// unrecovered and killed the whole process, so the container crash-looped.
	12	func TestRequestTransportFailureDoesNotPanic(t *testing.T) {
	13	// Port 1 on loopback: nothing listening, so the dial fails fast.
	14	r := request("http://127.0.0.1:1/nope")
	15
	16	if r.Err == nil {
	17	t.Fatal("expected Err to be set on a transport failure")
	18	}
	19	if r.Status != 0 {
	20	t.Fatalf("expected Status 0 on a failed request, got %d", r.Status)
	21	}
	22	if r.Body != "" {
	23	t.Fatalf("expected empty Body on a failed request, got %q", r.Body)
	24	}
	25	}
	26
	27	func TestLooksLikeJSON(t *testing.T) {
	28	jsonResp := reqrt{Body: `{"ok":true}`, Headers: http.Header{}}
	29	jsonResp.Headers.Set("Content-Type", "application/json; charset=utf-8")
	30	if !looksLikeJSON(jsonResp) {
	31	t.Error("a JSON body with a JSON content-type should look like JSON")
	32	}
	33
	34	htmlResp := reqrt{Body: "<!DOCTYPE HTML><html>nope</html>", Headers: http.Header{}}
	35	htmlResp.Headers.Set("Content-Type", "text/html")
	36	if looksLikeJSON(htmlResp) {
	37	t.Error("an HTML error page must never be treated as JSON")
	38	}
	39	}
	40
	41	// A CloudFront block is the exact failure that produced `invalid character '<'`;
	42	// it should now be reported in plain language.
	43	func TestDescribeDetectsCloudFrontBlock(t *testing.T) {
	44	r := reqrt{
	45	Status: 403,
	46	Body: "<!DOCTYPE HTML><HTML><H1>403 ERROR</H1>Request blocked.\nGenerated by cloudfront (CloudFront)",
	47	Headers: http.Header{},
	48	}
	49	r.Headers.Set("Content-Type", "text/html")
	50
	51	msg := describe(r)
	52	if !strings.Contains(msg, "CloudFront/WAF") {
	53	t.Errorf("want a CloudFront/WAF hint, got %q", msg)
	54	}
	55	if !strings.Contains(msg, "403") {
	56	t.Errorf("want the HTTP status in the message, got %q", msg)
	57	}
	58	}
	59
	60	// DA's own errors are JSON and must pass through intact for callers to unmarshal.
	61	func TestDescribePassesThroughAPIJSON(t *testing.T) {
	62	body := `{"error":"invalid_request","errorDescription":"Invalid or expired form submission"}`
	63	r := reqrt{Status: 400, Body: body, Headers: http.Header{}}
	64	r.Headers.Set("Content-Type", "application/json")
	65
	66	if got := describe(r); got != body {
	67	t.Errorf("JSON API errors should pass through unchanged:\n got %q\nwant %q", got, body)
	68	}
	69	}
	70
	71	// APIError must not emit a JSON parse error for a non-JSON (e.g. CDN block) body.
	72	func TestAPIErrorHandlesNonJSON(t *testing.T) {
	73	e := APIError(&stringErr{"devianter: HTTP 403 non-JSON response — blocked"})
	74	if e.Reason != "request_failed" {
	75	t.Errorf("want Reason=request_failed for non-JSON errors, got %q", e.Reason)
	76	}
	77	if !strings.Contains(e.Error, "blocked") {
	78	t.Errorf("want the underlying message preserved, got %q", e.Error)
	79	}
	80	}
	81
	82	type stringErr struct{ s string }
	83
	84	func (e *stringErr) Error() string { return e.s }

Commit 1044b66ee7

Commit `1044b66ee7`