krz/devianter
A DeviantArt guest API library for Go.
clone: git clone https://gitbay.org/krz/devianter.git
1044b66ee7180d28ade26c8174f3c5e2e1430b16
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-07-14T23:32:39Z
deviantion.go | 2 +- go.mod | 2 +- util.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++----- util_test.go | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 10 deletions(-) @@ -142,7 +142,7 @@ func GetDeviation(id string, user string) (st Post, err Error) { // базовая обработка описания txt := st.Deviation.TextContent.Html.Markup - if len(txt) > 0 && txt[1] == '{' { + if len(txt) > 1 && txt[1] == '{' { var description struct { Blocks []struct { Text string @@ -1,3 +1,3 @@ -module git.macaw.me/skunky/devianter +module github.com/zerolabsco/devianter go 1.18 @@ -5,7 +5,9 @@ import ( "errors" "io" "net/http" + "strconv" "strings" + "time" ) // функция для высера ошибки в stderr @@ -31,7 +33,13 @@ type Error struct { func APIError(inputError error) (err Error) { if inputError != nil { err.RAW = []byte(inputError.Error()) - try(json.Unmarshal(err.RAW, &err)) + // DA's API errors are JSON. Anything else (CDN block pages, transport + // failures) is surfaced as-is rather than spamming a JSON parse error — + // this is what used to print `invalid character '<'` on every page. + if json.Unmarshal(err.RAW, &err) != nil { + err.Reason = "request_failed" + err.Error = inputError.Error() + } } return } @@ -43,18 +51,30 @@ type reqrt struct { Status int Cookies []*http.Cookie Headers http.Header + // Err is set when the request never completed (transport error). Status is 0. + Err error } // функция для совершения запроса var UserAgent string +// Timeout bounds a single request end-to-end (dial, response, body read). +// Without it, a hung connection blocks its caller forever. +var Timeout = 30 * time.Second + func request(uri string, other ...string) reqrt { var r reqrt // создаём новый запрос - cli := &http.Client{} + // Transport is deliberately left nil so http.DefaultTransport applies: that + // keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit). + cli := &http.Client{Timeout: Timeout} req, e := http.NewRequest("GET", uri, nil) - try(e) + if e != nil { + try(e) + r.Err = e + return r + } req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") @@ -67,11 +87,20 @@ func request(uri string, other ...string) reqrt { } resp, e := cli.Do(req) - try(e) + if e != nil { + // resp is nil on error: returning here avoids dereferencing it, which + // used to panic and (from UpdateCSRF's goroutine) kill the process. + try(e) + r.Err = e + return r + } defer resp.Body.Close() body, e := io.ReadAll(resp.Body) - try(e) + if e != nil { + try(e) + r.Err = e + } // заполняем структуру r.Body = string(body) @@ -82,11 +111,44 @@ func request(uri string, other ...string) reqrt { return r } +// looksLikeJSON reports whether a response is actually JSON, so an HTML page from +// a CDN/edge never reaches json.Unmarshal. +func looksLikeJSON(r reqrt) bool { + if ct := r.Headers.Get("Content-Type"); ct != "" && !strings.Contains(ct, "json") { + return false + } + b := strings.TrimSpace(r.Body) + return len(b) > 0 && (b[0] == '{' || b[0] == '[') +} + +// describe renders a failed response as a readable message, instead of the opaque +// `invalid character '<'` you get from json.Unmarshal on an HTML error page. +func describe(r reqrt) string { + body := strings.TrimSpace(r.Body) + if looksLikeJSON(r) { + return body // DA's own JSON error; callers unmarshal it into Error + } + + msg := "devianter: HTTP " + strconv.Itoa(r.Status) + " non-JSON response from DeviantArt" + if strings.Contains(body, "Generated by cloudfront") || strings.Contains(body, "Request blocked") { + msg += ": blocked by CloudFront/WAF — this egress IP is likely banned" + } + if len(body) > 200 { + body = body[:200] + "..." + } + return msg + " — " + body +} + /* PUPPY aka DeviantArt API */ // получение или обновление токена var cookie string var token string +const ( + csrfPrefix = "window.__CSRF_TOKEN__ = '" + xhrMarker = "window.__XHR_LOCAL__" +) + func UpdateCSRF() error { if cookie == "" { req := request("https://www.deviantart.com/_puppy") @@ -97,10 +159,25 @@ func UpdateCSRF() error { } req := request("https://www.deviantart.com", cookie) + if req.Err != nil { + return req.Err + } if req.Status != 200 { - return errors.New(req.Body) + return errors.New(describe(req)) } - token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] + + // Bounds-check the markers. On a block/challenge page they are absent, and the + // old arithmetic sliced Body[24:-4] — a panic that killed the whole process. + start, end := strings.Index(req.Body, csrfPrefix), strings.Index(req.Body, xhrMarker) + if start < 0 || end < 0 { + return errors.New("devianter: CSRF token not found in homepage (blocked, challenged, or markup changed)") + } + start += len(csrfPrefix) + end -= 3 + if end <= start || end > len(req.Body) { + return errors.New("devianter: CSRF token markers out of order (markup changed)") + } + token = req.Body[start:end] return nil } @@ -114,10 +191,18 @@ func puppy(data string) (string, error) { url.WriteString("&da_minor_version=20230710") body := request(url.String(), cookie) + if body.Err != nil { + return "", body.Err + } // если код ответа не 200, возвращается ошибка if body.Status != 200 { - return "", errors.New(body.Body) + return "", errors.New(describe(body)) + } + + // A 200 that isn't JSON means an edge/CDN page slipped through. + if !looksLikeJSON(body) { + return "", errors.New(describe(body)) } return body.Body, nil new file mode 100644 @@ -0,0 +1,84 @@ +package devianter + +import ( + "net/http" + "strings" + "testing" +) + +// Regression: request() used to call try(e) and then dereference resp (nil on a +// transport error), panicking. From UpdateCSRF's goroutine that panic was +// unrecovered and killed the whole process, so the container crash-looped. +func TestRequestTransportFailureDoesNotPanic(t *testing.T) { + // Port 1 on loopback: nothing listening, so the dial fails fast. + r := request("http://127.0.0.1:1/nope") + + if r.Err == nil { + t.Fatal("expected Err to be set on a transport failure") + } + if r.Status != 0 { + t.Fatalf("expected Status 0 on a failed request, got %d", r.Status) + } + if r.Body != "" { + t.Fatalf("expected empty Body on a failed request, got %q", r.Body) + } +} + +func TestLooksLikeJSON(t *testing.T) { + jsonResp := reqrt{Body: `{"ok":true}`, Headers: http.Header{}} + jsonResp.Headers.Set("Content-Type", "application/json; charset=utf-8") + if !looksLikeJSON(jsonResp) { + t.Error("a JSON body with a JSON content-type should look like JSON") + } + + htmlResp := reqrt{Body: "<!DOCTYPE HTML><html>nope</html>", Headers: http.Header{}} + htmlResp.Headers.Set("Content-Type", "text/html") + if looksLikeJSON(htmlResp) { + t.Error("an HTML error page must never be treated as JSON") + } +} + +// A CloudFront block is the exact failure that produced `invalid character '<'`; +// it should now be reported in plain language. +func TestDescribeDetectsCloudFrontBlock(t *testing.T) { + r := reqrt{ + Status: 403, + Body: "<!DOCTYPE HTML><HTML><H1>403 ERROR</H1>Request blocked.\nGenerated by cloudfront (CloudFront)", + Headers: http.Header{}, + } + r.Headers.Set("Content-Type", "text/html") + + msg := describe(r) + if !strings.Contains(msg, "CloudFront/WAF") { + t.Errorf("want a CloudFront/WAF hint, got %q", msg) + } + if !strings.Contains(msg, "403") { + t.Errorf("want the HTTP status in the message, got %q", msg) + } +} + +// DA's own errors are JSON and must pass through intact for callers to unmarshal. +func TestDescribePassesThroughAPIJSON(t *testing.T) { + body := `{"error":"invalid_request","errorDescription":"Invalid or expired form submission"}` + r := reqrt{Status: 400, Body: body, Headers: http.Header{}} + r.Headers.Set("Content-Type", "application/json") + + if got := describe(r); got != body { + t.Errorf("JSON API errors should pass through unchanged:\n got %q\nwant %q", got, body) + } +} + +// APIError must not emit a JSON parse error for a non-JSON (e.g. CDN block) body. +func TestAPIErrorHandlesNonJSON(t *testing.T) { + e := APIError(&stringErr{"devianter: HTTP 403 non-JSON response — blocked"}) + if e.Reason != "request_failed" { + t.Errorf("want Reason=request_failed for non-JSON errors, got %q", e.Reason) + } + if !strings.Contains(e.Error, "blocked") { + t.Errorf("want the underlying message preserved, got %q", e.Error) + } +} + +type stringErr struct{ s string } + +func (e *stringErr) Error() string { return e.s }