krz/devianter

  1package devianter
  2
  3import (
  4	"encoding/json"
  5	"errors"
  6	"io"
  7	"net/http"
  8	"strconv"
  9	"strings"
 10	"time"
 11)
 12
 13// try prints a non-nil error to stderr and swallows it. It is how this package
 14// reports problems it does not propagate, such as a response that parsed only
 15// partially.
 16func try(txt error) {
 17	if txt != nil {
 18		println(txt.Error())
 19	}
 20}
 21
 22// ujson fetches a _puppy endpoint and unmarshals the response into output.
 23// data is the path and query string after the endpoint root, without a leading
 24// slash and without the csrf_token parameter, which puppy appends.
 25//
 26// A malformed response is reported through try and leaves output partially
 27// populated, so a returned Error with an empty Reason does not by itself
 28// guarantee that output is complete.
 29func ujson(data string, output any) Error {
 30	input, err := puppy(data)
 31	if err == nil {
 32		try(json.Unmarshal([]byte(input), output))
 33	}
 34	return APIError(err)
 35}
 36
 37// Error is a failed API call. It is a struct rather than an error interface, so
 38// a zero value means success: test Reason for emptiness rather than comparing
 39// against nil.
 40//
 41// For errors DeviantArt itself reports, Reason and Error hold its machine and
 42// human readable descriptions. For anything else (a transport failure, or a
 43// CloudFront block page) Reason is "request_failed" and Error carries the
 44// underlying message.
 45type Error struct {
 46	Reason string `json:"error"`
 47	Error  string `json:"errorDescription"`
 48	RAW    []byte `json:"-"`
 49}
 50
 51// APIError converts an error from the request layer into an [Error], decoding
 52// DeviantArt's JSON error body when that is what it is. A nil input yields the
 53// zero Error, which signals success.
 54func APIError(inputError error) (err Error) {
 55	if inputError != nil {
 56		err.RAW = []byte(inputError.Error())
 57		// DA's API errors are JSON. Anything else (CDN block pages, transport
 58		// failures) is surfaced as-is rather than spamming a JSON parse error —
 59		// this is what used to print `invalid character '<'` on every page.
 60		if json.Unmarshal(err.RAW, &err) != nil {
 61			err.Reason = "request_failed"
 62			err.Error = inputError.Error()
 63		}
 64	}
 65	return
 66}
 67
 68/* REQUEST SECTION */
 69// reqrt is a completed HTTP response, flattened into the pieces this package
 70// needs. On a transport failure Err is set and every other field is zero.
 71type reqrt struct {
 72	Body    string
 73	Status  int
 74	Cookies []*http.Cookie
 75	Headers http.Header
 76	// Err is set when the request never completed (transport error). Status is 0.
 77	Err error
 78}
 79
 80// UserAgent overrides the browser User-Agent this package sends by default.
 81// Setting it to something that identifies your client is polite, but DeviantArt
 82// is more likely to serve a block page to a non-browser agent.
 83var UserAgent string
 84
 85// Timeout bounds a single request end-to-end (dial, response, body read).
 86// Without it, a hung connection blocks its caller forever.
 87var Timeout = 30 * time.Second
 88
 89// request performs a GET and never panics or returns a partial response without
 90// saying so: any failure is reported in reqrt.Err. An optional second argument
 91// supplies the Cookie header.
 92func request(uri string, other ...string) reqrt {
 93	var r reqrt
 94
 95	// Transport is deliberately left nil so http.DefaultTransport applies: that
 96	// keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit).
 97	cli := &http.Client{Timeout: Timeout}
 98	req, e := http.NewRequest("GET", uri, nil)
 99	if e != nil {
100		try(e)
101		r.Err = e
102		return r
103	}
104
105	// Impersonate a browser by default: the endpoints are the web frontend's own,
106	// and an unfamiliar agent draws a block page.
107	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
108
109	if UserAgent != "" {
110		req.Header.Set("User-Agent", UserAgent)
111	}
112	if len(other) != 0 {
113		req.Header.Set("Cookie", other[0])
114	}
115
116	resp, e := cli.Do(req)
117	if e != nil {
118		// resp is nil on error: returning here avoids dereferencing it, which
119		// used to panic and (from UpdateCSRF's goroutine) kill the process.
120		try(e)
121		r.Err = e
122		return r
123	}
124	defer func() {
125		if err := resp.Body.Close(); err != nil {
126			try(err)
127		}
128	}()
129
130	body, e := io.ReadAll(resp.Body)
131	if e != nil {
132		try(e)
133		r.Err = e
134	}
135
136	r.Body = string(body)
137	r.Cookies = resp.Cookies()
138	r.Headers = resp.Header
139	r.Status = resp.StatusCode
140
141	return r
142}
143
144// looksLikeJSON reports whether a response is actually JSON, so an HTML page from
145// a CDN/edge never reaches json.Unmarshal.
146func looksLikeJSON(r reqrt) bool {
147	if ct := r.Headers.Get("Content-Type"); ct != "" && !strings.Contains(ct, "json") {
148		return false
149	}
150	b := strings.TrimSpace(r.Body)
151	return len(b) > 0 && (b[0] == '{' || b[0] == '[')
152}
153
154// describe renders a failed response as a readable message, instead of the opaque
155// `invalid character '<'` you get from json.Unmarshal on an HTML error page.
156func describe(r reqrt) string {
157	body := strings.TrimSpace(r.Body)
158	if looksLikeJSON(r) {
159		return body // DA's own JSON error; callers unmarshal it into Error
160	}
161
162	msg := "devianter: HTTP " + strconv.Itoa(r.Status) + " non-JSON response from DeviantArt"
163	if strings.Contains(body, "Generated by cloudfront") || strings.Contains(body, "Request blocked") {
164		msg += ": blocked by CloudFront/WAF — this egress IP is likely banned"
165	}
166	if len(body) > 200 {
167		body = body[:200] + "..."
168	}
169	return msg + " — " + body
170}
171
172/* PUPPY aka DeviantArt API */
173// The guest session: a cookie from the _puppy endpoint and a CSRF token scraped
174// from the homepage. UpdateCSRF populates both; puppy sends them on every call.
175//
176// These are package-level and unsynchronised, so a program that calls UpdateCSRF
177// concurrently with any other function of this package races on them.
178var cookie string
179var token string
180
181const (
182	csrfPrefix = "window.__CSRF_TOKEN__ = '"
183	xhrMarker  = "window.__XHR_LOCAL__"
184)
185
186// UpdateCSRF establishes the guest session that every other call in this package
187// depends on, and must be called before them. It fetches a session cookie (only
188// on the first call; later calls reuse it) and scrapes a fresh CSRF token from
189// the DeviantArt homepage.
190//
191// Tokens expire, so a long-running program should call this again when requests
192// begin to fail. It is not safe to call concurrently with other functions of
193// this package.
194//
195// An error means the session was not established: the homepage was blocked,
196// served a challenge, or changed its markup such that the token is no longer
197// where this package looks for it.
198func UpdateCSRF() error {
199	if cookie == "" {
200		req := request("https://www.deviantart.com/_puppy")
201
202		for _, content := range req.Cookies {
203			cookie = content.Raw
204		}
205	}
206
207	req := request("https://www.deviantart.com", cookie)
208	if req.Err != nil {
209		return req.Err
210	}
211	if req.Status != 200 {
212		return errors.New(describe(req))
213	}
214
215	// Bounds-check the markers. On a block/challenge page they are absent, and the
216	// old arithmetic sliced Body[24:-4] — a panic that killed the whole process.
217	start, end := strings.Index(req.Body, csrfPrefix), strings.Index(req.Body, xhrMarker)
218	if start < 0 || end < 0 {
219		return errors.New("devianter: CSRF token not found in homepage (blocked, challenged, or markup changed)")
220	}
221	start += len(csrfPrefix)
222	end -= 3
223	if end <= start || end > len(req.Body) {
224		return errors.New("devianter: CSRF token markers out of order (markup changed)")
225	}
226	token = req.Body[start:end]
227
228	return nil
229}
230
231// puppy calls a _puppy endpoint with the guest session applied and returns the
232// raw JSON body. data is a path and query string; the CSRF token and API version
233// are appended to it, so it must already end in a parameter (callers conclude
234// theirs with a trailing "&" or a final value).
235//
236// It returns an error for a transport failure, a non-200 status, or a 200 whose
237// body is not JSON, which is how a CDN block page arrives.
238func puppy(data string) (string, error) {
239	var url strings.Builder
240	url.WriteString("https://www.deviantart.com/_puppy/")
241	url.WriteString(data)
242	url.WriteString("&csrf_token=")
243	url.WriteString(token)
244	url.WriteString("&da_minor_version=20230710")
245
246	body := request(url.String(), cookie)
247	if body.Err != nil {
248		return "", body.Err
249	}
250
251	if body.Status != 200 {
252		return "", errors.New(describe(body))
253	}
254
255	// A 200 that isn't JSON means an edge/CDN page slipped through.
256	if !looksLikeJSON(body) {
257		return "", errors.New(describe(body))
258	}
259
260	return body.Body, nil
261}