| @@ -5,7 +5,9 @@ import ( |
| 5 | 5 | "errors" |
| 6 | 6 | "io" |
| 7 | 7 | "net/http" |
| 8 | "strconv" |
| 8 | 9 | "strings" |
| 10 | "time" |
| 9 | 11 | ) |
| 10 | 12 | |
| 11 | 13 | // функция для высера ошибки в stderr |
| @@ -31,7 +33,13 @@ type Error struct { |
| 31 | 33 | func APIError(inputError error) (err Error) { |
| 32 | 34 | if inputError != nil { |
| 33 | 35 | err.RAW = []byte(inputError.Error()) |
| 34 | | try(json.Unmarshal(err.RAW, &err)) |
| 36 | // DA's API errors are JSON. Anything else (CDN block pages, transport |
| 37 | // failures) is surfaced as-is rather than spamming a JSON parse error — |
| 38 | // this is what used to print `invalid character '<'` on every page. |
| 39 | if json.Unmarshal(err.RAW, &err) != nil { |
| 40 | err.Reason = "request_failed" |
| 41 | err.Error = inputError.Error() |
| 42 | } |
| 35 | 43 | } |
| 36 | 44 | return |
| 37 | 45 | } |
| @@ -43,18 +51,30 @@ type reqrt struct { |
| 43 | 51 | Status int |
| 44 | 52 | Cookies []*http.Cookie |
| 45 | 53 | Headers http.Header |
| 54 | // Err is set when the request never completed (transport error). Status is 0. |
| 55 | Err error |
| 46 | 56 | } |
| 47 | 57 | |
| 48 | 58 | // функция для совершения запроса |
| 49 | 59 | var UserAgent string |
| 50 | 60 | |
| 61 | // Timeout bounds a single request end-to-end (dial, response, body read). |
| 62 | // Without it, a hung connection blocks its caller forever. |
| 63 | var Timeout = 30 * time.Second |
| 64 | |
| 51 | 65 | func request(uri string, other ...string) reqrt { |
| 52 | 66 | var r reqrt |
| 53 | 67 | |
| 54 | 68 | // создаём новый запрос |
| 55 | | cli := &http.Client{} |
| 69 | // Transport is deliberately left nil so http.DefaultTransport applies: that |
| 70 | // keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit). |
| 71 | cli := &http.Client{Timeout: Timeout} |
| 56 | 72 | req, e := http.NewRequest("GET", uri, nil) |
| 57 | | try(e) |
| 73 | if e != nil { |
| 74 | try(e) |
| 75 | r.Err = e |
| 76 | return r |
| 77 | } |
| 58 | 78 | |
| 59 | 79 | req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") |
| 60 | 80 | |
| @@ -67,11 +87,20 @@ func request(uri string, other ...string) reqrt { |
| 67 | 87 | } |
| 68 | 88 | |
| 69 | 89 | resp, e := cli.Do(req) |
| 70 | | try(e) |
| 90 | if e != nil { |
| 91 | // resp is nil on error: returning here avoids dereferencing it, which |
| 92 | // used to panic and (from UpdateCSRF's goroutine) kill the process. |
| 93 | try(e) |
| 94 | r.Err = e |
| 95 | return r |
| 96 | } |
| 71 | 97 | defer resp.Body.Close() |
| 72 | 98 | |
| 73 | 99 | body, e := io.ReadAll(resp.Body) |
| 74 | | try(e) |
| 100 | if e != nil { |
| 101 | try(e) |
| 102 | r.Err = e |
| 103 | } |
| 75 | 104 | |
| 76 | 105 | // заполняем структуру |
| 77 | 106 | r.Body = string(body) |
| @@ -82,11 +111,44 @@ func request(uri string, other ...string) reqrt { |
| 82 | 111 | return r |
| 83 | 112 | } |
| 84 | 113 | |
| 114 | // looksLikeJSON reports whether a response is actually JSON, so an HTML page from |
| 115 | // a CDN/edge never reaches json.Unmarshal. |
| 116 | func looksLikeJSON(r reqrt) bool { |
| 117 | if ct := r.Headers.Get("Content-Type"); ct != "" && !strings.Contains(ct, "json") { |
| 118 | return false |
| 119 | } |
| 120 | b := strings.TrimSpace(r.Body) |
| 121 | return len(b) > 0 && (b[0] == '{' || b[0] == '[') |
| 122 | } |
| 123 | |
| 124 | // describe renders a failed response as a readable message, instead of the opaque |
| 125 | // `invalid character '<'` you get from json.Unmarshal on an HTML error page. |
| 126 | func describe(r reqrt) string { |
| 127 | body := strings.TrimSpace(r.Body) |
| 128 | if looksLikeJSON(r) { |
| 129 | return body // DA's own JSON error; callers unmarshal it into Error |
| 130 | } |
| 131 | |
| 132 | msg := "devianter: HTTP " + strconv.Itoa(r.Status) + " non-JSON response from DeviantArt" |
| 133 | if strings.Contains(body, "Generated by cloudfront") || strings.Contains(body, "Request blocked") { |
| 134 | msg += ": blocked by CloudFront/WAF — this egress IP is likely banned" |
| 135 | } |
| 136 | if len(body) > 200 { |
| 137 | body = body[:200] + "..." |
| 138 | } |
| 139 | return msg + " — " + body |
| 140 | } |
| 141 | |
| 85 | 142 | /* PUPPY aka DeviantArt API */ |
| 86 | 143 | // получение или обновление токена |
| 87 | 144 | var cookie string |
| 88 | 145 | var token string |
| 89 | 146 | |
| 147 | const ( |
| 148 | csrfPrefix = "window.__CSRF_TOKEN__ = '" |
| 149 | xhrMarker = "window.__XHR_LOCAL__" |
| 150 | ) |
| 151 | |
| 90 | 152 | func UpdateCSRF() error { |
| 91 | 153 | if cookie == "" { |
| 92 | 154 | req := request("https://www.deviantart.com/_puppy") |
| @@ -97,10 +159,25 @@ func UpdateCSRF() error { |
| 97 | 159 | } |
| 98 | 160 | |
| 99 | 161 | req := request("https://www.deviantart.com", cookie) |
| 162 | if req.Err != nil { |
| 163 | return req.Err |
| 164 | } |
| 100 | 165 | if req.Status != 200 { |
| 101 | | return errors.New(req.Body) |
| 166 | return errors.New(describe(req)) |
| 102 | 167 | } |
| 103 | | token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] |
| 168 | |
| 169 | // Bounds-check the markers. On a block/challenge page they are absent, and the |
| 170 | // old arithmetic sliced Body[24:-4] — a panic that killed the whole process. |
| 171 | start, end := strings.Index(req.Body, csrfPrefix), strings.Index(req.Body, xhrMarker) |
| 172 | if start < 0 || end < 0 { |
| 173 | return errors.New("devianter: CSRF token not found in homepage (blocked, challenged, or markup changed)") |
| 174 | } |
| 175 | start += len(csrfPrefix) |
| 176 | end -= 3 |
| 177 | if end <= start || end > len(req.Body) { |
| 178 | return errors.New("devianter: CSRF token markers out of order (markup changed)") |
| 179 | } |
| 180 | token = req.Body[start:end] |
| 104 | 181 | |
| 105 | 182 | return nil |
| 106 | 183 | } |
| @@ -114,10 +191,18 @@ func puppy(data string) (string, error) { |
| 114 | 191 | url.WriteString("&da_minor_version=20230710") |
| 115 | 192 | |
| 116 | 193 | body := request(url.String(), cookie) |
| 194 | if body.Err != nil { |
| 195 | return "", body.Err |
| 196 | } |
| 117 | 197 | |
| 118 | 198 | // если код ответа не 200, возвращается ошибка |
| 119 | 199 | if body.Status != 200 { |
| 120 | | return "", errors.New(body.Body) |
| 200 | return "", errors.New(describe(body)) |
| 201 | } |
| 202 | |
| 203 | // A 200 that isn't JSON means an edge/CDN page slipped through. |
| 204 | if !looksLikeJSON(body) { |
| 205 | return "", errors.New(describe(body)) |
| 121 | 206 | } |
| 122 | 207 | |
| 123 | 208 | return body.Body, nil |