Commit b80180e77e
Unregistered key
Layout: unified · split
CODEOWNERS added +1
| @@ -0,0 +1 @@ | |||
| 1 | * @ccleberg\ No newline at end of file | ||
comments.go +64 −21
| @@ -4,18 +4,28 @@ import ( | |||
| 4 | "encoding/json" | 4 | "encoding/json" |
| 5 | "net/url" | 5 | "net/url" |
| 6 | "strconv" | 6 | "strconv" |
| 7 | "strings" | ||
| 7 | ) | 8 | ) |
| 8 | 9 | ||
| 10 | // Thread is a single comment, despite the name. Replies are not nested inside | ||
| 11 | // it: a thread arrives flattened into [Comments].Thread, and the shape is | ||
| 12 | // recovered through Parent, which holds the ID of the comment being replied to | ||
| 13 | // and is 0 for a top-level comment. | ||
| 9 | type Thread struct { | 14 | type Thread struct { |
| 10 | Replies, Likes int | 15 | Replies, Likes int |
| 11 | ID int `json:"commentId"` | 16 | ID int `json:"commentId"` |
| 12 | Parent int `json:"parentId"` | 17 | Parent int `json:"parentId"` |
| 13 | 18 | ||
| 14 | Posted timeStamp | 19 | Posted timeStamp |
| 20 | // Author reports whether the commenter is the author of the deviation being | ||
| 21 | // commented on. | ||
| 15 | Author bool `json:"isAuthorHighlited"` | 22 | Author bool `json:"isAuthorHighlited"` |
| 16 | 23 | ||
| 17 | Desctiption string | 24 | Desctiption string |
| 18 | Comment string | 25 | |
| 26 | // Comment is the comment's plain text, which [GetComments] extracts from | ||
| 27 | // TextContent. Prefer it; TextContent is the unprocessed original. | ||
| 28 | Comment string | ||
| 19 | 29 | ||
| 20 | TextContent Text | 30 | TextContent Text |
| 21 | 31 | ||
| @@ -25,6 +35,10 @@ type Thread struct { | |||
| 25 | } | 35 | } |
| 26 | } | 36 | } |
| 27 | 37 | ||
| 38 | // Comments is one page of comments. Thread holds the comments themselves, | ||
| 39 | // flattened rather than nested; Total counts every comment on the item, not just | ||
| 40 | // this page. Cursor resumes from the end of this page and HasMore reports | ||
| 41 | // whether anything remains. | ||
| 28 | type Comments struct { | 42 | type Comments struct { |
| 29 | Cursor string | 43 | Cursor string |
| 30 | PrevOffset int | 44 | PrevOffset int |
| @@ -34,7 +48,17 @@ type Comments struct { | |||
| 34 | Thread []Thread | 48 | Thread []Thread |
| 35 | } | 49 | } |
| 36 | 50 | ||
| 37 | // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя | 51 | // GetComments retrieves comments on an item, 50 per page, with each comment's |
| 52 | // plain text extracted into [Thread].Comment. | ||
| 53 | // | ||
| 54 | // typ selects what postid refers to: 1 for comments on a deviation, 4 for those | ||
| 55 | // on a user's or group's profile wall. cursor resumes from a previous call's | ||
| 56 | // [Comments].Cursor; pass an empty string to start from the newest comment. | ||
| 57 | // | ||
| 58 | // page is an offset from cursor rather than an absolute page number, and it is | ||
| 59 | // walked one request at a time: page 5 costs six round-trips and returns only | ||
| 60 | // the sixth page. Paginating by feeding each result's Cursor back in with page 0 | ||
| 61 | // costs one request per page, and is the cheaper way to walk a long thread. | ||
| 38 | func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments, err Error) { | 62 | func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments, err Error) { |
| 39 | for x := 0; x <= page; x++ { | 63 | for x := 0; x <= page; x++ { |
| 40 | err = ujson( | 64 | err = ujson( |
| @@ -46,28 +70,47 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment | |||
| 46 | 70 | ||
| 47 | cursor = cmmts.Cursor | 71 | cursor = cmmts.Cursor |
| 48 | 72 | ||
| 49 | // парсинг json внутри json | ||
| 50 | for i := 0; i < len(cmmts.Thread); i++ { | 73 | for i := 0; i < len(cmmts.Thread); i++ { |
| 51 | m, l := cmmts.Thread[i].TextContent.Html.Markup, len(cmmts.Thread[i].TextContent.Html.Markup) | 74 | cmmts.Thread[i].Comment = flattenComment(cmmts.Thread[i].TextContent.Html.Markup) |
| 52 | cmmts.Thread[i].Comment = m | ||
| 53 | |||
| 54 | // если начало строки {, а конец }, то срабатывает этот иф | ||
| 55 | if m[0] == '{' && m[l-1] == '}' { | ||
| 56 | var content struct { | ||
| 57 | Blocks []struct { | ||
| 58 | Text string | ||
| 59 | } | ||
| 60 | } | ||
| 61 | |||
| 62 | e := json.Unmarshal([]byte(m), &content) | ||
| 63 | try(e) | ||
| 64 | |||
| 65 | for _, a := range content.Blocks { | ||
| 66 | cmmts.Thread[i].Comment = a.Text | ||
| 67 | } | ||
| 68 | } | ||
| 69 | } | 75 | } |
| 70 | } | 76 | } |
| 71 | 77 | ||
| 72 | return | 78 | return |
| 73 | } | 79 | } |
| 80 | |||
| 81 | // flattenComment renders a body of user-written markup as plain text, be it a | ||
| 82 | // comment or a deviation's description. Bodies are JSON inside JSON: newer ones | ||
| 83 | // are a Draft.js document encoded into the markup string, older ones are plain | ||
| 84 | // HTML, which passes through unchanged. Markup that does not parse, and empty | ||
| 85 | // markup, also pass through. | ||
| 86 | // | ||
| 87 | // A Draft.js document is a list of blocks, which are block-level elements | ||
| 88 | // (paragraphs, list items); they are joined with newlines, one block per line. | ||
| 89 | func flattenComment(m string) string { | ||
| 90 | l := len(m) | ||
| 91 | if l == 0 || m[0] != '{' || m[l-1] != '}' { | ||
| 92 | return m | ||
| 93 | } | ||
| 94 | |||
| 95 | var content struct { | ||
| 96 | Blocks []struct { | ||
| 97 | Text string | ||
| 98 | } | ||
| 99 | } | ||
| 100 | |||
| 101 | e := json.Unmarshal([]byte(m), &content) | ||
| 102 | try(e) | ||
| 103 | |||
| 104 | if len(content.Blocks) == 0 { | ||
| 105 | return m | ||
| 106 | } | ||
| 107 | |||
| 108 | var b strings.Builder | ||
| 109 | for i, a := range content.Blocks { | ||
| 110 | if i > 0 { | ||
| 111 | b.WriteString("\n") | ||
| 112 | } | ||
| 113 | b.WriteString(a.Text) | ||
| 114 | } | ||
| 115 | return b.String() | ||
| 116 | } | ||
comments_test.go added +51
| @@ -0,0 +1,51 @@ | |||
| 1 | package devianter | ||
| 2 | |||
| 3 | import "testing" | ||
| 4 | |||
| 5 | // Regression: flattenComment's shape check used to read m[0] and m[len(m)-1] | ||
| 6 | // without a length check, so a comment with an empty markup body panicked with | ||
| 7 | // index out of range and killed the caller's process. | ||
| 8 | func TestFlattenCommentEmptyMarkup(t *testing.T) { | ||
| 9 | if got := flattenComment(""); got != "" { | ||
| 10 | t.Errorf("want an empty comment for empty markup, got %q", got) | ||
| 11 | } | ||
| 12 | } | ||
| 13 | |||
| 14 | func TestFlattenComment(t *testing.T) { | ||
| 15 | // A newer, Draft.js-encoded body is flattened to its text. | ||
| 16 | draft := `{"blocks":[{"text":"hello there"}]}` | ||
| 17 | if got := flattenComment(draft); got != "hello there" { | ||
| 18 | t.Errorf("want the Draft.js block text, got %q", got) | ||
| 19 | } | ||
| 20 | |||
| 21 | // An older, plain-HTML body passes through untouched. | ||
| 22 | html := "<b>hello</b> there" | ||
| 23 | if got := flattenComment(html); got != html { | ||
| 24 | t.Errorf("want plain HTML passed through, got %q", got) | ||
| 25 | } | ||
| 26 | |||
| 27 | // Regression: the block loop used to assign rather than accumulate, so every | ||
| 28 | // block but the last was silently dropped and a multi-paragraph comment came | ||
| 29 | // back as its closing line only. | ||
| 30 | multi := `{"blocks":[{"text":"first"},{"text":"second"},{"text":"third"}]}` | ||
| 31 | if got, want := flattenComment(multi), "first\nsecond\nthird"; got != want { | ||
| 32 | t.Errorf("want every block, one per line:\n got %q\nwant %q", got, want) | ||
| 33 | } | ||
| 34 | |||
| 35 | // An empty block is a blank line in the comment, not something to skip. | ||
| 36 | blank := `{"blocks":[{"text":"first"},{"text":""},{"text":"third"}]}` | ||
| 37 | if got, want := flattenComment(blank), "first\n\nthird"; got != want { | ||
| 38 | t.Errorf("want an empty block preserved as a blank line:\n got %q\nwant %q", got, want) | ||
| 39 | } | ||
| 40 | |||
| 41 | // Brace-shaped markup that isn't a Draft.js document falls back to itself | ||
| 42 | // rather than to an empty string. | ||
| 43 | if got := flattenComment("{}"); got != "{}" { | ||
| 44 | t.Errorf("want the original markup when there are no blocks, got %q", got) | ||
| 45 | } | ||
| 46 | |||
| 47 | // A single brace satisfies neither end of the shape check. | ||
| 48 | if got := flattenComment("{"); got != "{" { | ||
| 49 | t.Errorf("want a lone brace passed through, got %q", got) | ||
| 50 | } | ||
| 51 | } | ||
deviantion.go +44 −26
| @@ -1,13 +1,14 @@ | |||
| 1 | package devianter | 1 | package devianter |
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | "encoding/json" | ||
| 5 | "strconv" | 4 | "strconv" |
| 6 | "strings" | 5 | "strings" |
| 7 | "time" | 6 | "time" |
| 8 | ) | 7 | ) |
| 9 | 8 | ||
| 10 | // хрень для парсинга времени публикации | 9 | // timeStamp is a time.Time that parses DeviantArt's publication timestamps, |
| 10 | // which are ISO 8601 with no colon in the zone offset and so are rejected by | ||
| 11 | // encoding/json's default time handling. | ||
| 11 | type timeStamp struct { | 12 | type timeStamp struct { |
| 12 | time.Time | 13 | time.Time |
| 13 | } | 14 | } |
| @@ -20,7 +21,14 @@ func (t *timeStamp) UnmarshalJSON(b []byte) (err error) { | |||
| 20 | return | 21 | return |
| 21 | } | 22 | } |
| 22 | 23 | ||
| 23 | // самая главная структура для поста | 24 | // Deviation is a single artwork and its metadata: the central type of this |
| 25 | // package. Most endpoints return these, either alone or in slices. | ||
| 26 | // | ||
| 27 | // How much of it is populated depends on the endpoint. Search results and | ||
| 28 | // gallery listings return a shallow Deviation — enough for a thumbnail and a | ||
| 29 | // title — while [GetDeviation] fills in Extended, with the tags, original file | ||
| 30 | // details, and description. A zero-valued field usually means the endpoint did | ||
| 31 | // not send it rather than that the artwork lacks it. | ||
| 24 | type Deviation struct { | 32 | type Deviation struct { |
| 25 | Title, Url, License string | 33 | Title, Url, License string |
| 26 | PublishedTime timeStamp | 34 | PublishedTime timeStamp |
| @@ -55,17 +63,25 @@ type Deviation struct { | |||
| 55 | TextContent Text | 63 | TextContent Text |
| 56 | } | 64 | } |
| 57 | 65 | ||
| 58 | // её выпердыши | 66 | // Media locates a deviation's image files. It is not a usable URL on its own: |
| 67 | // the pieces have to be assembled, and the result signed with a token. Pass it | ||
| 68 | // to [UrlFromMedia] rather than building the URL by hand. | ||
| 59 | type Media struct { | 69 | type Media struct { |
| 60 | BaseUri string | 70 | BaseUri string |
| 61 | Name string `json:"prettyName"` | 71 | Name string `json:"prettyName"` |
| 62 | Token []string | 72 | Token []string |
| 63 | Types []struct { | 73 | // Types are the renditions available (thumbnails, preview, "fullview"), each |
| 74 | // with its own dimensions. | ||
| 75 | Types []struct { | ||
| 64 | T string | 76 | T string |
| 65 | H, W int | 77 | H, W int |
| 66 | } | 78 | } |
| 67 | } | 79 | } |
| 68 | 80 | ||
| 81 | // Text is a block of user-written text — a description, a comment, a group's | ||
| 82 | // about page. Markup holds either HTML or a JSON-encoded Draft.js document, | ||
| 83 | // distinguished by Type; the functions that return a Text generally extract the | ||
| 84 | // plain text into a neighbouring field, which is easier to use. | ||
| 69 | type Text struct { | 85 | type Text struct { |
| 70 | Excerpt string | 86 | Excerpt string |
| 71 | Html struct { | 87 | Html struct { |
| @@ -73,7 +89,13 @@ type Text struct { | |||
| 73 | } | 89 | } |
| 74 | } | 90 | } |
| 75 | 91 | ||
| 76 | // структура поста | 92 | // Post is a deviation together with its comment metadata, as returned by |
| 93 | // [GetDeviation]. IMG and Description are conveniences that GetDeviation derives | ||
| 94 | // from the Deviation, so callers need not assemble a URL or decode Draft.js | ||
| 95 | // markup themselves. | ||
| 96 | // | ||
| 97 | // Comments holds only a total and a cursor. To retrieve the comments, pass them | ||
| 98 | // to [GetComments] with type 1. | ||
| 77 | type Post struct { | 99 | type Post struct { |
| 78 | Deviation Deviation | 100 | Deviation Deviation |
| 79 | Comments struct { | 101 | Comments struct { |
| @@ -90,7 +112,14 @@ type Post struct { | |||
| 90 | IMG, Description string | 112 | IMG, Description string |
| 91 | } | 113 | } |
| 92 | 114 | ||
| 93 | // преобразование урла в правильный | 115 | // UrlFromMedia assembles a usable, token-signed image URL from a [Media], along |
| 116 | // with the filename DeviantArt would serve it under. It selects the "fullview" | ||
| 117 | // rendition and returns empty strings if the media has none. | ||
| 118 | // | ||
| 119 | // An optional thumb argument scales the request down towards that many pixels | ||
| 120 | // per side, for fetching a smaller copy than the original. GIFs and very large | ||
| 121 | // images (beyond roughly 33 megapixels) are returned at their original URL | ||
| 122 | // without resizing, as DeviantArt's resizer refuses them. | ||
| 94 | func UrlFromMedia(m Media, thumb ...int) (urlParsed, wellFormattedFilename string) { | 123 | func UrlFromMedia(m Media, thumb ...int) (urlParsed, wellFormattedFilename string) { |
| 95 | var url strings.Builder | 124 | var url strings.Builder |
| 96 | 125 | ||
| @@ -131,7 +160,13 @@ func UrlFromMedia(m Media, thumb ...int) (urlParsed, wellFormattedFilename strin | |||
| 131 | return | 160 | return |
| 132 | } | 161 | } |
| 133 | 162 | ||
| 134 | // для работы функции нужно ID поста и имя пользователя. | 163 | // GetDeviation retrieves a single deviation by its numeric ID and its author's |
| 164 | // username. Both are required: the endpoint will not resolve an ID alone. They | ||
| 165 | // appear in a deviation's page URL, which ends in a slug of the form | ||
| 166 | // title-by-author-123456789. | ||
| 167 | // | ||
| 168 | // The returned Post has its IMG and Description already derived, and its | ||
| 169 | // Deviation is fully populated, including Extended. | ||
| 135 | func GetDeviation(id string, user string) (st Post, err Error) { | 170 | func GetDeviation(id string, user string) (st Post, err Error) { |
| 136 | err = ujson( | 171 | err = ujson( |
| 137 | "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", | 172 | "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", |
| @@ -140,24 +175,7 @@ func GetDeviation(id string, user string) (st Post, err Error) { | |||
| 140 | 175 | ||
| 141 | st.IMG, _ = UrlFromMedia(st.Deviation.Media) | 176 | st.IMG, _ = UrlFromMedia(st.Deviation.Media) |
| 142 | 177 | ||
| 143 | // базовая обработка описания | 178 | st.Description = flattenComment(st.Deviation.TextContent.Html.Markup) |
| 144 | txt := st.Deviation.TextContent.Html.Markup | ||
| 145 | if len(txt) > 1 && txt[1] == '{' { | ||
| 146 | var description struct { | ||
| 147 | Blocks []struct { | ||
| 148 | Text string | ||
| 149 | } | ||
| 150 | } | ||
| 151 | |||
| 152 | if err := json.Unmarshal([]byte(txt), &description); err != nil { | ||
| 153 | // Handle error appropriately | ||
| 154 | try(err) // or log/return the error | ||
| 155 | } | ||
| 156 | for _, a := range description.Blocks { | ||
| 157 | txt = a.Text | ||
| 158 | } | ||
| 159 | } | ||
| 160 | st.Description = txt | ||
| 161 | 179 | ||
| 162 | return | 180 | return |
| 163 | } | 181 | } |
doc.go added +45
| @@ -0,0 +1,45 @@ | |||
| 1 | // Package devianter is a client for DeviantArt's internal "_puppy" API, the | ||
| 2 | // JSON backend that deviantart.com's own web frontend calls. | ||
| 3 | // | ||
| 4 | // This is not the official, documented DeviantArt API. There is no application | ||
| 5 | // registration and no OAuth: the package authenticates the way a logged-out | ||
| 6 | // browser does, by fetching a guest session cookie and a CSRF token from the | ||
| 7 | // homepage. Everything reachable here is what an anonymous visitor can see. | ||
| 8 | // Because the endpoints are internal, DeviantArt can change or remove them | ||
| 9 | // without notice. | ||
| 10 | // | ||
| 11 | // # Usage | ||
| 12 | // | ||
| 13 | // Call [UpdateCSRF] once before anything else to establish the guest session. | ||
| 14 | // Every other call depends on the cookie and token it stores, and will fail | ||
| 15 | // until it has run: | ||
| 16 | // | ||
| 17 | // if err := devianter.UpdateCSRF(); err != nil { | ||
| 18 | // log.Fatal(err) | ||
| 19 | // } | ||
| 20 | // | ||
| 21 | // post, apiErr := devianter.GetDeviation("123456789", "someuser") | ||
| 22 | // if apiErr.Reason != "" { | ||
| 23 | // log.Fatal(apiErr.Error) | ||
| 24 | // } | ||
| 25 | // fmt.Println(post.Deviation.Title, post.IMG) | ||
| 26 | // | ||
| 27 | // The session does not refresh itself. A long-running program should call | ||
| 28 | // [UpdateCSRF] again when calls start failing, since tokens expire. | ||
| 29 | // | ||
| 30 | // # Errors | ||
| 31 | // | ||
| 32 | // Most functions return an [Error] value rather than a Go error. It is a struct, | ||
| 33 | // not an interface, so it is never nil; a call succeeded if Error.Reason is | ||
| 34 | // empty. Functions that can also fail on their arguments before any request is | ||
| 35 | // made (such as [PerformSearch] and [Group.Gallery]) return an ordinary error | ||
| 36 | // alongside it for that case. | ||
| 37 | // | ||
| 38 | // # Rate limiting and blocking | ||
| 39 | // | ||
| 40 | // DeviantArt sits behind CloudFront, which blocks IP addresses that request too | ||
| 41 | // aggressively. A blocked request surfaces as an [Error] whose Error field | ||
| 42 | // mentions CloudFront/WAF. This package does no rate limiting, retrying, or | ||
| 43 | // backoff of its own; a caller making bulk requests is expected to pace itself. | ||
| 44 | // Set [UserAgent] to identify your client and [Timeout] to bound each request. | ||
| 45 | package devianter | ||
misc.go +55 −18
| @@ -2,7 +2,6 @@ package devianter | |||
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | "errors" | 4 | "errors" |
| 5 | "log" | ||
| 6 | "math" | 5 | "math" |
| 7 | "net/url" | 6 | "net/url" |
| 8 | "strconv" | 7 | "strconv" |
| @@ -10,20 +9,28 @@ import ( | |||
| 10 | ) | 9 | ) |
| 11 | 10 | ||
| 12 | /* AVATARS AND EMOJIS */ | 11 | /* AVATARS AND EMOJIS */ |
| 12 | // AEmedia fetches a user's avatar or a site emoji by name. t selects which: | ||
| 13 | // 'a' for an avatar, 'e' for an emoji. | ||
| 14 | // | ||
| 15 | // It returns the image data itself, not a URL. DeviantArt does not say which | ||
| 16 | // format a given name is stored in, so this tries .jpg, .png, and .gif in turn | ||
| 17 | // and returns the first that exists — up to three requests per call, and three | ||
| 18 | // for a name that does not exist. | ||
| 19 | // | ||
| 20 | // Passing any other t returns an error without making a request. | ||
| 13 | func AEmedia(name string, t rune) (string, error) { | 21 | func AEmedia(name string, t rune) (string, error) { |
| 14 | if len(name) < 2 { | 22 | if len(name) < 2 { |
| 15 | return "", errors.New("name must be specified") | 23 | return "", errors.New("name must be specified") |
| 16 | } | 24 | } |
| 17 | // список всех возможных расширений | ||
| 18 | var extensions = [3]string{ | 25 | var extensions = [3]string{ |
| 19 | ".jpg", | 26 | ".jpg", |
| 20 | ".png", | 27 | ".png", |
| 21 | ".gif", | 28 | ".gif", |
| 22 | } | 29 | } |
| 23 | // надо | ||
| 24 | name = strings.ToLower(name) | 30 | name = strings.ToLower(name) |
| 25 | 31 | ||
| 26 | // построение ссылок. билдер потому что он быстрее обычного сложения строк. | 32 | // Avatars and emoji are sharded into directories by the leading characters of |
| 33 | // the name; avatars additionally normalise dashes to underscores first. | ||
| 27 | var b strings.Builder | 34 | var b strings.Builder |
| 28 | switch t { | 35 | switch t { |
| 29 | case 'a': | 36 | case 'a': |
| @@ -38,11 +45,11 @@ func AEmedia(name string, t rune) (string, error) { | |||
| 38 | b.WriteString(name[:1]) | 45 | b.WriteString(name[:1]) |
| 39 | b.WriteString("/") | 46 | b.WriteString("/") |
| 40 | default: | 47 | default: |
| 41 | log.Fatalln("Invalid type.\n- 'a' -- avatar;\n- 'e' -- emoji.") | 48 | return "", errors.New("invalid type: want 'a' (avatar) or 'e' (emoji)") |
| 42 | } | 49 | } |
| 43 | b.WriteString(name) | 50 | b.WriteString(name) |
| 44 | 51 | ||
| 45 | // проверка ссылки на доступность | 52 | // Probe each extension; the first 200 is the real format. |
| 46 | for x := 0; x < len(extensions); x++ { | 53 | for x := 0; x < len(extensions); x++ { |
| 47 | req := request(b.String() + extensions[x]) | 54 | req := request(b.String() + extensions[x]) |
| 48 | if req.Status == 200 { | 55 | if req.Status == 200 { |
| @@ -54,6 +61,9 @@ func AEmedia(name string, t rune) (string, error) { | |||
| 54 | } | 61 | } |
| 55 | 62 | ||
| 56 | /* DAILY DEVIATIONS */ | 63 | /* DAILY DEVIATIONS */ |
| 64 | // DailyDeviations is the staff-curated front page selection. The picks are | ||
| 65 | // grouped into Strips, each a titled row as the site presents it; Deviations is | ||
| 66 | // the ungrouped listing. | ||
| 57 | type DailyDeviations struct { | 67 | type DailyDeviations struct { |
| 58 | HasMore bool | 68 | HasMore bool |
| 59 | Strips []struct { | 69 | Strips []struct { |
| @@ -64,30 +74,54 @@ type DailyDeviations struct { | |||
| 64 | Deviations []Deviation | 74 | Deviations []Deviation |
| 65 | } | 75 | } |
| 66 | 76 | ||
| 77 | // GetDailyDeviations retrieves a page of the daily deviation selection. Pages | ||
| 78 | // are zero-based; check the returned HasMore before asking for the next. | ||
| 67 | func GetDailyDeviations(page int) (dd DailyDeviations, err Error) { | 79 | func GetDailyDeviations(page int) (dd DailyDeviations, err Error) { |
| 68 | err = ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) | 80 | err = ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) |
| 69 | return | 81 | return |
| 70 | } | 82 | } |
| 71 | 83 | ||
| 72 | /* SEARCH */ | 84 | /* SEARCH */ |
| 85 | // Search is a page of search results. Read the matches from Results, which | ||
| 86 | // [PerformSearch] populates whichever field the endpoint used. | ||
| 87 | // | ||
| 88 | // Total is DeviantArt's own estimate and is approximate. Pages is derived from | ||
| 89 | // it and capped at 417, the depth a guest session can reach before the API stops | ||
| 90 | // paginating. | ||
| 73 | type Search struct { | 91 | type Search struct { |
| 74 | Total int `json:"estTotal"` | 92 | Total int `json:"estTotal"` |
| 75 | Pages int // only for 'a' and 'g' scope. | 93 | Pages int // only for 'a' and 'g' scope. |
| 76 | HasMore bool | 94 | HasMore bool |
| 77 | Results []Deviation `json:"deviations"` | 95 | Results []Deviation `json:"deviations"` |
| 96 | // ResultsGalleryTemp receives the results of gallery and collection searches, | ||
| 97 | // which return them under a different key. PerformSearch copies it into | ||
| 98 | // Results; callers should not need this field. | ||
| 78 | ResultsGalleryTemp []Deviation `json:"results"` | 99 | ResultsGalleryTemp []Deviation `json:"results"` |
| 79 | } | 100 | } |
| 80 | 101 | ||
| 102 | // PerformSearch searches DeviantArt. scope selects what is being searched: | ||
| 103 | // | ||
| 104 | // 'a' — everything, by title and description | ||
| 105 | // 't' — by tag | ||
| 106 | // 'g' — within one user's or group's gallery | ||
| 107 | // 'f' — within one user's or group's collections (favourites) | ||
| 108 | // | ||
| 109 | // Scopes 'g' and 'f' search a particular account, so they require the username | ||
| 110 | // as the final argument and return an error without it. The other two ignore it. | ||
| 111 | // | ||
| 112 | // Pages are zero-based. A guest session cannot page beyond roughly 417 pages | ||
| 113 | // deep regardless of how many results Total claims. | ||
| 114 | // | ||
| 115 | // Passing any other scope returns an error without making a request. | ||
| 81 | func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, daError Error, err error) { | 116 | func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, daError Error, err error) { |
| 82 | var buildurl strings.Builder | 117 | var buildurl strings.Builder |
| 83 | 118 | ||
| 84 | // о5 построение ссылок. | ||
| 85 | switch scope { | 119 | switch scope { |
| 86 | case 'a': // поиск артов по названию | 120 | case 'a': |
| 87 | buildurl.WriteString("dabrowse/search/all?q=") | 121 | buildurl.WriteString("dabrowse/search/all?q=") |
| 88 | case 't': // поиск артов по тегам | 122 | case 't': |
| 89 | buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=") | 123 | buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=") |
| 90 | case 'g', 'f': // поиск артов пользователя или группы | 124 | case 'g', 'f': |
| 91 | if user == nil { | 125 | if user == nil { |
| 92 | err = errors.New("missing username (last argument)") | 126 | err = errors.New("missing username (last argument)") |
| 93 | return | 127 | return |
| @@ -103,13 +137,15 @@ func PerformSearch(query string, page int, scope rune, user ...string) (ss Searc | |||
| 103 | } | 137 | } |
| 104 | buildurl.WriteString("&order=most-recent&init=true&limit=50&q=") | 138 | buildurl.WriteString("&order=most-recent&init=true&limit=50&q=") |
| 105 | default: | 139 | default: |
| 106 | log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery\n- 'f' - folders.") | 140 | err = errors.New("invalid scope: want 'a' (all), 't' (tag), 'g' (gallery) or 'f' (favourites)") |
| 141 | return | ||
| 107 | } | 142 | } |
| 108 | 143 | ||
| 109 | buildurl.WriteString(url.QueryEscape(query)) | 144 | buildurl.WriteString(url.QueryEscape(query)) |
| 110 | if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код | 145 | // Gallery search paginates by item offset rather than page number. |
| 146 | if scope != 'g' { | ||
| 111 | buildurl.WriteString("&page=") | 147 | buildurl.WriteString("&page=") |
| 112 | } else { // иначе вместо страницы будет оффсет и страница умножится на 50 | 148 | } else { |
| 113 | buildurl.WriteString("&offset=") | 149 | buildurl.WriteString("&offset=") |
| 114 | page = 50 * page | 150 | page = 50 * page |
| 115 | } | 151 | } |
| @@ -121,7 +157,8 @@ func PerformSearch(query string, page int, scope rune, user ...string) (ss Searc | |||
| 121 | ss.Results = ss.ResultsGalleryTemp | 157 | ss.Results = ss.ResultsGalleryTemp |
| 122 | } | 158 | } |
| 123 | 159 | ||
| 124 | // расчёт, сколько всего страниц по запросу. без токена, 417 страниц - максимум | 160 | // Derive the page count from the result estimate, clamped to the 417 pages a |
| 161 | // guest session can actually reach. | ||
| 125 | totalfloat := int(math.Round(float64(ss.Total / 25))) | 162 | totalfloat := int(math.Round(float64(ss.Total / 25))) |
| 126 | for x := 0; x < totalfloat; x++ { | 163 | for x := 0; x < totalfloat; x++ { |
| 127 | if x <= 417 { | 164 | if x <= 417 { |
misc_test.go added +40
| @@ -0,0 +1,40 @@ | |||
| 1 | package devianter | ||
| 2 | |||
| 3 | import "testing" | ||
| 4 | |||
| 5 | // Regression: AEmedia used to call log.Fatalln on an unknown type, terminating | ||
| 6 | // the calling process. If this test ever regresses it will not fail — the test | ||
| 7 | // binary will exit(1) mid-run. | ||
| 8 | func TestAEmediaInvalidTypeReturnsError(t *testing.T) { | ||
| 9 | _, err := AEmedia("someuser", 'z') | ||
| 10 | if err == nil { | ||
| 11 | t.Fatal("want an error for an unknown type, got nil") | ||
| 12 | } | ||
| 13 | } | ||
| 14 | |||
| 15 | // The argument checks must run in an order that never leaves a valid-looking | ||
| 16 | // call unreported: a short name is an error regardless of type. | ||
| 17 | func TestAEmediaShortNameReturnsError(t *testing.T) { | ||
| 18 | if _, err := AEmedia("", 'a'); err == nil { | ||
| 19 | t.Fatal("want an error for an empty name, got nil") | ||
| 20 | } | ||
| 21 | } | ||
| 22 | |||
| 23 | // Regression: PerformSearch used to call log.Fatalln on an unknown scope. As | ||
| 24 | // above, a regression exits the test binary rather than failing this test. | ||
| 25 | func TestPerformSearchInvalidScopeReturnsError(t *testing.T) { | ||
| 26 | _, _, err := PerformSearch("cats", 0, 'z') | ||
| 27 | if err == nil { | ||
| 28 | t.Fatal("want an error for an unknown scope, got nil") | ||
| 29 | } | ||
| 30 | } | ||
| 31 | |||
| 32 | // The account-scoped searches need a username, and must say so rather than | ||
| 33 | // requesting a URL with an empty one. | ||
| 34 | func TestPerformSearchAccountScopesRequireUser(t *testing.T) { | ||
| 35 | for _, scope := range []rune{'g', 'f'} { | ||
| 36 | if _, _, err := PerformSearch("cats", 0, scope); err == nil { | ||
| 37 | t.Errorf("want an error for scope %q with no username, got nil", scope) | ||
| 38 | } | ||
| 39 | } | ||
| 40 | } | ||
user-group.go +72 −11
| @@ -6,7 +6,14 @@ import ( | |||
| 6 | "strings" | 6 | "strings" |
| 7 | ) | 7 | ) |
| 8 | 8 | ||
| 9 | // структура группы или пользователя | 9 | // GRuser is a profile — a user's or a group's, as DeviantArt models both the |
| 10 | // same way. Owner.Group distinguishes them, and determines which of the | ||
| 11 | // ModuleData fields are populated: GroupAbout and GroupAdmins for a group, the | ||
| 12 | // embedded users for a person. | ||
| 13 | // | ||
| 14 | // The Page.Modules slice mirrors the site's own profile layout, so a caller | ||
| 15 | // looking for one piece of information has to search the slice for the module | ||
| 16 | // that carries it rather than reading a field directly. | ||
| 10 | type GRuser struct { | 17 | type GRuser struct { |
| 11 | ErrorDescription string | 18 | ErrorDescription string |
| 12 | Owner struct { | 19 | Owner struct { |
| @@ -35,6 +42,12 @@ type GRuser struct { | |||
| 35 | } `json:"pageExtraData"` | 42 | } `json:"pageExtraData"` |
| 36 | } | 43 | } |
| 37 | 44 | ||
| 45 | // Gallery is a listing of deviations from a profile, returned by | ||
| 46 | // [Group.Gallery] and [Group.Favourites]. | ||
| 47 | // | ||
| 48 | // Where the deviations land depends on the call. Results is the flat listing; | ||
| 49 | // folder-scoped requests instead nest them inside the Modules slice, under | ||
| 50 | // Folder for a gallery or Folders for the folder index itself. | ||
| 38 | type Gallery struct { | 51 | type Gallery struct { |
| 39 | Gruser struct { | 52 | Gruser struct { |
| 40 | ID int `json:"gruserId"` | 53 | ID int `json:"gruserId"` |
| @@ -42,7 +55,8 @@ type Gallery struct { | |||
| 42 | Modules []struct { | 55 | Modules []struct { |
| 43 | Name string | 56 | Name string |
| 44 | ModuleData struct { | 57 | ModuleData struct { |
| 45 | // группы | 58 | // Folders is the index of a profile's folders, each with a |
| 59 | // representative thumbnail. | ||
| 46 | Folders struct { | 60 | Folders struct { |
| 47 | HasMore bool | 61 | HasMore bool |
| 48 | Results []struct { | 62 | Results []struct { |
| @@ -54,7 +68,7 @@ type Gallery struct { | |||
| 54 | } | 68 | } |
| 55 | } | 69 | } |
| 56 | 70 | ||
| 57 | // галерея | 71 | // Folder is the contents of one folder. |
| 58 | Folder struct { | 72 | Folder struct { |
| 59 | HasMore bool | 73 | HasMore bool |
| 60 | Username string | 74 | Username string |
| @@ -69,12 +83,23 @@ type Gallery struct { | |||
| 69 | Results []Deviation | 83 | Results []Deviation |
| 70 | } | 84 | } |
| 71 | 85 | ||
| 86 | // Group is the entry point for everything scoped to one profile. Despite the | ||
| 87 | // name it addresses users as well as groups, since DeviantArt treats the two | ||
| 88 | // alike. | ||
| 89 | // | ||
| 90 | // Name is the profile's username and must be set; the methods return an error | ||
| 91 | // otherwise. Construct it directly: | ||
| 92 | // | ||
| 93 | // g := devianter.Group{Name: "someuser"} | ||
| 94 | // profile, apiErr, err := g.Get() | ||
| 72 | type Group struct { | 95 | type Group struct { |
| 73 | Name string // обязательно заполнить | 96 | Name string // required |
| 74 | Content Gallery | 97 | Content Gallery |
| 75 | } | 98 | } |
| 76 | 99 | ||
| 77 | // подходит как группа, так и пользователь | 100 | // Get retrieves the profile itself — its about page, statistics, and, for a |
| 101 | // group, its admins. It works for both users and groups; inspect | ||
| 102 | // Owner.Group on the result to tell which was returned. | ||
| 78 | func (s Group) Get() (g GRuser, daError Error, err error) { | 103 | func (s Group) Get() (g GRuser, daError Error, err error) { |
| 79 | if s.Name == "" { | 104 | if s.Name == "" { |
| 80 | return g, daError, errors.New("missing Name field") | 105 | return g, daError, errors.New("missing Name field") |
| @@ -84,10 +109,24 @@ func (s Group) Get() (g GRuser, daError Error, err error) { | |||
| 84 | return | 109 | return |
| 85 | } | 110 | } |
| 86 | 111 | ||
| 112 | // Favourites retrieves a page of the profile's favourites (its collections), 50 | ||
| 113 | // at a time, zero-based. | ||
| 114 | // | ||
| 115 | // Set all to gather every folder's contents into one listing. Otherwise pass a | ||
| 116 | // positive folderid to read a single folder, or 0 for the profile's default | ||
| 117 | // favourites listing. | ||
| 118 | // | ||
| 119 | // folderid is optional; omitting it is the same as passing 0. Only the first | ||
| 120 | // value is used. | ||
| 87 | func (s Group) Favourites(page int, all bool, folderid ...int) (g Group, err Error) { | 121 | func (s Group) Favourites(page int, all bool, folderid ...int) (g Group, err Error) { |
| 88 | var url strings.Builder | 122 | var url strings.Builder |
| 89 | 123 | ||
| 90 | if fid := folderid[0]; fid > 0 || all { | 124 | fid := 0 |
| 125 | if len(folderid) > 0 { | ||
| 126 | fid = folderid[0] | ||
| 127 | } | ||
| 128 | |||
| 129 | if fid > 0 || all { | ||
| 91 | url.WriteString("dashared/gallection/contents") | 130 | url.WriteString("dashared/gallection/contents") |
| 92 | if all { | 131 | if all { |
| 93 | url.WriteString("?all_folder=true") | 132 | url.WriteString("?all_folder=true") |
| @@ -109,19 +148,31 @@ func (s Group) Favourites(page int, all bool, folderid ...int) (g Group, err Err | |||
| 109 | return | 148 | return |
| 110 | } | 149 | } |
| 111 | 150 | ||
| 112 | // гарелея пользователя или группы | 151 | // Gallery retrieves a page of the profile's gallery, 50 deviations at a time. |
| 152 | // Pass a positive folderid to read one folder, or 0 for the whole gallery. | ||
| 153 | // | ||
| 154 | // folderid is optional; omitting it is the same as passing 0. Only the first | ||
| 155 | // value is used. | ||
| 156 | // | ||
| 157 | // Note that page is interpreted differently by the two paths this takes: the | ||
| 158 | // whole-gallery listing is zero-based, while a folder listing is one-based. | ||
| 113 | func (s Group) Gallery(page int, folderid ...int) (g Group, daError Error, err error) { | 159 | func (s Group) Gallery(page int, folderid ...int) (g Group, daError Error, err error) { |
| 114 | if s.Name == "" { | 160 | if s.Name == "" { |
| 115 | return g, daError, errors.New("missing Name field") | 161 | return g, daError, errors.New("missing Name field") |
| 116 | } | 162 | } |
| 117 | 163 | ||
| 164 | fid := 0 | ||
| 165 | if len(folderid) > 0 { | ||
| 166 | fid = folderid[0] | ||
| 167 | } | ||
| 168 | |||
| 118 | var url strings.Builder | 169 | var url strings.Builder |
| 119 | if folderid[0] > 0 { | 170 | if fid > 0 { |
| 120 | page-- | 171 | page-- |
| 121 | url.WriteString("dashared/gallection/contents?username=") | 172 | url.WriteString("dashared/gallection/contents?username=") |
| 122 | url.WriteString(s.Name) | 173 | url.WriteString(s.Name) |
| 123 | url.WriteString("&folderid=") | 174 | url.WriteString("&folderid=") |
| 124 | url.WriteString(strconv.Itoa(folderid[0])) | 175 | url.WriteString(strconv.Itoa(fid)) |
| 125 | url.WriteString("&offset=") | 176 | url.WriteString("&offset=") |
| 126 | url.WriteString(strconv.Itoa(page * 50)) | 177 | url.WriteString(strconv.Itoa(page * 50)) |
| 127 | url.WriteString("&type=gallery&") | 178 | url.WriteString("&type=gallery&") |
| @@ -139,10 +190,14 @@ func (s Group) Gallery(page int, folderid ...int) (g Group, daError Error, err e | |||
| 139 | return | 190 | return |
| 140 | } | 191 | } |
| 141 | 192 | ||
| 193 | // GroupAbout is a group's about page: when it was founded and its description. | ||
| 142 | type GroupAbout struct { | 194 | type GroupAbout struct { |
| 143 | FoundatedAt timeStamp `json:"foundationTs"` | 195 | FoundatedAt timeStamp `json:"foundationTs"` |
| 144 | Description Text | 196 | Description Text |
| 145 | } | 197 | } |
| 198 | |||
| 199 | // GroupAdmins lists a group's staff. TypeId encodes each member's role | ||
| 200 | // (founder, co-founder, contributor). | ||
| 146 | type GroupAdmins struct { | 201 | type GroupAdmins struct { |
| 147 | Results []struct { | 202 | Results []struct { |
| 148 | TypeId int | 203 | TypeId int |
| @@ -152,10 +207,14 @@ type GroupAdmins struct { | |||
| 152 | } | 207 | } |
| 153 | } | 208 | } |
| 154 | 209 | ||
| 210 | // About is a person's profile information, all of it self-reported and any of | ||
| 211 | // it possibly empty. | ||
| 155 | type About struct { | 212 | type About struct { |
| 156 | Country, Website, WebsiteLabel, Gender string | 213 | Country, Website, WebsiteLabel, Gender string |
| 157 | RegDate int64 `json:"deviantFor"` | 214 | // RegDate is how long the account has existed, in seconds — an age, not a |
| 158 | Description Text `json:"textContent"` | 215 | // registration date, despite the name. |
| 216 | RegDate int64 `json:"deviantFor"` | ||
| 217 | Description Text `json:"textContent"` | ||
| 159 | 218 | ||
| 160 | SocialLinks []struct { | 219 | SocialLinks []struct { |
| 161 | Value string | 220 | Value string |
| @@ -165,6 +224,8 @@ type About struct { | |||
| 165 | } | 224 | } |
| 166 | } | 225 | } |
| 167 | 226 | ||
| 227 | // users is the person-specific half of a profile's module data, embedded into | ||
| 228 | // [GRuser] so its fields surface inline. | ||
| 168 | type users struct { | 229 | type users struct { |
| 169 | About About | 230 | About About |
| 170 | CoverDeviation struct { | 231 | CoverDeviation struct { |
user-group_test.go added +43
| @@ -0,0 +1,43 @@ | |||
| 1 | package devianter | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "testing" | ||
| 5 | "time" | ||
| 6 | ) | ||
| 7 | |||
| 8 | // Regression: Favourites and Gallery took folderid as a variadic, which invites | ||
| 9 | // omitting it, but then indexed folderid[0] with no length check — so the call | ||
| 10 | // the signature invites most panicked with index out of range. | ||
| 11 | // | ||
| 12 | // Both reach a request before returning, and neither takes a base URL, so these | ||
| 13 | // squeeze Timeout down to make that request fail immediately. The failure is | ||
| 14 | // expected and ignored: only the absence of a panic is under test. | ||
| 15 | func TestFolderidIsOptional(t *testing.T) { | ||
| 16 | defer func(d time.Duration) { Timeout = d }(Timeout) | ||
| 17 | Timeout = time.Millisecond | ||
| 18 | |||
| 19 | s := Group{Name: "someuser"} | ||
| 20 | |||
| 21 | t.Run("Gallery", func(t *testing.T) { | ||
| 22 | if _, _, err := s.Gallery(0); err != nil { | ||
| 23 | t.Errorf("omitting folderid is not an argument error, got %v", err) | ||
| 24 | } | ||
| 25 | }) | ||
| 26 | |||
| 27 | t.Run("Favourites all", func(t *testing.T) { | ||
| 28 | _, _ = s.Favourites(0, true) | ||
| 29 | }) | ||
| 30 | |||
| 31 | t.Run("Favourites default listing", func(t *testing.T) { | ||
| 32 | _, _ = s.Favourites(0, false) | ||
| 33 | }) | ||
| 34 | } | ||
| 35 | |||
| 36 | // Name is what every request is scoped to, so Gallery reports its absence | ||
| 37 | // rather than requesting a URL with an empty username. | ||
| 38 | func TestGalleryRequiresName(t *testing.T) { | ||
| 39 | var s Group | ||
| 40 | if _, _, err := s.Gallery(0, 0); err == nil { | ||
| 41 | t.Fatal("want an error for a Group with no Name, got nil") | ||
| 42 | } | ||
| 43 | } | ||
util.go +55 −8
| @@ -10,13 +10,22 @@ import ( | |||
| 10 | "time" | 10 | "time" |
| 11 | ) | 11 | ) |
| 12 | 12 | ||
| 13 | // функция для высера ошибки в stderr | 13 | // try prints a non-nil error to stderr and swallows it. It is how this package |
| 14 | // reports problems it does not propagate, such as a response that parsed only | ||
| 15 | // partially. | ||
| 14 | func try(txt error) { | 16 | func try(txt error) { |
| 15 | if txt != nil { | 17 | if txt != nil { |
| 16 | println(txt.Error()) | 18 | println(txt.Error()) |
| 17 | } | 19 | } |
| 18 | } | 20 | } |
| 19 | 21 | ||
| 22 | // ujson fetches a _puppy endpoint and unmarshals the response into output. | ||
| 23 | // data is the path and query string after the endpoint root, without a leading | ||
| 24 | // slash and without the csrf_token parameter, which puppy appends. | ||
| 25 | // | ||
| 26 | // A malformed response is reported through try and leaves output partially | ||
| 27 | // populated, so a returned Error with an empty Reason does not by itself | ||
| 28 | // guarantee that output is complete. | ||
| 20 | func ujson(data string, output any) Error { | 29 | func ujson(data string, output any) Error { |
| 21 | input, err := puppy(data) | 30 | input, err := puppy(data) |
| 22 | if err == nil { | 31 | if err == nil { |
| @@ -25,12 +34,23 @@ func ujson(data string, output any) Error { | |||
| 25 | return APIError(err) | 34 | return APIError(err) |
| 26 | } | 35 | } |
| 27 | 36 | ||
| 37 | // Error is a failed API call. It is a struct rather than an error interface, so | ||
| 38 | // a zero value means success: test Reason for emptiness rather than comparing | ||
| 39 | // against nil. | ||
| 40 | // | ||
| 41 | // For errors DeviantArt itself reports, Reason and Error hold its machine and | ||
| 42 | // human readable descriptions. For anything else (a transport failure, or a | ||
| 43 | // CloudFront block page) Reason is "request_failed" and Error carries the | ||
| 44 | // underlying message. | ||
| 28 | type Error struct { | 45 | type Error struct { |
| 29 | Reason string `json:"error"` | 46 | Reason string `json:"error"` |
| 30 | Error string `json:"errorDescription"` | 47 | Error string `json:"errorDescription"` |
| 31 | RAW []byte `json:"-"` | 48 | RAW []byte `json:"-"` |
| 32 | } | 49 | } |
| 33 | 50 | ||
| 51 | // APIError converts an error from the request layer into an [Error], decoding | ||
| 52 | // DeviantArt's JSON error body when that is what it is. A nil input yields the | ||
| 53 | // zero Error, which signals success. | ||
| 34 | func APIError(inputError error) (err Error) { | 54 | func APIError(inputError error) (err Error) { |
| 35 | if inputError != nil { | 55 | if inputError != nil { |
| 36 | err.RAW = []byte(inputError.Error()) | 56 | err.RAW = []byte(inputError.Error()) |
| @@ -46,7 +66,8 @@ func APIError(inputError error) (err Error) { | |||
| 46 | } | 66 | } |
| 47 | 67 | ||
| 48 | /* REQUEST SECTION */ | 68 | /* REQUEST SECTION */ |
| 49 | // структура для ответа сервера | 69 | // reqrt is a completed HTTP response, flattened into the pieces this package |
| 70 | // needs. On a transport failure Err is set and every other field is zero. | ||
| 50 | type reqrt struct { | 71 | type reqrt struct { |
| 51 | Body string | 72 | Body string |
| 52 | Status int | 73 | Status int |
| @@ -56,17 +77,21 @@ type reqrt struct { | |||
| 56 | Err error | 77 | Err error |
| 57 | } | 78 | } |
| 58 | 79 | ||
| 59 | // функция для совершения запроса | 80 | // UserAgent overrides the browser User-Agent this package sends by default. |
| 81 | // Setting it to something that identifies your client is polite, but DeviantArt | ||
| 82 | // is more likely to serve a block page to a non-browser agent. | ||
| 60 | var UserAgent string | 83 | var UserAgent string |
| 61 | 84 | ||
| 62 | // Timeout bounds a single request end-to-end (dial, response, body read). | 85 | // Timeout bounds a single request end-to-end (dial, response, body read). |
| 63 | // Without it, a hung connection blocks its caller forever. | 86 | // Without it, a hung connection blocks its caller forever. |
| 64 | var Timeout = 30 * time.Second | 87 | var Timeout = 30 * time.Second |
| 65 | 88 | ||
| 89 | // request performs a GET and never panics or returns a partial response without | ||
| 90 | // saying so: any failure is reported in reqrt.Err. An optional second argument | ||
| 91 | // supplies the Cookie header. | ||
| 66 | func request(uri string, other ...string) reqrt { | 92 | func request(uri string, other ...string) reqrt { |
| 67 | var r reqrt | 93 | var r reqrt |
| 68 | 94 | ||
| 69 | // создаём новый запрос | ||
| 70 | // Transport is deliberately left nil so http.DefaultTransport applies: that | 95 | // Transport is deliberately left nil so http.DefaultTransport applies: that |
| 71 | // keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit). | 96 | // keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit). |
| 72 | cli := &http.Client{Timeout: Timeout} | 97 | cli := &http.Client{Timeout: Timeout} |
| @@ -77,9 +102,10 @@ func request(uri string, other ...string) reqrt { | |||
| 77 | return r | 102 | return r |
| 78 | } | 103 | } |
| 79 | 104 | ||
| 105 | // Impersonate a browser by default: the endpoints are the web frontend's own, | ||
| 106 | // and an unfamiliar agent draws a block page. | ||
| 80 | req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") | 107 | req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") |
| 81 | 108 | ||
| 82 | // куки и UA-шник | ||
| 83 | if UserAgent != "" { | 109 | if UserAgent != "" { |
| 84 | req.Header.Set("User-Agent", UserAgent) | 110 | req.Header.Set("User-Agent", UserAgent) |
| 85 | } | 111 | } |
| @@ -107,7 +133,6 @@ func request(uri string, other ...string) reqrt { | |||
| 107 | r.Err = e | 133 | r.Err = e |
| 108 | } | 134 | } |
| 109 | 135 | ||
| 110 | // заполняем структуру | ||
| 111 | r.Body = string(body) | 136 | r.Body = string(body) |
| 112 | r.Cookies = resp.Cookies() | 137 | r.Cookies = resp.Cookies() |
| 113 | r.Headers = resp.Header | 138 | r.Headers = resp.Header |
| @@ -145,7 +170,11 @@ func describe(r reqrt) string { | |||
| 145 | } | 170 | } |
| 146 | 171 | ||
| 147 | /* PUPPY aka DeviantArt API */ | 172 | /* PUPPY aka DeviantArt API */ |
| 148 | // получение или обновление токена | 173 | // The guest session: a cookie from the _puppy endpoint and a CSRF token scraped |
| 174 | // from the homepage. UpdateCSRF populates both; puppy sends them on every call. | ||
| 175 | // | ||
| 176 | // These are package-level and unsynchronised, so a program that calls UpdateCSRF | ||
| 177 | // concurrently with any other function of this package races on them. | ||
| 149 | var cookie string | 178 | var cookie string |
| 150 | var token string | 179 | var token string |
| 151 | 180 | ||
| @@ -154,6 +183,18 @@ const ( | |||
| 154 | xhrMarker = "window.__XHR_LOCAL__" | 183 | xhrMarker = "window.__XHR_LOCAL__" |
| 155 | ) | 184 | ) |
| 156 | 185 | ||
| 186 | // UpdateCSRF establishes the guest session that every other call in this package | ||
| 187 | // depends on, and must be called before them. It fetches a session cookie (only | ||
| 188 | // on the first call; later calls reuse it) and scrapes a fresh CSRF token from | ||
| 189 | // the DeviantArt homepage. | ||
| 190 | // | ||
| 191 | // Tokens expire, so a long-running program should call this again when requests | ||
| 192 | // begin to fail. It is not safe to call concurrently with other functions of | ||
| 193 | // this package. | ||
| 194 | // | ||
| 195 | // An error means the session was not established: the homepage was blocked, | ||
| 196 | // served a challenge, or changed its markup such that the token is no longer | ||
| 197 | // where this package looks for it. | ||
| 157 | func UpdateCSRF() error { | 198 | func UpdateCSRF() error { |
| 158 | if cookie == "" { | 199 | if cookie == "" { |
| 159 | req := request("https://www.deviantart.com/_puppy") | 200 | req := request("https://www.deviantart.com/_puppy") |
| @@ -187,6 +228,13 @@ func UpdateCSRF() error { | |||
| 187 | return nil | 228 | return nil |
| 188 | } | 229 | } |
| 189 | 230 | ||
| 231 | // puppy calls a _puppy endpoint with the guest session applied and returns the | ||
| 232 | // raw JSON body. data is a path and query string; the CSRF token and API version | ||
| 233 | // are appended to it, so it must already end in a parameter (callers conclude | ||
| 234 | // theirs with a trailing "&" or a final value). | ||
| 235 | // | ||
| 236 | // It returns an error for a transport failure, a non-200 status, or a 200 whose | ||
| 237 | // body is not JSON, which is how a CDN block page arrives. | ||
| 190 | func puppy(data string) (string, error) { | 238 | func puppy(data string) (string, error) { |
| 191 | var url strings.Builder | 239 | var url strings.Builder |
| 192 | url.WriteString("https://www.deviantart.com/_puppy/") | 240 | url.WriteString("https://www.deviantart.com/_puppy/") |
| @@ -200,7 +248,6 @@ func puppy(data string) (string, error) { | |||
| 200 | return "", body.Err | 248 | return "", body.Err |
| 201 | } | 249 | } |
| 202 | 250 | ||
| 203 | // если код ответа не 200, возвращается ошибка | ||
| 204 | if body.Status != 200 { | 251 | if body.Status != 200 { |
| 205 | return "", errors.New(describe(body)) | 252 | return "", errors.New(describe(body)) |
| 206 | } | 253 | } |