Commit 42ea2980f9

42ea2980f9be365c97b17d5273270fa1862ee9fc

parent: 4d166ad5f9

Unsigned

lost+skunk <skunky@ebloid.ru> · 2024-07-29 22:20 UTC

Парсер медии с кастомным разрешением

Layout: unified · split

.vscode/launch.json deleted −16
@@ -1,16 +0,0 @@
1{
2 // Use IntelliSense to learn about possible attributes.
3 // Hover to view descriptions of existing attributes.
4 // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
5 "version": "0.2.0",
6 "configurations": [
7 {
8 "name": "Launch Package",
9 "type": "go",
10 "request": "launch",
11 "mode": "debug",
12 "program": "${fileDirname}",
13 "showLog": true
14 }
15 ]
16}
\ No newline at end of file
.vscode/settings.json deleted −5
@@ -1,5 +0,0 @@
1{
2 "go.toolsEnvVars": {
3 "GOROOT": ""
4 }
5}
\ No newline at end of file
README.md +2 −2
@@ -1,9 +1,9 @@
11# Devianter
22
3[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
3[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter)
44
55
6A DeviantART API library for Go.
6A DeviantART guest API library for Go.
77
88
99I'll probably write up some documentation, but more on that later.
comments.go +3 −3
@@ -11,7 +11,7 @@ type Thread struct {
1111 ID int `json:"commentId"`
1212 Parent int `json:"parentId"`
1313
14 Posted time
14 Posted timeStamp
1515 Author bool `json:"isAuthorHighlited"`
1616
1717 Desctiption string
@@ -35,7 +35,7 @@ type Comments struct {
3535}
3636
3737// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя
38func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Comments) {
38func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) {
3939 for x := 0; x <= page; x++ {
4040 ujson(
4141 "dashared/comments/thread?typeid="+strconv.Itoa(typ)+
@@ -60,7 +60,7 @@ func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Commen
6060 }
6161
6262 e := json.Unmarshal([]byte(m), &content)
63 err(e)
63 try(e)
6464
6565 for _, a := range content.Blocks {
6666 cmmts.Thread[i].Comment = a.Text
deviantion.go +23 −9
@@ -4,26 +4,26 @@ import (
44 "encoding/json"
55 "strconv"
66 "strings"
7 timelib "time"
7 "time"
88)
99
1010// хрень для парсинга времени публикации
11type time struct {
12 timelib.Time
11type timeStamp struct {
12 time.Time
1313}
1414
15func (t *time) UnmarshalJSON(b []byte) (err error) {
15func (t *timeStamp) UnmarshalJSON(b []byte) (err error) {
1616 if b[0] == '"' && b[len(b)-1] == '"' {
1717 b = b[1 : len(b)-1]
1818 }
19 t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b))
19 t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b))
2020 return
2121}
2222
2323// самая главная структура для поста
2424type Deviation struct {
2525 Title, Url, License string
26 PublishedTime time
26 PublishedTime timeStamp
2727 ID int `json:"deviationId"`
2828
2929 NSFW bool `json:"isMature"`
@@ -82,7 +82,7 @@ type Post struct {
8282
8383 ParsedComments []struct {
8484 Author string
85 Posted time
85 Posted timeStamp
8686 Replies, Likes int
8787 }
8888
@@ -90,12 +90,25 @@ type Post struct {
9090}
9191
9292// преобразование урла в правильный
93func UrlFromMedia(m Media) string {
93func UrlFromMedia(m Media, thumb ...int) string {
9494 var url strings.Builder
95
96 subtractWidthHeight := func(to int, target ...*int) {
97 for i, l := 0, len(target); i < l; i++ {
98 for x := *target[i]; x > to; x -= to {
99 *target[i] = x
100 }
101 }
102 }
103
95104 for _, t := range m.Types {
96105 if t.T == "fullview" {
97106 url.WriteString(m.BaseUri)
98107 if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 {
108 if len(thumb) != 0 {
109 subtractWidthHeight(thumb[0], &t.W, &t.H)
110 }
111
99112 url.WriteString("/v1/fit/w_")
100113 url.WriteString(strconv.Itoa(t.W))
101114 url.WriteString(",h_")
@@ -110,11 +123,12 @@ func UrlFromMedia(m Media) string {
110123 }
111124 }
112125 }
126
113127 return url.String()
114128}
115129
116130// для работы функции нужно ID поста и имя пользователя.
117func DeviationFunc(id string, user string) Post {
131func GetDeviation(id string, user string) Post {
118132 var st Post
119133 ujson(
120134 "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true",
misc.go +20 −18
@@ -4,13 +4,16 @@ import (
44 "errors"
55 "log"
66 "math"
7 u "net/url"
7 "net/url"
88 "strconv"
99 "strings"
1010)
1111
1212/* AVATARS AND EMOJIS */
1313func AEmedia(name string, t rune) (string, error) {
14 if len(name) < 2 {
15 return "", errors.New("name must be specified")
16 }
1417 // список всех возможных расширений
1518 var extensions = [3]string{
1619 ".jpg",
@@ -25,9 +28,10 @@ func AEmedia(name string, t rune) (string, error) {
2528 switch t {
2629 case 'a':
2730 b.WriteString("https://a.deviantart.net/avatars-big/")
28 b.WriteString(name[:1])
31 name_without_dashes := strings.ReplaceAll(name, "-", "_")
32 b.WriteString(name_without_dashes[:1])
2933 b.WriteString("/")
30 b.WriteString(name[1:2])
34 b.WriteString(name_without_dashes[1:2])
3135 b.WriteString("/")
3236 case 'e':
3337 b.WriteString("https://e.deviantart.net/emoticons/")
@@ -60,7 +64,7 @@ type DailyDeviations struct {
6064 Deviations []Deviation
6165}
6266
63func DailyDeviationsFunc(page int) (dd DailyDeviations) {
67func GetDailyDeviations(page int) (dd DailyDeviations) {
6468 ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd)
6569 return
6670}
@@ -74,23 +78,21 @@ type Search struct {
7478 ResultsGalleryTemp []Deviation `json:"results"`
7579}
7680
77func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) {
78 var url strings.Builder
81func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) {
82 var buildurl strings.Builder
7983 e = nil
8084
8185 // о5 построение ссылок.
8286 switch scope {
8387 case 'a': // поиск артов по названию
84 url.WriteString("dabrowse/search/all?q=")
88 buildurl.WriteString("dabrowse/search/all?q=")
8589 case 't': // поиск артов по тегам
86 url.WriteString("dabrowse/networkbar/tag/deviations?tag=")
90 buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=")
8791 case 'g': // поиск артов пользователя или группы
8892 if user != nil {
89 url.WriteString("dashared/gallection/search?username=")
90 for _, a := range user {
91 url.WriteString(a)
92 }
93 url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
93 buildurl.WriteString("dashared/gallection/search?username=")
94 buildurl.WriteString(user[0])
95 buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
9496 } else {
9597 e = errors.New("missing username (last argument)")
9698 return
@@ -99,16 +101,16 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search,
99101 log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.")
100102 }
101103
102 url.WriteString(u.QueryEscape(query))
104 buildurl.WriteString(url.QueryEscape(query))
103105 if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код
104 url.WriteString("&page=")
106 buildurl.WriteString("&page=")
105107 } else { // иначе вместо страницы будет оффсет и страница умножится на 50
106 url.WriteString("&offset=")
108 buildurl.WriteString("&offset=")
107109 page = 50 * page
108110 }
109 url.WriteString(strconv.Itoa(page))
111 buildurl.WriteString(strconv.Itoa(page))
110112
111 ujson(url.String(), &ss)
113 ujson(buildurl.String(), &ss)
112114
113115 if scope == 'g' {
114116 ss.Results = ss.ResultsGalleryTemp
todo.md deleted −2
@@ -1,2 +0,0 @@
1- groups
2- images in comments
\ No newline at end of file
user-group.go +43 −35
@@ -1,44 +1,12 @@
11package devianter
22
33import (
4 "errors"
45 "strconv"
56 "strings"
67)
78
89// структура группы или пользователя
9type GroupAbout struct {
10 FoundatedAt time `json:"foundationTs"`
11 Description Text
12}
13type GroupAdmins struct {
14 Results []struct {
15 TypeId int
16 User struct {
17 Username string
18 }
19 }
20}
21
22type About struct {
23 Country, Website, WebsiteLabel, Gender string
24 RegDate int64 `json:"deviantFor"`
25 Description Text `json:"textContent"`
26
27 SocialLinks []struct {
28 Value string
29 }
30 Interests []struct {
31 Label, Value string
32 }
33}
34
35type users struct {
36 About About
37 CoverDeviation struct {
38 Deviation Deviation `json:"coverDeviation"`
39 }
40}
41
4210type GRuser struct {
4311 ErrorDescription string
4412 Owner struct {
@@ -106,14 +74,21 @@ type Group struct {
10674}
10775
10876// подходит как группа, так и пользователь
109func (s Group) GroupFunc() (g GRuser) {
77func (s Group) GetGroup() (g GRuser, err error) {
78 if s.Name == "" {
79 return g, errors.New("missing Name field")
80 }
11081 ujson("dauserprofile/init/about?username="+s.Name, &g)
11182
11283 return
11384}
11485
11586// гарелея пользователя или группы
116func (s Group) Gallery(page int, folderid ...int) (g Group) {
87func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) {
88 if s.Name == "" {
89 return g, errors.New("missing Name field")
90 }
91
11792 var url strings.Builder
11893 if folderid[0] > 0 {
11994 page--
@@ -137,3 +112,36 @@ func (s Group) Gallery(page int, folderid ...int) (g Group) {
137112 ujson(url.String(), &g.Content)
138113 return
139114}
115
116type GroupAbout struct {
117 FoundatedAt timeStamp `json:"foundationTs"`
118 Description Text
119}
120type GroupAdmins struct {
121 Results []struct {
122 TypeId int
123 User struct {
124 Username string
125 }
126 }
127}
128
129type About struct {
130 Country, Website, WebsiteLabel, Gender string
131 RegDate int64 `json:"deviantFor"`
132 Description Text `json:"textContent"`
133
134 SocialLinks []struct {
135 Value string
136 }
137 Interests []struct {
138 Label, Value string
139 }
140}
141
142type users struct {
143 About About
144 CoverDeviation struct {
145 Deviation Deviation `json:"coverDeviation"`
146 }
147}
util.go +14 −16
@@ -9,7 +9,7 @@ import (
99)
1010
1111// функция для высера ошибки в stderr
12func err(txt error) {
12func try(txt error) {
1313 if txt != nil {
1414 println(txt.Error())
1515 }
@@ -17,11 +17,9 @@ func err(txt error) {
1717
1818// сокращение для вызова щенка и парсинга жсона
1919func ujson(data string, output any) {
20 input, e := puppy(data)
21 err(e)
22
23 eee := json.Unmarshal([]byte(input), output)
24 err(eee)
20 input, err := puppy(data)
21 try(err)
22 try(json.Unmarshal([]byte(input), output))
2523}
2624
2725/* REQUEST SECTION */
@@ -34,32 +32,32 @@ type reqrt struct {
3432}
3533
3634// функция для совершения запроса
35var UserAgent string
36
3737func request(uri string, other ...string) reqrt {
3838 var r reqrt
3939
4040 // создаём новый запрос
4141 cli := &http.Client{}
4242 req, e := http.NewRequest("GET", uri, nil)
43 err(e)
43 try(e)
4444
4545 req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
4646
4747 // куки и UA-шник
48 for num, rng := range other {
49 switch num {
50 case 1:
51 req.Header.Set("User-Agent", rng)
52 case 0:
53 req.Header.Set("Cookie", rng)
54 }
48 if UserAgent != "" {
49 req.Header.Set("User-Agent", UserAgent)
50 }
51 if len(other) != 0 {
52 req.Header.Set("Cookie", other[0])
5553 }
5654
5755 resp, e := cli.Do(req)
58 err(e)
56 try(e)
5957 defer resp.Body.Close()
6058
6159 body, e := io.ReadAll(resp.Body)
62 err(e)
60 try(e)
6361
6462 // заполняем структуру
6563 r.Body = string(body)