Browse Source

added worker code

David Alasow 4 months ago
parent
commit
173cadabea
3 changed files with 54813 additions and 55 deletions
  1. 59
    55
      compareID.go
  2. 138
    0
      thing/main.go
  3. 54616
    0
      thing/result.json

+ 59
- 55
compareID.go View File

@@ -52,7 +52,7 @@ type MatchedAnime struct {
52 52
 	Score       int
53 53
 }
54 54
 
55
-//check does a default error check
55
+//Check does a default error check
56 56
 func check(e error) {
57 57
 	if e != nil {
58 58
 		panic(e)
@@ -115,62 +115,66 @@ func SubAnimeDates(Date, Date2 time.Time) (total int) {
115 115
 //checkResults takes an Anime struct and a channel.
116 116
 //It compares the supplied anime to all the search results returned from searching the title.
117 117
 //If the score is over 100 it should be a match.
118
-func CheckResults(anime Anime, c chan MatchedAnime) {
119
-	fmt.Println(`Searching anime`, anime.ID)
120
-	var highest, hI int
121
-	var err error
122
-	anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
123
-	var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`))
124
-	resp, err := http.Post("https://api.meikan.moe/v1/anime?incl=start_date,end_date", "application/json", search)
125
-	check(err)
126
-	fmt.Println(`Completed meikan search for`, anime.ID)
127
-	// nolint: errcheck
128
-	defer resp.Body.Close()
129
-	var result Result
130
-	body, err := ioutil.ReadAll(resp.Body)
131
-	check(err)
132
-	_ = json.NewDecoder(bytes.NewReader(body)).Decode(&result)
133
-	anime.Type = FixType(anime.Type)
134
-	if anime.Episodes < 1 {
135
-		anime.Episodes = 1
136
-	}
137
-	for i := 0; i < len(result.Anime); i++ {
138
-		var total int
139
-		meikan := result.Anime[i]
140
-		stDate, enDate := FixMeikanDates(meikan.StartDate, meikan.EndDate)
141
-		if meikan.Type == anime.Type {
142
-			total += 25
143
-		} else {
144
-			MType := LessType(meikan.Type)
145
-			AType := LessType(anime.Type)
146
-			if MType == AType {
147
-				total += 12
148
-			}
149
-		}
150
-		score := levenshtein.Match(meikan.Title, anime.Title, nil)
151
-		if score > 0.95 {
152
-			total += 50
118
+func CheckResults(jobs chan Anime, c chan MatchedAnime) {
119
+	for anime := range jobs {
120
+		fmt.Println(`Searching anime`, anime.ID)
121
+		var highest, hI int
122
+		var err error
123
+		anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
124
+		var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`))
125
+		resp, err := http.Post("https://api.meikan.moe/v1/anime", "application/json", search)
126
+		check(err)
127
+		fmt.Println(`Completed meikan search for`, anime.ID)
128
+		//fmt.Println(resp)
129
+		// nolint: errcheck
130
+		defer resp.Body.Close()
131
+		var result Result
132
+		body, err := ioutil.ReadAll(resp.Body)
133
+		check(err)
134
+		_ = json.NewDecoder(bytes.NewReader(body)).Decode(&result)
135
+		anime.Type = FixType(anime.Type)
136
+		if anime.Episodes < 1 {
137
+			anime.Episodes = 1
153 138
 		}
154
-		if meikan.Episodes == anime.Episodes {
155
-			total += 25
156
-		}
157
-		total += SubAnimeDates(stDate, anime.StartDate)
158
-		if anime.Episodes != 1 {
159
-			total += SubAnimeDates(enDate, anime.EndDate)
160
-		}
161
-		if total > highest {
162
-			highest = total
163
-			hI = i
139
+		for i := 0; i < len(result.Anime); i++ {
140
+			var total int
141
+			meikan := result.Anime[i]
142
+			stDate, enDate := FixMeikanDates(meikan.StartDate, meikan.EndDate)
143
+			if meikan.Type == anime.Type {
144
+				total += 25
145
+			} else {
146
+				MType := LessType(meikan.Type)
147
+				AType := LessType(anime.Type)
148
+				if MType == AType {
149
+					total += 12
150
+				}
151
+			}
152
+			score := levenshtein.Match(meikan.Title, anime.Title, nil)
153
+			if score > 0.95 {
154
+				total += 50
155
+			}
156
+			if meikan.Episodes == anime.Episodes {
157
+				total += 25
158
+			}
159
+			total += SubAnimeDates(stDate, anime.StartDate)
160
+			if anime.Episodes != 1 {
161
+				total += SubAnimeDates(enDate, anime.EndDate)
162
+			}
163
+			if total > highest {
164
+				highest = total
165
+				hI = i
166
+			}
164 167
 		}
165
-	}
166
-	//return highest, hI
167
-	if len(result.Anime) != 0 {
168
-		c <- MatchedAnime{
169
-			MeikanTitle: result.Anime[hI].Title,
170
-			AnidbTitle:  anime.Title,
171
-			MeikanID:    result.Anime[hI].ID,
172
-			AnidbID:     anime.ID,
173
-			Score:       highest,
168
+		//return highest, hI
169
+		//fmt.Println(len(jobs))
170
+		if len(result.Anime) != 0 {
171
+			c <- MatchedAnime{
172
+				MeikanTitle: result.Anime[hI].Title,
173
+				AnidbTitle:  anime.Title,
174
+				MeikanID:    result.Anime[hI].ID,
175
+				AnidbID:     anime.ID,
176
+				Score:       highest,
177
+			}
174 178
 		}
175 179
 	}
176 180
 }

+ 138
- 0
thing/main.go View File

@@ -0,0 +1,138 @@
1
+package main
2
+
3
+import (
4
+	"encoding/csv"
5
+	"encoding/json"
6
+	"fmt"
7
+	"git.fuyu.moe/Tracreed/compareID"
8
+	"io"
9
+	"io/ioutil"
10
+	"net/http"
11
+	"regexp"
12
+	"strconv"
13
+	"strings"
14
+	"time"
15
+)
16
+
17
+func check(e error) {
18
+	if e != nil {
19
+		panic(e)
20
+	}
21
+}
22
+
23
+type Result struct {
24
+	EndDate   string `json:"end_date"`
25
+	Episodes  int    `json:"total_episodes"`
26
+	ID        int    `json:"id"`
27
+	StartDate string `json:"start_date"`
28
+	Title     string `json:"title_english"`
29
+	Type      string `json:"type"`
30
+}
31
+
32
+//fixAnidbDates checks if a string slice with the length of two have matching dates
33
+func FixAnidbDates(Dates []string) (startDate, endDate time.Time) {
34
+	if !strings.ContainsAny(Dates[0], "?") && Dates[0] != "" {
35
+		startDate, _ = time.Parse(`02.01.2006`, Dates[0])
36
+	}
37
+	if len(Dates) == 2 {
38
+		if !strings.ContainsAny(Dates[1], "?") && Dates[1] != "" {
39
+			endDate, _ = time.Parse(`02.01.2006`, Dates[1])
40
+		}
41
+	}
42
+	return startDate, endDate
43
+}
44
+
45
+func ParseCsv(jobs chan compareID.Anime, c chan compareID.MatchedAnime, sleep time.Duration) {
46
+	dat, err := ioutil.ReadFile("/home/trac/coding/compareID/1.-Main-data.csv")
47
+	check(err)
48
+	r := csv.NewReader(strings.NewReader(string(dat)))
49
+	_, err = r.Read()
50
+	check(err)
51
+	for {
52
+		record, err := r.Read()
53
+		if err == io.EOF {
54
+			break
55
+
56
+		}
57
+		EpRegex := regexp.MustCompile(`.*, (\d*) .*`)
58
+
59
+		ep := strings.Split(record[11], ",")
60
+		eps := 1
61
+
62
+		episodes, err := strconv.Atoi(EpRegex.ReplaceAllString(record[11], "$1"))
63
+		if err == nil {
64
+			eps = episodes
65
+		}
66
+		id, err := strconv.Atoi(record[1])
67
+		check(err)
68
+		Date := strings.Split(record[12], " till ")
69
+		startDate, endDate := FixAnidbDates(Date)
70
+		a := compareID.Anime{
71
+			StartDate: startDate,
72
+			EndDate:   endDate,
73
+			ID:        id,
74
+			Type:      strings.TrimSpace(ep[0]),
75
+			Episodes:  eps,
76
+			Title:     strings.Replace(record[3], "Anime: ", "", 1),
77
+		}
78
+		if a.Title == "ERROR" || a.Title == "?" {
79
+			continue
80
+		}
81
+		jobs <- a
82
+		//go compareID.CheckResults(a, c)
83
+
84
+		//time.Sleep(sleep * time.Millisecond)
85
+	}
86
+	fmt.Println("Finished parsing csv")
87
+}
88
+
89
+func main() {
90
+	http.DefaultTransport = &http.Transport{MaxIdleConnsPerHost: 50, MaxIdleConns: 50}
91
+	c := make(chan compareID.MatchedAnime)
92
+	jobs := make(chan compareID.Anime)
93
+	var animes compareID.Animes
94
+	/*client := &http.Client{}
95
+	req, err := http.NewRequest("GET", "https://anilist.co/api/anime/21898/page", nil)
96
+	req.Header.Add("x-csrf-token", `jN6gFN0UdzpxSrZUHcT8RW6h4hJpgqKv85la63ON`)
97
+	req.Header.Add(`cookie`, `__cfduid=d254ce47b42a2a6eb3428bea8dbd6dfec1508141286; XSRF-TOKEN=jN6gFN0UdzpxSrZUHcT8RW6h4hJpgqKv85la63ON; laravel_session=eyJpdiI6Im5ENmxURHQwdWt4WldJelc5clBUbXc9PSIsInZhbHVlIjoid2NnakM0RGMwRDR3K09pelM3M045SlJSNm5tRnk1Y0cwQThcL01xTkhva0hLd2tEOElwcDBYWjdWWXNPUTVzWnpXVk5uaUJWb3o4WnBvaFpsWWZFQkFnPT0iLCJtYWMiOiJlODlkMmJiNzQ1ZTg3NjM0NDFiZTljNGVhZDQzZGRlMDhhNWMyNWNiMmIwZWYwZjI5MTg3ODk0YjRmYWVmYTA4In0%3D`)
98
+	resp, err := client.Do(req)
99
+	check(err)
100
+	// nolint: errcheck
101
+	defer resp.Body.Close()
102
+	var result Result
103
+	body, err := ioutil.ReadAll(resp.Body)
104
+	check(err)
105
+	_ = json.NewDecoder(bytes.NewReader(body)).Decode(&result)*/
106
+	for i := 1; i <= 50; i++ {
107
+		go compareID.CheckResults(jobs, c)
108
+	}
109
+	go ParseCsv(jobs, c, 7)
110
+
111
+	//fmt.Println(result)
112
+	/*
113
+		a := compareID.Anime{
114
+			StartDate: result.StartDate,
115
+			EndDate:   result.EndDate,
116
+			ID:        result.ID,
117
+			Type:      result.Type,
118
+			Episodes:  result.Episodes,
119
+			Title:     result.Title,
120
+		}
121
+
122
+		go compareID.CheckResults(a, c)*/
123
+MainLoop:
124
+	for {
125
+		select {
126
+		case m := <-c:
127
+			animes = append(animes, m)
128
+		case <-time.After(2 * time.Second):
129
+			break MainLoop
130
+		}
131
+	}
132
+	//fmt.Println(animes)
133
+	resJSON, err := json.MarshalIndent(animes, "", "\t")
134
+	check(err)
135
+	jsonfile := []byte(resJSON)
136
+	err = ioutil.WriteFile("./result.json", jsonfile, 0644)
137
+	check(err)
138
+}

+ 54616
- 0
thing/result.json
File diff suppressed because it is too large
View File


Loading…
Cancel
Save