added worker code

compareID.go

@@ -52,7 +52,7 @@ type MatchedAnime struct {
 	Score       int
 }
 
-//check does a default error check
+//Check does a default error check
 func check(e error) {
 	if e != nil {
 		panic(e)
@@ -115,62 +115,66 @@ func SubAnimeDates(Date, Date2 time.Time) (total int) {
 //checkResults takes an Anime struct and a channel.
 //It compares the supplied anime to all the search results returned from searching the title.
 //If the score is over 100 it should be a match.
-func CheckResults(anime Anime, c chan MatchedAnime) {
+func CheckResults(jobs chan Anime, c chan MatchedAnime) {
-	fmt.Println(`Searching anime`, anime.ID)
+	for anime := range jobs {
-	var highest, hI int
+		fmt.Println(`Searching anime`, anime.ID)
-	var err error
+		var highest, hI int
-	anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
+		var err error
-	var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`))
+		anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
-	resp, err := http.Post("https://api.meikan.moe/v1/anime?incl=start_date,end_date", "application/json", search)
+		var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`))
-	check(err)
+		resp, err := http.Post("https://api.meikan.moe/v1/anime", "application/json", search)
-	fmt.Println(`Completed meikan search for`, anime.ID)
+		check(err)
-	// nolint: errcheck
+		fmt.Println(`Completed meikan search for`, anime.ID)
-	defer resp.Body.Close()
+		//fmt.Println(resp)
-	var result Result
+		// nolint: errcheck
-	body, err := ioutil.ReadAll(resp.Body)
+		defer resp.Body.Close()
-	check(err)
+		var result Result
-	_ = json.NewDecoder(bytes.NewReader(body)).Decode(&result)
+		body, err := ioutil.ReadAll(resp.Body)
-	anime.Type = FixType(anime.Type)
+		check(err)
-	if anime.Episodes < 1 {
+		_ = json.NewDecoder(bytes.NewReader(body)).Decode(&result)
-		anime.Episodes = 1
+		anime.Type = FixType(anime.Type)
-	}
+		if anime.Episodes < 1 {
-	for i := 0; i < len(result.Anime); i++ {
+			anime.Episodes = 1
-		var total int
+		}
-		meikan := result.Anime[i]
+		for i := 0; i < len(result.Anime); i++ {
-		stDate, enDate := FixMeikanDates(meikan.StartDate, meikan.EndDate)
+			var total int
-		if meikan.Type == anime.Type {
+			meikan := result.Anime[i]
-			total += 25
+			stDate, enDate := FixMeikanDates(meikan.StartDate, meikan.EndDate)
-		} else {
+			if meikan.Type == anime.Type {
-			MType := LessType(meikan.Type)
+				total += 25
-			AType := LessType(anime.Type)
+			} else {
-			if MType == AType {
+				MType := LessType(meikan.Type)
-				total += 12
+				AType := LessType(anime.Type)
+				if MType == AType {
+					total += 12
+				}
+			}
+			score := levenshtein.Match(meikan.Title, anime.Title, nil)
+			if score > 0.95 {
+				total += 50
+			}
+			if meikan.Episodes == anime.Episodes {
+				total += 25
+			}
+			total += SubAnimeDates(stDate, anime.StartDate)
+			if anime.Episodes != 1 {
+				total += SubAnimeDates(enDate, anime.EndDate)
+			}
+			if total > highest {
+				highest = total
+				hI = i
 			}
 		}
-		score := levenshtein.Match(meikan.Title, anime.Title, nil)
+		//return highest, hI
-		if score > 0.95 {
+		//fmt.Println(len(jobs))
-			total += 50
+		if len(result.Anime) != 0 {
-		}
+			c <- MatchedAnime{
-		if meikan.Episodes == anime.Episodes {
+				MeikanTitle: result.Anime[hI].Title,
-			total += 25
+				AnidbTitle:  anime.Title,
-		}
+				MeikanID:    result.Anime[hI].ID,
-		total += SubAnimeDates(stDate, anime.StartDate)
+				AnidbID:     anime.ID,
-		if anime.Episodes != 1 {
+				Score:       highest,
-			total += SubAnimeDates(enDate, anime.EndDate)
+			}
-		}
-		if total > highest {
-			highest = total
-			hI = i
-		}
-	}
-	//return highest, hI
-	if len(result.Anime) != 0 {
-		c <- MatchedAnime{
-			MeikanTitle: result.Anime[hI].Title,
-			AnidbTitle:  anime.Title,
-			MeikanID:    result.Anime[hI].ID,
-			AnidbID:     anime.ID,
-			Score:       highest,
 		}
 	}
 }

thing/main.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"git.fuyu.moe/Tracreed/compareID"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func check(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
+type Result struct {
+	EndDate   string `json:"end_date"`
+	Episodes  int    `json:"total_episodes"`
+	ID        int    `json:"id"`
+	StartDate string `json:"start_date"`
+	Title     string `json:"title_english"`
+	Type      string `json:"type"`
+}
+
+//fixAnidbDates checks if a string slice with the length of two have matching dates
+func FixAnidbDates(Dates []string) (startDate, endDate time.Time) {
+	if !strings.ContainsAny(Dates[0], "?") && Dates[0] != "" {
+		startDate, _ = time.Parse(`02.01.2006`, Dates[0])
+	}
+	if len(Dates) == 2 {
+		if !strings.ContainsAny(Dates[1], "?") && Dates[1] != "" {
+			endDate, _ = time.Parse(`02.01.2006`, Dates[1])
+		}
+	}
+	return startDate, endDate
+}
+
+func ParseCsv(jobs chan compareID.Anime, c chan compareID.MatchedAnime, sleep time.Duration) {
+	dat, err := ioutil.ReadFile("/home/trac/coding/compareID/1.-Main-data.csv")
+	check(err)
+	r := csv.NewReader(strings.NewReader(string(dat)))
+	_, err = r.Read()
+	check(err)
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+
+		}
+		EpRegex := regexp.MustCompile(`.*, (\d*) .*`)
+
+		ep := strings.Split(record[11], ",")
+		eps := 1
+
+		episodes, err := strconv.Atoi(EpRegex.ReplaceAllString(record[11], "$1"))
+		if err == nil {
+			eps = episodes
+		}
+		id, err := strconv.Atoi(record[1])
+		check(err)
+		Date := strings.Split(record[12], " till ")
+		startDate, endDate := FixAnidbDates(Date)
+		a := compareID.Anime{
+			StartDate: startDate,
+			EndDate:   endDate,
+			ID:        id,
+			Type:      strings.TrimSpace(ep[0]),
+			Episodes:  eps,
+			Title:     strings.Replace(record[3], "Anime: ", "", 1),
+		}
+		if a.Title == "ERROR" || a.Title == "?" {
+			continue
+		}
+		jobs <- a
+		//go compareID.CheckResults(a, c)
+
+		//time.Sleep(sleep * time.Millisecond)
+	}
+	fmt.Println("Finished parsing csv")
+}
+
+func main() {
+	http.DefaultTransport = &http.Transport{MaxIdleConnsPerHost: 50, MaxIdleConns: 50}
+	c := make(chan compareID.MatchedAnime)
+	jobs := make(chan compareID.Anime)
+	var animes compareID.Animes
+	/*client := &http.Client{}
+	req, err := http.NewRequest("GET", "https://anilist.co/api/anime/21898/page", nil)
+	req.Header.Add("x-csrf-token", `jN6gFN0UdzpxSrZUHcT8RW6h4hJpgqKv85la63ON`)
+	req.Header.Add(`cookie`, `__cfduid=d254ce47b42a2a6eb3428bea8dbd6dfec1508141286; XSRF-TOKEN=jN6gFN0UdzpxSrZUHcT8RW6h4hJpgqKv85la63ON; laravel_session=eyJpdiI6Im5ENmxURHQwdWt4WldJelc5clBUbXc9PSIsInZhbHVlIjoid2NnakM0RGMwRDR3K09pelM3M045SlJSNm5tRnk1Y0cwQThcL01xTkhva0hLd2tEOElwcDBYWjdWWXNPUTVzWnpXVk5uaUJWb3o4WnBvaFpsWWZFQkFnPT0iLCJtYWMiOiJlODlkMmJiNzQ1ZTg3NjM0NDFiZTljNGVhZDQzZGRlMDhhNWMyNWNiMmIwZWYwZjI5MTg3ODk0YjRmYWVmYTA4In0%3D`)
+	resp, err := client.Do(req)
+	check(err)
+	// nolint: errcheck
+	defer resp.Body.Close()
+	var result Result
+	body, err := ioutil.ReadAll(resp.Body)
+	check(err)
+	_ = json.NewDecoder(bytes.NewReader(body)).Decode(&result)*/
+	for i := 1; i <= 50; i++ {
+		go compareID.CheckResults(jobs, c)
+	}
+	go ParseCsv(jobs, c, 7)
+
+	//fmt.Println(result)
+	/*
+		a := compareID.Anime{
+			StartDate: result.StartDate,
+			EndDate:   result.EndDate,
+			ID:        result.ID,
+			Type:      result.Type,
+			Episodes:  result.Episodes,
+			Title:     result.Title,
+		}
+
+		go compareID.CheckResults(a, c)*/
+MainLoop:
+	for {
+		select {
+		case m := <-c:
+			animes = append(animes, m)
+		case <-time.After(2 * time.Second):
+			break MainLoop
+		}
+	}
+	//fmt.Println(animes)
+	resJSON, err := json.MarshalIndent(animes, "", "\t")
+	check(err)
+	jsonfile := []byte(resJSON)
+	err = ioutil.WriteFile("./result.json", jsonfile, 0644)
+	check(err)
+}