package main import ( "bytes" "encoding/csv" "encoding/json" "fmt" "github.com/agext/levenshtein" "io" "io/ioutil" "net/http" "regexp" "strconv" "strings" "time" ) //Anime is AniDB structure type Anime struct { ID int Type string Episodes int Title string StartDate time.Time EndDate time.Time } //Result contains the results from the meikan search type Result struct { Total int `json:"total"` Anime []AnimeRes `json:"data"` } //AnimeRes is how the search results from meikan are built up in the JSON type AnimeRes struct { EndDate string `json:"end_date"` Episodes int `json:"episodes"` ID int `json:"id"` Rating string `json:"rating"` StartDate string `json:"start_date"` State string `json:"state"` Title string `json:"title"` Type string `json:"type"` } //Animes is a collection of animes type Animes []MatchedAnime //MatchedAnime contains all the neccessary information after matching two animes type MatchedAnime struct { MeikanTitle string AnidbTitle string MeikanID int AnidbID int Score int } func check(e error) { if e != nil { panic(e) } } func fixAnidbDates(Dates []string) (startDate, endDate time.Time) { if !strings.ContainsAny(Dates[0], "?") && Dates[0] != "" { startDate, _ = time.Parse(`02.01.2006`, Dates[0]) } if len(Dates) == 2 { if !strings.ContainsAny(Dates[1], "?") && Dates[1] != "" { endDate, _ = time.Parse(`02.01.2006`, Dates[1]) } } return startDate, endDate } func parseCsv(file string, c chan MatchedAnime) { dat, err := ioutil.ReadFile(file) check(err) r := csv.NewReader(strings.NewReader(string(dat))) _, err = r.Read() check(err) for { record, err := r.Read() if err == io.EOF { break } EpRegex := regexp.MustCompile(`.*, (\d*) .*`) ep := strings.Split(record[11], ",") eps := 1 episodes, err := strconv.Atoi(EpRegex.ReplaceAllString(record[11], "$1")) if err == nil { eps = episodes } id, err := strconv.Atoi(record[1]) check(err) Date := strings.Split(record[12], " till ") startDate, endDate := fixAnidbDates(Date) a := Anime{ StartDate: startDate, EndDate: endDate, ID: id, Type: strings.TrimSpace(ep[0]), Episodes: eps, Title: strings.Replace(record[3], "Anime: ", "", 1), } if a.Title == "ERROR" || a.Title == "?" { continue } go checkResults(a, c) time.Sleep(10 * time.Millisecond) } fmt.Println("Finished parsing csv") } func lessType(Type string) string { switch Type { case "TV", "ONA": return "Shows" case "OVA", "Specials": return "Extra" } return Type } func fixType(Type string) string { switch Type { case "TV Series": return "TV" case "TV Special": return "Special" case "Web": return "ONA" } return Type } func fixMeikanDates(Date, Date2 string) (date, date2 time.Time) { if Date != "" { date, _ = time.Parse(`2006-01-02`, Date) } if Date2 != "" { date2, _ = time.Parse(`2006-01-02`, Date) } return date, date2 } func subAnimeDates(Date, Date2 time.Time) (total int) { if Date.Year() == 1 && Date2.Year() == 1 { return 0 } diff := Date.Sub(Date2) if diff < 0 { diff = diff * -1 } if diff <= 2190*time.Hour { return 50 } return -150 } func checkResults(anime Anime, c chan MatchedAnime) { fmt.Println(`Searching anime`, anime.ID) var highest, hI int var err error anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1) var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`)) resp, err := http.Post("https://api.meikan.moe/v1/anime?incl=start_date,end_date", "application/json", search) check(err) fmt.Println(`Completed meikan search for`, anime.ID) defer resp.Body.Close() var result Result body, err := ioutil.ReadAll(resp.Body) check(err) _ = json.NewDecoder(bytes.NewReader(body)).Decode(&result) anime.Type = fixType(anime.Type) if anime.Episodes < 1 { anime.Episodes = 1 } for i := 0; i < len(result.Anime); i++ { var total int meikan := result.Anime[i] stDate, enDate := fixMeikanDates(meikan.StartDate, meikan.EndDate) if meikan.Type == anime.Type { total += 25 } else { MType := lessType(meikan.Type) AType := lessType(anime.Type) if MType == AType { total += 12 } } score := levenshtein.Match(meikan.Title, anime.Title, nil) if score > 0.95 { total += 50 } if meikan.Episodes == anime.Episodes { total += 25 } total += subAnimeDates(stDate, anime.StartDate) if anime.Episodes != 1 { total += subAnimeDates(enDate, anime.EndDate) } if total > highest { highest = total hI = i } } //return highest, hI if len(result.Anime) != 0 { c <- MatchedAnime{ MeikanTitle: result.Anime[hI].Title, AnidbTitle: anime.Title, MeikanID: result.Anime[hI].ID, AnidbID: anime.ID, Score: highest, } } } func main() { http.DefaultTransport = &http.Transport{MaxIdleConnsPerHost: 50, MaxIdleConns: 50} c := make(chan MatchedAnime) var animes Animes parseCsv("/home/trac/coding/compareID/1.-Main-data.csv", c) MainLoop: for { select { case m := <-c: animes = append(animes, m) case <-time.After(2 * time.Second): break MainLoop } } resJSON, err := json.MarshalIndent(animes, "", "\t") check(err) jsonfile := []byte(resJSON) err = ioutil.WriteFile("./result.json", jsonfile, 0644) check(err) }