package main import ( "bytes" "encoding/csv" "encoding/json" "fmt" "gopkg.in/cheggaaa/pb.v1" "io" "io/ioutil" "net/http" "os" "regexp" "strconv" "strings" "time" "github.com/agext/levenshtein" ) //Anime is AniDB structure type Anime struct { ID int Type string Episodes int Title string StartDate time.Time EndDate time.Time } //Result contains the results from the meikan search type Result struct { Total int `json:"total"` Anime []AnimeRes `json:"data"` } //AnimeRes is how the search results from meikan are built up in the JSON type AnimeRes struct { EndDate string `json:"end_date"` Episodes int `json:"episodes"` ID int `json:"id"` Rating string `json:"rating"` StartDate string `json:"start_date"` State string `json:"state"` Title string `json:"title"` Type string `json:"type"` } type CompareAni struct { AnidbTitle string MeikanTitle string AnidbID int MeikanID int } //Animes is a collection of animes type animes []Anime type resAnimes []CompareAni func check(e error) { if e != nil { panic(e) } } func parseCsv (file string) animes { anidbLayout := "02.01.2006" dat, err := ioutil.ReadFile(file) var ani animes check(err) r := csv.NewReader(strings.NewReader(string(dat))) _, er := r.Read() check(er) for { record, err := r.Read() if err == io.EOF { break } EpRegex := regexp.MustCompile(`.*, (\d*) .*`) ep := strings.Split(record[11], ",") eps := 1 episodes, err := strconv.Atoi(EpRegex.ReplaceAllString(record[11], "$1")) if err == nil { eps = episodes } id, err := strconv.Atoi(record[1]) check(err) Date := strings.Split(record[12], " till ") var startDate time.Time if !strings.ContainsAny(Date[0], "?") && Date[0] != "" { startDate, err = time.Parse(anidbLayout, Date[0]) check(err) } var endDate time.Time if len(Date) == 2 { if !strings.ContainsAny(Date[1], "?") && Date[1] != "" { endDate, err = time.Parse(anidbLayout, Date[1]) check(err) } } a := Anime{ StartDate: startDate, EndDate: endDate, ID: id, Type: strings.TrimSpace(ep[0]), Episodes: eps, Title: strings.Replace(record[3], "Anime: ", "", 1), } if a.Title == "ERROR" { continue } ani = append(ani, a) } return ani } func lessType (Type string) (string) { switch (Type) { case "TV", "ONA": return "Shows" case "OVA", "Specials": return "Extra" } return Type } func checkResults(r Result, anime Anime, highest int, c chan int) { //var re = regexp.MustCompile(`\s\(\d*\)`) var hI int for i := 0;i 0.95 { total += 50 } if meikan.Episodes == anime.Episodes { total += 25 } if anime.StartDate.Year() != 1 && stDate.Year() != 1 { diff := stDate.Sub(anime.StartDate) if diff < 0 { diff = diff * -1 } if diff <= 2190 * time.Hour { total += 50 } } else { total = -150 } if anime.Episodes != 1 && anime.EndDate.Year() != 1 && enDate.Year() != 1 { diff := enDate.Sub(anime.EndDate) if diff < 0 { diff = diff * -1 } if diff <= 2190 * time.Hour { total += 50 } //total += 50 } if total > highest { highest = total hI = i } } //return highest, hI c <- highest c <- hI } func main() { var resultCompare resAnimes f, err := os.Create("./data") check(err) defer f.Close() check (err) animes := parseCsv("/home/trac/coding/compareID/1.-Main-data.csv") count := len(animes) bar := pb.StartNew(count - 1) var matched, notMatched int for i := 1; i < len(animes); i++ { anime := animes[i] anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1) var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`)) resp, err := http.Post("https://api.meikan.moe/v1/anime?incl=start_date,end_date", "application/json", search) check(err) defer resp.Body.Close() check (err) var result Result body, err := ioutil.ReadAll(resp.Body) check(err) if string(body) == "null" { continue } er := json.NewDecoder(bytes.NewReader(body)).Decode(&result) check (er) switch anime.Type { case "TV Series": anime.Type = "TV" case "TV Special": anime.Type = "Special" if anime.Episodes < 1 { anime.Episodes = 1 } case "OVA": if anime.Episodes < 1 { anime.Episodes = 1 } case "Web": anime.Type = "ONA" } c := make(chan int) go checkResults(result, anime, 0, c) highest, hI := <-c, <-c //highest, hI := checkResults(result, anime, 0) if highest < 100 { _, err := f.WriteString(strconv.Itoa(anime.ID) + " null\n") check(err) if len(result.Anime) > 0 { fmt.Printf("Found no match for %s (Best result: %s)\n", anime.Title, result.Anime[hI].Title) //fmt.Println(highest) //fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", anime.Title, anime.Episodes, anime.Type, anime.ID, anime.StartDate, anime.EndDate) //fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate) } else { fmt.Printf("Found no match for %s\n", anime.Title) fmt.Println("No search results returned") } notMatched++ } else { _, err := f.WriteString(strconv.Itoa(anime.ID) + " " + strconv.Itoa(result.Anime[hI].ID) + "\n") check(err) /*fmt.Println(highest, hI) fmt.Printf("Matched %s -> %s\n", anime.Title, result.Anime[hI].Title)*/ //fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate) a := CompareAni{ AnidbTitle: anime.Title, MeikanTitle: result.Anime[hI].Title, AnidbID: anime.ID, MeikanID: result.Anime[hI].ID, } matched++ resultCompare = append(resultCompare, a) } bar.Increment() //fmt.Printf("ID: %d, Type: %s, Episodes: %d, Title: %s\n", a.Id, a.Type, a.Episodes, a.Title) } resJson, err := json.MarshalIndent(resultCompare, "", "\t") check(err) jsonfile := []byte(resJson) err = ioutil.WriteFile("./result.json", jsonfile, 0644) fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched) check(err) }