diff --git a/main.go b/main.go index e02d90f..54926d8 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "encoding/csv" "encoding/json" "fmt" - //"gopkg.in/cheggaaa/pb.v1" + "github.com/agext/levenshtein" "io" "io/ioutil" "net/http" @@ -13,7 +13,6 @@ import ( "strconv" "strings" "time" - "github.com/agext/levenshtein" ) //Anime is AniDB structure @@ -25,11 +24,13 @@ type Anime struct { StartDate time.Time EndDate time.Time } + //Result contains the results from the meikan search type Result struct { Total int `json:"total"` Anime []AnimeRes `json:"data"` } + //AnimeRes is how the search results from meikan are built up in the JSON type AnimeRes struct { EndDate string `json:"end_date"` @@ -43,11 +44,12 @@ type AnimeRes struct { } type CompareAni struct { - AnidbTitle string + AnidbTitle string MeikanTitle string - AnidbID int - MeikanID int + AnidbID int + MeikanID int } + //Animes is a collection of animes type Animes []MatchedAnime @@ -55,10 +57,10 @@ type resAnimes []CompareAni type MatchedAnime struct { MeikanTitle string - AnidbTitle string - MeikanID int - AnidbID int - Score int + AnidbTitle string + MeikanID int + AnidbID int + Score int } func check(e error) { @@ -67,7 +69,7 @@ func check(e error) { } } -func parseCsv (file string, c chan MatchedAnime) { +func parseCsv(file string, c chan MatchedAnime) { anidbLayout := "02.01.2006" dat, err := ioutil.ReadFile(file) check(err) @@ -79,7 +81,7 @@ func parseCsv (file string, c chan MatchedAnime) { if err == io.EOF { break - } + } EpRegex := regexp.MustCompile(`.*, (\d*) .*`) ep := strings.Split(record[11], ",") @@ -112,28 +114,27 @@ func parseCsv (file string, c chan MatchedAnime) { Episodes: eps, Title: strings.Replace(record[3], "Anime: ", "", 1), } - if a.Title == "ERROR" { + if a.Title == "ERROR" || a.Title == "?" { continue } go checkResults(a, c) - time.Sleep(5 * time.Millisecond) + time.Sleep(10 * time.Millisecond) } fmt.Println("Finished parsing csv") } -func lessType (Type string) (string) { - switch (Type) { - case "TV", "ONA": - return "Shows" - case "OVA", "Specials": - return "Extra" +func lessType(Type string) string { + switch Type { + case "TV", "ONA": + return "Shows" + case "OVA", "Specials": + return "Extra" } return Type } func checkResults(anime Anime, c chan MatchedAnime) { var highest int - //var re = regexp.MustCompile(`\s\(\d*\)`) var err error anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1) @@ -149,6 +150,7 @@ func checkResults(anime Anime, c chan MatchedAnime) { err = json.NewDecoder(bytes.NewReader(body)).Decode(&result) if err != nil { fmt.Println(string(body)) + fmt.Println(anime.Title) return } switch anime.Type { @@ -167,7 +169,7 @@ func checkResults(anime Anime, c chan MatchedAnime) { anime.Type = "ONA" } var hI int - for i := 0;i 0.95 { total += 50 } if meikan.Episodes == anime.Episodes { - total += 25 + total += 25 } if anime.StartDate.Year() != 1 && stDate.Year() != 1 { diff := stDate.Sub(anime.StartDate) if diff < 0 { diff = diff * -1 } - if diff <= 2190 * time.Hour { + if diff <= 2190*time.Hour { total += 50 } } else { @@ -213,10 +214,9 @@ func checkResults(anime Anime, c chan MatchedAnime) { if diff < 0 { diff = diff * -1 } - if diff <= 2190 * time.Hour { + if diff <= 2190*time.Hour { total += 50 } - //total += 50 } if total > highest { highest = total @@ -227,78 +227,33 @@ func checkResults(anime Anime, c chan MatchedAnime) { if len(result.Anime) != 0 { c <- MatchedAnime{ MeikanTitle: result.Anime[hI].Title, - AnidbTitle: anime.Title, - MeikanID: result.Anime[hI].ID, - AnidbID: anime.ID, - Score: highest, + AnidbTitle: anime.Title, + MeikanID: result.Anime[hI].ID, + AnidbID: anime.ID, + Score: highest, } } } func main() { + http.DefaultTransport = &http.Transport{MaxIdleConnsPerHost: 50, MaxIdleConns: 50} c := make(chan MatchedAnime) - //var resultCompare resAnimes var animes Animes parseCsv("/home/trac/coding/compareID/1.-Main-data.csv", c) - //count := len(animes) - //bar := pb.StartNew(count - 1) - test := 0 +MainLoop: for { - /*anime := animes[i] - highest, hI := <-c, <-c - //highest, hI := checkResults(result, anime, 0) - if highest < 100 { - check(err) - if len(result.Anime) > 0 { - fmt.Printf("Found no match for %s (Best result: %s)\n", anime.Title, result.Anime[hI].Title) - //fmt.Println(highest) - //fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", anime.Title, anime.Episodes, anime.Type, anime.ID, anime.StartDate, anime.EndDate) - //fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate) - } else { - fmt.Printf("Found no match for %s\n", anime.Title) - fmt.Println("No search results returned") - } - notMatched++ - - } else { - check(err) - /*fmt.Println(highest, hI) - fmt.Printf("Matched %s -> %s\n", anime.Title, result.Anime[hI].Title) - //fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate) - a := CompareAni{ - AnidbTitle: anime.Title, - MeikanTitle: result.Anime[hI].Title, - AnidbID: anime.ID, - MeikanID: result.Anime[hI].ID, - } - matched++ - resultCompare = append(resultCompare, a) - } - bar.Increment() - //fmt.Printf("ID: %d, Type: %s, Episodes: %d, Title: %s\n", a.Id, a.Type, a.Episodes, a.Title) - */ select { - case m := <- c: - fmt.Println(m) + case m := <-c: animes = append(animes, m) - test = 0 - default: - time.Sleep(1000 * time.Millisecond) - if (test > 5) { - break - } - test = test + 1 - } - //break - if (test > 5) { - break + case <-time.After(2 * time.Second): + break MainLoop } } fmt.Println(animes) resJson, err := json.MarshalIndent(animes, "", "\t") check(err) jsonfile := []byte(resJson) - err = ioutil.WriteFile("./result.json", jsonfile, 0644) - //fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched) - check(err) + err = ioutil.WriteFile("./result.json", jsonfile, 0644) + //fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched) + check(err) }