Tracreed
/
compareID
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ran gofmt and cleaned up code

This commit is contained in:
David Alasow 2018-01-20 11:35:36 +01:00
parent 292f6cc4dc
commit c7176ea47e
1 changed files with 38 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
main.go
View File

@ -5,7 +5,7 @@ import (
"encoding/csv" "encoding/csv"
"encoding/json" "encoding/json"
"fmt" "fmt"
//"gopkg.in/cheggaaa/pb.v1" "github.com/agext/levenshtein"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -13,7 +13,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/agext/levenshtein"
) )
//Anime is AniDB structure //Anime is AniDB structure
@ -25,11 +24,13 @@ type Anime struct {
StartDate time.Time StartDate time.Time
EndDate time.Time EndDate time.Time
} }
//Result contains the results from the meikan search //Result contains the results from the meikan search
type Result struct { type Result struct {
Total int `json:"total"` Total int `json:"total"`
Anime []AnimeRes `json:"data"` Anime []AnimeRes `json:"data"`
} }
//AnimeRes is how the search results from meikan are built up in the JSON //AnimeRes is how the search results from meikan are built up in the JSON
type AnimeRes struct { type AnimeRes struct {
EndDate string `json:"end_date"` EndDate string `json:"end_date"`
@ -43,11 +44,12 @@ type AnimeRes struct {
} }
type CompareAni struct { type CompareAni struct {
AnidbTitle string AnidbTitle string
MeikanTitle string MeikanTitle string
AnidbID int AnidbID int
MeikanID int MeikanID int
} }
//Animes is a collection of animes //Animes is a collection of animes
type Animes []MatchedAnime type Animes []MatchedAnime
@ -55,10 +57,10 @@ type resAnimes []CompareAni
type MatchedAnime struct { type MatchedAnime struct {
MeikanTitle string MeikanTitle string
AnidbTitle string AnidbTitle string
MeikanID int MeikanID int
AnidbID int AnidbID int
Score int Score int
} }
func check(e error) { func check(e error) {
@ -67,7 +69,7 @@ func check(e error) {
} }
} }
func parseCsv (file string, c chan MatchedAnime) { func parseCsv(file string, c chan MatchedAnime) {
anidbLayout := "02.01.2006" anidbLayout := "02.01.2006"
dat, err := ioutil.ReadFile(file) dat, err := ioutil.ReadFile(file)
check(err) check(err)
@ -79,7 +81,7 @@ func parseCsv (file string, c chan MatchedAnime) {
if err == io.EOF { if err == io.EOF {
break break
} }
EpRegex := regexp.MustCompile(`.*, (\d*) .*`) EpRegex := regexp.MustCompile(`.*, (\d*) .*`)
ep := strings.Split(record[11], ",") ep := strings.Split(record[11], ",")
@ -112,28 +114,27 @@ func parseCsv (file string, c chan MatchedAnime) {
Episodes: eps, Episodes: eps,
Title: strings.Replace(record[3], "Anime: ", "", 1), Title: strings.Replace(record[3], "Anime: ", "", 1),
} }
if a.Title == "ERROR" { if a.Title == "ERROR" || a.Title == "?" {
continue continue
} }
go checkResults(a, c) go checkResults(a, c)
time.Sleep(5 * time.Millisecond) time.Sleep(10 * time.Millisecond)
} }
fmt.Println("Finished parsing csv") fmt.Println("Finished parsing csv")
} }
func lessType (Type string) (string) { func lessType(Type string) string {
switch (Type) { switch Type {
case "TV", "ONA": case "TV", "ONA":
return "Shows" return "Shows"
case "OVA", "Specials": case "OVA", "Specials":
return "Extra" return "Extra"
} }
return Type return Type
} }
func checkResults(anime Anime, c chan MatchedAnime) { func checkResults(anime Anime, c chan MatchedAnime) {
var highest int var highest int
//var re = regexp.MustCompile(`\s\(\d*\)`)
var err error var err error
anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1) anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
@ -149,6 +150,7 @@ func checkResults(anime Anime, c chan MatchedAnime) {
err = json.NewDecoder(bytes.NewReader(body)).Decode(&result) err = json.NewDecoder(bytes.NewReader(body)).Decode(&result)
if err != nil { if err != nil {
fmt.Println(string(body)) fmt.Println(string(body))
fmt.Println(anime.Title)
return return
} }
switch anime.Type { switch anime.Type {
@ -167,7 +169,7 @@ func checkResults(anime Anime, c chan MatchedAnime) {
anime.Type = "ONA" anime.Type = "ONA"
} }
var hI int var hI int
for i := 0;i<len(result.Anime); i++ { for i := 0; i < len(result.Anime); i++ {
var total int var total int
meikan := result.Anime[i] meikan := result.Anime[i]
var stDate time.Time var stDate time.Time
@ -190,19 +192,18 @@ func checkResults(anime Anime, c chan MatchedAnime) {
} }
} }
score := levenshtein.Match(meikan.Title, anime.Title, nil) score := levenshtein.Match(meikan.Title, anime.Title, nil)
//fmt.Print(score)
if score > 0.95 { if score > 0.95 {
total += 50 total += 50
} }
if meikan.Episodes == anime.Episodes { if meikan.Episodes == anime.Episodes {
total += 25 total += 25
} }
if anime.StartDate.Year() != 1 && stDate.Year() != 1 { if anime.StartDate.Year() != 1 && stDate.Year() != 1 {
diff := stDate.Sub(anime.StartDate) diff := stDate.Sub(anime.StartDate)
if diff < 0 { if diff < 0 {
diff = diff * -1 diff = diff * -1
} }
if diff <= 2190 * time.Hour { if diff <= 2190*time.Hour {
total += 50 total += 50
} }
} else { } else {
@ -213,10 +214,9 @@ func checkResults(anime Anime, c chan MatchedAnime) {
if diff < 0 { if diff < 0 {
diff = diff * -1 diff = diff * -1
} }
if diff <= 2190 * time.Hour { if diff <= 2190*time.Hour {
total += 50 total += 50
} }
//total += 50
} }
if total > highest { if total > highest {
highest = total highest = total
@ -227,78 +227,33 @@ func checkResults(anime Anime, c chan MatchedAnime) {
if len(result.Anime) != 0 { if len(result.Anime) != 0 {
c <- MatchedAnime{ c <- MatchedAnime{
MeikanTitle: result.Anime[hI].Title, MeikanTitle: result.Anime[hI].Title,
AnidbTitle: anime.Title, AnidbTitle: anime.Title,
MeikanID: result.Anime[hI].ID, MeikanID: result.Anime[hI].ID,
AnidbID: anime.ID, AnidbID: anime.ID,
Score: highest, Score: highest,
} }
} }
} }
func main() { func main() {
http.DefaultTransport = &http.Transport{MaxIdleConnsPerHost: 50, MaxIdleConns: 50}
c := make(chan MatchedAnime) c := make(chan MatchedAnime)
//var resultCompare resAnimes
var animes Animes var animes Animes
parseCsv("/home/trac/coding/compareID/1.-Main-data.csv", c) parseCsv("/home/trac/coding/compareID/1.-Main-data.csv", c)
//count := len(animes) MainLoop:
//bar := pb.StartNew(count - 1)
test := 0
for { for {
/*anime := animes[i]
highest, hI := <-c, <-c
//highest, hI := checkResults(result, anime, 0)
if highest < 100 {
check(err)
if len(result.Anime) > 0 {
fmt.Printf("Found no match for %s (Best result: %s)\n", anime.Title, result.Anime[hI].Title)
//fmt.Println(highest)
//fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", anime.Title, anime.Episodes, anime.Type, anime.ID, anime.StartDate, anime.EndDate)
//fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate)
} else {
fmt.Printf("Found no match for %s\n", anime.Title)
fmt.Println("No search results returned")
}
notMatched++
} else {
check(err)
/*fmt.Println(highest, hI)
fmt.Printf("Matched %s -> %s\n", anime.Title, result.Anime[hI].Title)
//fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate)
a := CompareAni{
AnidbTitle: anime.Title,
MeikanTitle: result.Anime[hI].Title,
AnidbID: anime.ID,
MeikanID: result.Anime[hI].ID,
}
matched++
resultCompare = append(resultCompare, a)
}
bar.Increment()
//fmt.Printf("ID: %d, Type: %s, Episodes: %d, Title: %s\n", a.Id, a.Type, a.Episodes, a.Title)
*/
select { select {
case m := <- c: case m := <-c:
fmt.Println(m)
animes = append(animes, m) animes = append(animes, m)
test = 0 case <-time.After(2 * time.Second):
default: break MainLoop
time.Sleep(1000 * time.Millisecond)
if (test > 5) {
break
}
test = test + 1
}
//break
if (test > 5) {
break
} }
} }
fmt.Println(animes) fmt.Println(animes)
resJson, err := json.MarshalIndent(animes, "", "\t") resJson, err := json.MarshalIndent(animes, "", "\t")
check(err) check(err)
jsonfile := []byte(resJson) jsonfile := []byte(resJson)
err = ioutil.WriteFile("./result.json", jsonfile, 0644) err = ioutil.WriteFile("./result.json", jsonfile, 0644)
//fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched) //fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched)
check(err) check(err)
} }