Added concurrency

main.go

@@ -5,11 +5,10 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
-	"gopkg.in/cheggaaa/pb.v1"
+	//"gopkg.in/cheggaaa/pb.v1"
 	"io"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"regexp"
 	"strconv"
 	"strings"
@@ -50,20 +49,27 @@ type CompareAni struct {
 	MeikanID int
 }
 //Animes is a collection of animes
-type animes []Anime
+type Animes []MatchedAnime
 
 type resAnimes []CompareAni
 
+type MatchedAnime struct {
+	MeikanTitle string
+	AnidbTitle string
+	MeikanID int
+	AnidbID int
+	Score int
+}
+
 func check(e error) {
 	if e != nil {
 		panic(e)
 	}
 }
 
-func parseCsv (file string) animes {
+func parseCsv (file string, c chan MatchedAnime) {
 	anidbLayout := "02.01.2006"
 	dat, err := ioutil.ReadFile(file)
-	var ani animes
 	check(err)
 	r := csv.NewReader(strings.NewReader(string(dat)))
 	_, er := r.Read()
@@ -72,6 +78,7 @@ func parseCsv (file string) animes {
 		record, err := r.Read()
 		if err == io.EOF {
 			break
+
 		}	
 		EpRegex := regexp.MustCompile(`.*, (\d*) .*`)
 
@@ -108,9 +115,9 @@ func parseCsv (file string) animes {
 		if a.Title == "ERROR" {
 			continue
 		}
-		ani = append(ani, a)
+		go checkResults(a, c)
+		time.Sleep(10 * time.Millisecond)
 	}
-	return ani
 }
 
 func lessType (Type string) (string) {
@@ -123,13 +130,42 @@ func lessType (Type string) (string) {
 	return Type
 }
 
-func checkResults(r Result, anime Anime, highest int, c chan int) {
+func checkResults(anime Anime, c chan MatchedAnime) {
+	var highest int
 	//var re = regexp.MustCompile(`\s\(\d*\)`)
-	var hI int
-	for i := 0;i<len(r.Anime); i++ {
-		var total int
-		meikan := r.Anime[i]
 	var err error
+	anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
+
+	var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`))
+	resp, err := http.Post("https://api.meikan.moe/v1/anime?incl=start_date,end_date", "application/json", search)
+	check(err)
+	defer resp.Body.Close()
+	var result Result
+	body, err := ioutil.ReadAll(resp.Body)
+	check(err)
+	er := json.NewDecoder(bytes.NewReader(body)).Decode(&result)
+	if er != nil {
+		fmt.Println(string(body))
+	}
+	switch anime.Type {
+	case "TV Series":
+		anime.Type = "TV"
+	case "TV Special":
+		anime.Type = "Special"
+		if anime.Episodes < 1 {
+			anime.Episodes = 1
+		}
+	case "OVA":
+		if anime.Episodes < 1 {
+			anime.Episodes = 1
+		}
+	case "Web":
+		anime.Type = "ONA"
+	}
+	var hI int
+	for i := 0;i<len(result.Anime); i++ {
+		var total int
+		meikan := result.Anime[i]
 		var stDate time.Time
 		if meikan.StartDate != "" {
 			stDate, err = time.Parse("2006-01-02", meikan.StartDate)
@@ -184,58 +220,30 @@ func checkResults(r Result, anime Anime, highest int, c chan int) {
 		}
 	}
 	//return highest, hI
-	c <- highest
+	if len(result.Anime) != 0 {
-	c <- hI
+		c <- MatchedAnime{
+			MeikanTitle: result.Anime[hI].Title,
+			AnidbTitle: anime.Title,
+			MeikanID: result.Anime[hI].ID,
+			AnidbID: anime.ID,
+			Score: highest,
+		}
+	}
 }
 
 func main() {
-	var resultCompare resAnimes
+	c := make(chan MatchedAnime)
-	f, err := os.Create("./data")
+	//var resultCompare resAnimes
-	check(err)
+	var animes Animes
-	defer f.Close()
+	parseCsv("/home/trac/coding/compareID/1.-Main-data.csv", c)
-	check (err)
+	//count := len(animes)
-	animes := parseCsv("/home/trac/coding/compareID/1.-Main-data.csv")
+	//bar := pb.StartNew(count - 1)
-	count := len(animes)
+	test := 0
-	bar := pb.StartNew(count - 1)
+	for {
-	var matched, notMatched int
+		/*anime := animes[i]
-	for i := 1; i < len(animes); i++ {
-		anime := animes[i]
-		anime.Title = strings.Replace(strings.Replace(anime.Title, `\`, `\\`, -1), `"`, `\"`, -1)
-
-		var search = bytes.NewBuffer([]byte(`{"title":"` + anime.Title + `", "show_r18": true}`))
-		resp, err := http.Post("https://api.meikan.moe/v1/anime?incl=start_date,end_date", "application/json", search)
-		check(err)
-		defer resp.Body.Close()
-		check (err)
-		var result Result
-		body, err := ioutil.ReadAll(resp.Body)
-		check(err)
-		if string(body) == "null" {
-			continue
-		}
-		er 	:= json.NewDecoder(bytes.NewReader(body)).Decode(&result)
-		check (er)
-		switch anime.Type {
-		case "TV Series":
-			anime.Type = "TV"
-		case "TV Special":
-			anime.Type = "Special"
-			if anime.Episodes < 1 {
-				anime.Episodes = 1
-			}
-		case "OVA":
-			if anime.Episodes < 1 {
-				anime.Episodes = 1
-			}
-		case "Web":
-			anime.Type = "ONA"
-		}
-		c := make(chan int)
-		go checkResults(result, anime, 0, c)
 		highest, hI := <-c, <-c
 		//highest, hI := checkResults(result, anime, 0)
 		if highest < 100 {
-			_, err := f.WriteString(strconv.Itoa(anime.ID) + " null\n")
 			check(err)
 			if len(result.Anime) > 0 {
 				fmt.Printf("Found no match for %s (Best result: %s)\n", anime.Title, result.Anime[hI].Title)
@@ -249,10 +257,9 @@ func main() {
 			notMatched++
 
 		} else {
-			_, err := f.WriteString(strconv.Itoa(anime.ID) + " " + strconv.Itoa(result.Anime[hI].ID) + "\n")
 			check(err)
 			/*fmt.Println(highest, hI)
-			fmt.Printf("Matched %s -> %s\n", anime.Title, result.Anime[hI].Title)*/
+			fmt.Printf("Matched %s -> %s\n", anime.Title, result.Anime[hI].Title)
 			//fmt.Printf("Title: %s, Episodes: %d, Type: %s, ID: %d, StartDate: %s, EndDate: %s\n", result.Anime[hI].Title, result.Anime[hI].Episodes, result.Anime[hI].Type, result.Anime[hI].ID, result.Anime[hI].StartDate, result.Anime[hI].EndDate)
 			a := CompareAni{
 				AnidbTitle: anime.Title,
@@ -265,11 +272,23 @@ func main() {
 		}
 		bar.Increment()
 		//fmt.Printf("ID: %d, Type: %s, Episodes: %d, Title: %s\n", a.Id, a.Type, a.Episodes, a.Title)
+		*/
+		select {
+		case m := <- c:
+			animes = append(animes, m)
+			test = 0
+		default:
+			time.Sleep(1000 * time.Millisecond)
+			if (test > 2) {
+				return
 			}
-	resJson, err := json.MarshalIndent(resultCompare, "", "\t")
+			test = test + 1
+		}
+	}
+	resJson, err := json.MarshalIndent(animes, "", "\t")
 	check(err)
 	jsonfile := []byte(resJson)
     err = ioutil.WriteFile("./result.json", jsonfile, 0644)
-    fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched)
+    //fmt.Printf("Matched: %d\nNot Matched: %d\nTotal: %d", matched, notMatched, matched + notMatched)
     check(err)
 }

qc.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"bytes"
+	"bufio"
+	"os"
+	"strings"
+)
+
+type CompareAni struct {
+	AnidbTitle string `json:"AnidbTitle"`
+	MeikanTitle string `json:"MeikanTitle"`
+	AnidbID int  `json:"AnidbID"`
+	MeikanID int `json:"MeikanID"`
+	QCMatched bool
+}
+
+type resAnimes []CompareAni
+
+func check(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
+func main() {
+	var animes resAnimes
+	var err error
+	dat, err := ioutil.ReadFile("./result.json")
+	check(err)
+	err = json.NewDecoder(bytes.NewReader(dat)).Decode(&animes)
+	check(err)
+	for i := 0; i < len(animes); i++ {
+		anime := animes[i]
+		reader := bufio.NewReader(os.Stdin)
+		fmt.Printf("%s (Anidb %d)\n%s (Meikan %d)\nAre they the same? (Y/N)", anime.AnidbTitle, anime.AnidbID, anime.MeikanTitle, anime.MeikanID)
+		text, _ := reader.ReadString('\n')
+		text = strings.ToLower(text)
+		if (text == "y") {
+			anime.QCMatched = true
+		} else {
+			anime.QCMatched = false
+		}
+	}
+	fmt.Println(animes[0])
+}

test.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"fmt"
+	"github.com/agext/levenshtein"
+)
+
+func main() {
+	score := levenshtein.Match("Vampire Hunter D (2000)", "Vampire Hunter D (2001)", nil)
+	fmt.Print(score)
+}