diff --git a/extractor/twitcasting_extractor/.gitignore b/extractor/twitcasting_extractor/.gitignore new file mode 100644 index 0000000..dea3a6c --- /dev/null +++ b/extractor/twitcasting_extractor/.gitignore @@ -0,0 +1,4 @@ +*.html +log +screenlog.* +twitcasting_extractor \ No newline at end of file diff --git a/extractor/twitcasting_extractor/twitcasting_extractor.go b/extractor/twitcasting_extractor/twitcasting_extractor.go index 44e059f..0550e6a 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor.go @@ -24,40 +24,39 @@ type LiveData struct { Follower string `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"Text"` MaxViews string `exp:"//span[@id='max_viewer_count']" method:"Text"` LiveTitle string `exp:"//meta[@property='og:title']" method:"AttributeValue,content"` - LiveStart string `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-started-at"` + LiveStart string `exp:"//time[@data-kind='relative']" method:"AttributeValue,datetime"` LiveDuration string `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"` - Tags []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag tag-info']"` + Tags []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag tag-info']" method:"Text"` } func main() { ps := intimate.NewPerfectShutdown() + ses := requests.NewSession() for !ps.IsClose() { streamer, err := estore.Pop(intimate.Ptwitcasting) if err != nil { - log.Println(err) + log.Println(err, streamer.UserId) } - ses := requests.NewSession() - resp, err := ses.Get("https://twitcasting.tv/kyunenee09").Execute() + streamer.LiveUrl = sql.NullString{String: "https://twitcasting.tv/" + streamer.UserId, Valid: true} + resp, err := ses.Get(streamer.LiveUrl.String).Execute() if err != nil { - log.Panic(err) + estore.UpdateError(streamer, err) + log.Println(err, streamer.UserId) + continue } var ldata *LiveData etor := extractor.ExtractXml(resp.Content()) ldata = etor.GetObjectByTag(LiveData{}).(*LiveData) - ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews) - ldata.LiveStart = ldata.LiveStart[:len(ldata.LiveStart)-3] - ldata.LiveDuration = ldata.LiveDuration[:len(ldata.LiveDuration)-3] - // log.Println(etor.GetObjectByTag(LiveData{})) coincount := 0 for i := 0; ; i++ { - giverurl := "https://twitcasting.tv/kyunenee09/backers/" + strconv.Itoa(i) + giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i) resp, err = ses.Get(giverurl).Execute() if err != nil { log.Panic(err) @@ -69,7 +68,6 @@ func main() { } coins := xp.GetTexts() - for _, cointxt := range coins { scointxt := strings.Split(cointxt, "/") if len(scointxt) == 2 { @@ -78,7 +76,7 @@ func main() { if err == nil { coincount += c } - log.Println(coin, coincount) + // log.Println(coin, coincount) } else { log.Println("coin error: ", cointxt) } @@ -91,19 +89,28 @@ func main() { streamer.Platform = intimate.Ptwitcasting streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true} - streamer.UpdateInterval = 60 streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true} - streamer.Operator = 0 - + streamer.Operator = 10 + // streamer.UpdateInterval = 60 clog := &intimate.CollectLog{} clog.UserId = streamer.UserId clog.Gratuity = sql.NullInt64{Int64: int64(coincount), Valid: true} clog.Platform = streamer.Platform clog.UpdateTime = streamer.UpdateTime clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true} - fl, err := strconv.Atoi(ldata.Follower) + fl, err := intimate.ParseNumberEx(ldata.Follower) if err == nil { clog.Followers = sql.NullInt64{Int64: int64(fl), Valid: true} + switch { + case fl <= 100: + streamer.UpdateInterval = 360 + case fl <= 1000: + streamer.UpdateInterval = 240 + case fl <= 100: + streamer.UpdateInterval = 120 + default: + streamer.UpdateInterval = 60 + } } else { log.Println(err) } @@ -112,29 +119,31 @@ func main() { if err == nil { clog.Views = sql.NullInt64{Int64: int64(views), Valid: true} } else { - log.Println(err) + clog.Views = sql.NullInt64{Int64: int64(0), Valid: true} + // log.Println(err, streamer.UserId) } - st, err := strconv.Atoi(ldata.LiveStart) + // st, err := strconv.Atoi(ldata.LiveStart) + st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart) if err == nil { - startTime := time.Unix(int64(st), 0) + startTime := st clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true} dt, err := strconv.Atoi(ldata.LiveDuration) - if err == nil { - - endTime := startTime.Add((time.Duration)(dt) * time.Second) - clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true} - } else { - log.Println(err) + if time.Now().Sub(startTime) >= time.Hour*24*90 { + streamer.Operator = 5 } + if err == nil { + endTime := startTime.Add((time.Duration)(dt) * time.Millisecond) + clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true} + } else { + log.Println(err, streamer.UserId) + } } else { - log.Println(err) + log.Println(err, streamer.UserId) } streamer.LatestLogUid = estore.InsertClog(clog) estore.UpdateStreamer(streamer) - - break } } diff --git a/extractor/twitcasting_extractor/twitcasting_extractor_test.go b/extractor/twitcasting_extractor/twitcasting_extractor_test.go index 2d4df4f..811b2d3 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor_test.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor_test.go @@ -1,6 +1,9 @@ package main -import "testing" +import ( + "testing" + "time" +) // type LiveData struct { // UserName string `exp:".//span[@class='tw-live-author__info-username']" method:"Text"` @@ -10,3 +13,11 @@ import "testing" func TestMain(t *testing.T) { main() } + +func TestDateFormat(t *testing.T) { + df := "Sat, 09 Sep 2017 18:19:17 +0900" + + if _, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", df); err != nil { + t.Error(err) + } +} diff --git a/utils.go b/utils.go index bad4a08..cb7d782 100644 --- a/utils.go +++ b/utils.go @@ -34,6 +34,27 @@ func ParseNumber(number string) (int64, error) { return strconv.ParseInt(number, 10, 64) } +// ParseNumberEx 解析带字符的数字 +func ParseNumberEx(num string) (float64, error) { + num = strings.Trim(num, " ") + last := num[len(num)-1] + factor := 1.0 + switch { + case last == 'k' || last == 'K': + factor = 1000.0 + num = num[0 : len(num)-1] + case last == 'm' || last == 'M': + factor = 1000000.0 + num = num[0 : len(num)-1] + } + i, err := strconv.ParseFloat(num, 64) + if err != nil { + return 0, err + } + + return i * factor, nil +} + // ParseDuration time to duration eg: 1:40:00 -> time.Duration func ParseDuration(dt string) (time.Duration, error) {