test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

finish twitcasting

Browse Source
This commit is contained in:
eson 2020-08-12 12:22:46 +08:00
parent b82b4f5b77
commit 16c3ed93b7
4 changed files with 75 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extractor/twitcasting_extractor/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*.html
log
screenlog.*
twitcasting_extractor

67
extractor/twitcasting_extractor/twitcasting_extractor.go
View File

@ -24,40 +24,39 @@ type LiveData struct {
Follower string `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"Text"`
MaxViews string `exp:"//span[@id='max_viewer_count']" method:"Text"`
LiveTitle string `exp:"//meta[@property='og:title']" method:"AttributeValue,content"`
LiveStart string `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-started-at"`
LiveStart string `exp:"//time[@data-kind='relative']" method:"AttributeValue,datetime"`
LiveDuration string `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"`
Tags []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag tag-info']"`
Tags []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag tag-info']" method:"Text"`
}
func main() {
ps := intimate.NewPerfectShutdown()
ses := requests.NewSession()
for !ps.IsClose() {
streamer, err := estore.Pop(intimate.Ptwitcasting)
if err != nil {
log.Println(err)
log.Println(err, streamer.UserId)
}
ses := requests.NewSession()
resp, err := ses.Get("https://twitcasting.tv/kyunenee09").Execute()
streamer.LiveUrl = sql.NullString{String: "https://twitcasting.tv/" + streamer.UserId, Valid: true}
resp, err := ses.Get(streamer.LiveUrl.String).Execute()
if err != nil {
log.Panic(err)
estore.UpdateError(streamer, err)
log.Println(err, streamer.UserId)
continue
}
var ldata *LiveData
etor := extractor.ExtractXml(resp.Content())
ldata = etor.GetObjectByTag(LiveData{}).(*LiveData)
ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews)
ldata.LiveStart = ldata.LiveStart[:len(ldata.LiveStart)-3]
ldata.LiveDuration = ldata.LiveDuration[:len(ldata.LiveDuration)-3]
// log.Println(etor.GetObjectByTag(LiveData{}))
coincount := 0
for i := 0; ; i++ {
giverurl := "https://twitcasting.tv/kyunenee09/backers/" + strconv.Itoa(i)
giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i)
resp, err = ses.Get(giverurl).Execute()
if err != nil {
log.Panic(err)
@ -69,7 +68,6 @@ func main() {
}
coins := xp.GetTexts()
for _, cointxt := range coins {
scointxt := strings.Split(cointxt, "/")
if len(scointxt) == 2 {
@ -78,7 +76,7 @@ func main() {
if err == nil {
coincount += c
}
log.Println(coin, coincount)
// log.Println(coin, coincount)
} else {
log.Println("coin error: ", cointxt)
}
@ -91,19 +89,28 @@ func main() {
streamer.Platform = intimate.Ptwitcasting
streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
streamer.UpdateInterval = 60
streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true}
streamer.Operator = 0
streamer.Operator = 10
// streamer.UpdateInterval = 60
clog := &intimate.CollectLog{}
clog.UserId = streamer.UserId
clog.Gratuity = sql.NullInt64{Int64: int64(coincount), Valid: true}
clog.Platform = streamer.Platform
clog.UpdateTime = streamer.UpdateTime
clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true}
fl, err := strconv.Atoi(ldata.Follower)
fl, err := intimate.ParseNumberEx(ldata.Follower)
if err == nil {
clog.Followers = sql.NullInt64{Int64: int64(fl), Valid: true}
switch {
case fl <= 100:
streamer.UpdateInterval = 360
case fl <= 1000:
streamer.UpdateInterval = 240
case fl <= 100:
streamer.UpdateInterval = 120
default:
streamer.UpdateInterval = 60
}
} else {
log.Println(err)
}
@ -112,29 +119,31 @@ func main() {
if err == nil {
clog.Views = sql.NullInt64{Int64: int64(views), Valid: true}
} else {
log.Println(err)
clog.Views = sql.NullInt64{Int64: int64(0), Valid: true}
// log.Println(err, streamer.UserId)
}
st, err := strconv.Atoi(ldata.LiveStart)
// st, err := strconv.Atoi(ldata.LiveStart)
st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart)
if err == nil {
startTime := time.Unix(int64(st), 0)
startTime := st
clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true}
dt, err := strconv.Atoi(ldata.LiveDuration)
if err == nil {
endTime := startTime.Add((time.Duration)(dt) * time.Second)
clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true}
} else {
log.Println(err)
if time.Now().Sub(startTime) >= time.Hour*24*90 {
streamer.Operator = 5
}
if err == nil {
endTime := startTime.Add((time.Duration)(dt) * time.Millisecond)
clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true}
} else {
log.Println(err, streamer.UserId)
}
} else {
log.Println(err)
log.Println(err, streamer.UserId)
}
streamer.LatestLogUid = estore.InsertClog(clog)
estore.UpdateStreamer(streamer)
break
}
}

13
extractor/twitcasting_extractor/twitcasting_extractor_test.go
View File

@ -1,6 +1,9 @@
package main
import "testing"
import (
"testing"
"time"
)
// type LiveData struct {
// UserName string `exp:".//span[@class='tw-live-author__info-username']" method:"Text"`
@ -10,3 +13,11 @@ import "testing"
func TestMain(t *testing.T) {
main()
}
func TestDateFormat(t *testing.T) {
df := "Sat, 09 Sep 2017 18:19:17 +0900"
if _, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", df); err != nil {
t.Error(err)
}
}

21
utils.go
View File

@ -34,6 +34,27 @@ func ParseNumber(number string) (int64, error) {
return strconv.ParseInt(number, 10, 64)
}
// ParseNumberEx 解析带字符的数字
func ParseNumberEx(num string) (float64, error) {
num = strings.Trim(num, " ")
last := num[len(num)-1]
factor := 1.0
switch {
case last == 'k' || last == 'K':
factor = 1000.0
num = num[0 : len(num)-1]
case last == 'm' || last == 'M':
factor = 1000000.0
num = num[0 : len(num)-1]
}
i, err := strconv.ParseFloat(num, 64)
if err != nil {
return 0, err
}
return i * factor, nil
}
// ParseDuration time to duration eg: 1:40:00 -> time.Duration
func ParseDuration(dt string) (time.Duration, error) {