From eee4e305858ba105c9dffaba55cf6bc6b1a98dc7 Mon Sep 17 00:00:00 2001 From: eson Date: Tue, 18 Aug 2020 16:50:49 +0800 Subject: [PATCH] fix: twistcasting tags error. change: twistcasting, some filed type to number --- .../twitcasting_extractor.go | 95 +++++++++++-------- .../twitcasting_extractor_test.go | 2 +- go.mod | 2 +- go.sum | 10 +- 4 files changed, 60 insertions(+), 49 deletions(-) diff --git a/extractor/twitcasting_extractor/twitcasting_extractor.go b/extractor/twitcasting_extractor/twitcasting_extractor.go index 9f0b031..13f419e 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor.go @@ -2,9 +2,10 @@ package main import ( "database/sql" + "encoding/json" "intimate" "log" - "regexp" + "os" "strconv" "strings" "time" @@ -21,8 +22,8 @@ var estore *intimate.StoreExtractor = intimate.NewStoreExtractor() type LiveData struct { UserName string `exp:"//span[@class='tw-live-author__info-username']" method:"Text"` - Follower string `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"Text"` - MaxViews string `exp:"//span[@id='max_viewer_count']" method:"Text"` + Follower int64 `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"r:ExtractNumber"` + MaxViews int64 `exp:"//span[@id='max_viewer_count']/text()" method:"r:ExtractNumber"` LiveTitle string `exp:"//meta[@property='og:title']" method:"AttributeValue,content"` LiveStart string `exp:"//time[@data-kind='relative']" method:"AttributeValue,datetime"` LiveDuration string `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"` @@ -49,9 +50,11 @@ func main() { continue } var ldata *LiveData + f, _ := os.OpenFile("./twistcasting.html", os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm) + f.Write(resp.Content()) etor := extractor.ExtractHtml(resp.Content()) ldata = etor.GetObjectByTag(LiveData{}).(*LiveData) - ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews) + // ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews) coincount := 0 for i := 0; ; i++ { @@ -90,10 +93,17 @@ func main() { } } + var tags []byte + tags, err = json.Marshal(ldata.Tags) + if err != nil { + log.Println(err, streamer.UserId) + } + streamer.Platform = intimate.Ptwitcasting streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true} streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true} streamer.Operator = 0 + streamer.Tags = tags // streamer.UpdateInterval = 60 clog := &intimate.CollectLog{} clog.UserId = streamer.UserId @@ -101,52 +111,59 @@ func main() { clog.Platform = streamer.Platform clog.UpdateTime = streamer.UpdateTime clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true} - fl, err := intimate.ParseNumberEx(ldata.Follower) - if err == nil { - clog.Followers = sql.NullInt64{Int64: int64(fl), Valid: true} - switch { - case fl <= 100: - streamer.UpdateInterval = 360 - case fl <= 1000: - streamer.UpdateInterval = 240 - case fl <= 100: - streamer.UpdateInterval = 120 - default: - streamer.UpdateInterval = 60 - } - } else { - log.Println(err) - } + clog.Tags = tags - views, err := strconv.Atoi(ldata.MaxViews) - if err == nil { - clog.Views = sql.NullInt64{Int64: int64(views), Valid: true} - } else { - clog.Views = sql.NullInt64{Int64: int64(0), Valid: true} - // log.Println(err, streamer.UserId) + // fl, err := intimate.ParseNumberEx(ldata.Follower) + // if err == nil { + clog.Followers = sql.NullInt64{Int64: int64(ldata.Follower), Valid: true} + switch { + case ldata.Follower <= 100: + streamer.UpdateInterval = 360 + case ldata.Follower <= 1000: + streamer.UpdateInterval = 240 + case ldata.Follower <= 100: + streamer.UpdateInterval = 120 + default: + streamer.UpdateInterval = 60 } + streamer.UpdateInterval = 60 + // } else { + // log.Println(err) + // } + + clog.Views = sql.NullInt64{Int64: ldata.MaxViews, Valid: true} + // views, err := strconv.Atoi(ldata.MaxViews) + // if err == nil { + // clog.Views = sql.NullInt64{Int64: int64(views), Valid: true} + // } else { + // clog.Views = sql.NullInt64{Int64: int64(0), Valid: true} + // // log.Println(err, streamer.UserId) + // } // st, err := strconv.Atoi(ldata.LiveStart) - st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart) - if err == nil { - startTime := st - clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true} - dt, err := strconv.Atoi(ldata.LiveDuration) - if time.Now().Sub(startTime) >= time.Hour*24*90 { - streamer.Operator = 5 - } - + if ldata.LiveStart != "" { + st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart) if err == nil { - endTime := startTime.Add((time.Duration)(dt) * time.Millisecond) - clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true} + startTime := st + clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true} + dt, err := strconv.Atoi(ldata.LiveDuration) + if time.Now().Sub(startTime) >= time.Hour*24*90 { + streamer.Operator = 5 + } + + if err == nil { + endTime := startTime.Add((time.Duration)(dt) * time.Millisecond) + clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true} + } else { + log.Println(err, streamer.UserId) + } } else { log.Println(err, streamer.UserId) } - } else { - log.Println(err, streamer.UserId) } streamer.LatestLogUid = estore.InsertClog(clog) estore.UpdateStreamer(streamer) + log.Println(streamer.UserId) } } diff --git a/extractor/twitcasting_extractor/twitcasting_extractor_test.go b/extractor/twitcasting_extractor/twitcasting_extractor_test.go index 3ed48ff..811b2d3 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor_test.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor_test.go @@ -10,7 +10,7 @@ import ( // Follower string `exp:".//span[@class='tw-user-nav-list-count']" method:"Text"` // } -func estMain(t *testing.T) { +func TestMain(t *testing.T) { main() } diff --git a/go.mod b/go.mod index 312edd1..7794c12 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module intimate go 1.15 require ( - github.com/474420502/extractor v0.9.5 + github.com/474420502/extractor v0.9.6 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.2.0 github.com/474420502/requests v1.7.0 diff --git a/go.sum b/go.sum index 65f21fa..0ac5376 100644 --- a/go.sum +++ b/go.sum @@ -2,18 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/extractor v0.9.5 h1:nM3/1tNL8BHS3PN9eXvm8Ve3hqTReKNB+ba8VpdL8bw= -github.com/474420502/extractor v0.9.5/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY= +github.com/474420502/extractor v0.9.6 h1:mwwzwVeu/sZ4FV190Bl32ta4iVjERBWL6eGLXRDlAtg= +github.com/474420502/extractor v0.9.6/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= -github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= -github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= github.com/474420502/gcurl v0.2.0 h1:m6+vw4NX4f5Tfp7c3nuaIgHUE/7zTX6K3xK+pTCBoCo= github.com/474420502/gcurl v0.2.0/go.mod h1:kJZDbgXn5wbAaR+hhBi4Sbw44P4igJ7qYXC6mejLuhQ= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU= -github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= -github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw= github.com/474420502/requests v1.7.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -24,8 +20,6 @@ github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnRO github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k= github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP3VrhEcx8= github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= -github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= -github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=