TODO: 完善主播表的更新方式. 存在更新最新状态时, 会被改版类似情况覆盖, 从而找不到原信息数据

2020-07-15 15:44:21 +08:00 · 2020-07-15 15:44:21 +08:00 · 75b1d9fb77
commit 75b1d9fb77
parent 0d8b456f41
7 changed files with 163 additions and 103 deletions
--- a/extractor/openrec/openrec_test.go
+++ b/extractor/openrec/openrec_test.go
@ -4,85 +4,85 @@ import (
 	"database/sql"
 	"encoding/json"
 	"intimate"
+	"io/ioutil"
 	"log"
 	"os"
 	"regexp"
 	"strconv"
 	"strings"
 	"testing"
+	"time"

 	"github.com/474420502/hunter"
 	"github.com/474420502/requests"
+	"github.com/lestrrat-go/libxml2"
 	"github.com/tidwall/gjson"
 )

-func preNUm(data byte) int {
-	var mask byte = 0x80
-	var num int = 0
-	//8bit中首个0bit前有多少个1bits
-	for i := 0; i < 8; i++ {
-		if (data & mask) == mask {
-			num++
-			mask = mask >> 1
-		} else {
-			break
-		}
-	}
-	return num
-}
-func isUtf8(data []byte) bool {
-	i := 0
-	for i < len(data) {
-		if (data[i] & 0x80) == 0x00 {
-			// 0XXX_XXXX
-			i++
-			continue
-		} else if num := preNUm(data[i]); num > 2 {
-			// 110X_XXXX 10XX_XXXX
-			// 1110_XXXX 10XX_XXXX 10XX_XXXX
-			// 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
-			// 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
-			// 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
-			// preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数
-			i++
-			for j := 0; j < num-1; j++ {
-				//判断后面的 num - 1 个字节是不是都是10开头
-				if (data[i] & 0xc0) != 0x80 {
-					return false
-				}
-				i++
-			}
-		} else {
-			//其他情况说明不是utf-8
-			return false
-		}
-	}
-	return true
+func TestCase1(t *testing.T) {
+	date := "2020-07-13T18:58:24+09:00"
+
+	tm, err := time.Parse("2006-01-02T15:04:05Z07:00", date)
+	t.Error(err)
+	t.Error(time.Now())
+	t.Error(tm.Local().UTC(), tm.Local())
+
 }

-func isGBK(data []byte) bool {
-	length := len(data)
-	var i int = 0
-	for i < length {
-		if data[i] <= 0x7f {
-			//编码0~127,只有一个字节的编码，兼容ASCII码
-			i++
-			continue
-		} else {
-			//大于127的使用双字节编码，落在gbk编码范围内的字符
-			if data[i] >= 0x81 &&
-				data[i] <= 0xfe &&
-				data[i+1] >= 0x40 &&
-				data[i+1] <= 0xfe &&
-				data[i+1] != 0xf7 {
-				i += 2
-				continue
-			} else {
-				return false
-			}
-		}
+func TestCase2(t *testing.T) {
+	duration1 := "0:00:00"
+	duration2 := "4:56:04"
+	tm2, err := time.Parse("15:04:05", duration2)
+	tm1, err := time.Parse("15:04:05", duration1)
+
+	tm2.Sub(tm1)
+
+	t.Error(err)
+	t.Error(tm2.Sub(tm1))
+
+}
+
+func TestCase(t *testing.T) {
+	f, _ := os.Open("./test.html")
+	data, _ := ioutil.ReadAll(f)
+
+	doc, err := libxml2.ParseHTML(data)
+	if err != nil {
+		panic(err)
 	}
-	return true
+	// doc.CreateElement("meta")
+	// "<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">"
+
+	xresult, err := doc.Find("/html/head")
+	ele, err := doc.CreateElement(`META`)
+
+	if err != nil {
+		panic(err)
+	}
+	ele.SetAttribute("charset", "utf-8")
+
+	if err != nil {
+		panic(err)
+	}
+
+	iter := xresult.NodeIter()
+	if iter.Next() {
+		n := iter.Node()
+
+		err = n.AddChild(ele)
+		// childs, err := n.ChildNodes()
+		if err != nil {
+			t.Error(err)
+		}
+		t.Error(n)
+	}
+
+	xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]")
+	if err != nil {
+		panic(nil)
+	}
+
+	t.Error(xr)
 }

 func TestExtractor(t *testing.T) {
@ -96,6 +96,15 @@ func TestExtractor(t *testing.T) {
 	collect := intimate.NewExtractorStore()
 	store := intimate.NewSourceStore("source_openrec")
 	source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
+
+	anchorId := source.GetSource().String
+
+	ai := &intimate.AnchorInfo{}
+	ai.SetAnchorId(anchorId)
+	ai.SetPlatform(string(intimate.Popenrec))
+
+	collect.InsertAnchorInfo(ai)
+
 	// if source != nil {
 	// 	defer store.Restore(source)
 	// }
@ -110,7 +119,7 @@ func TestExtractor(t *testing.T) {

 		user := m["user"]

-		ai := &intimate.CollectLog{}
+		clog := &intimate.CollectLog{}
 		extractor := hunter.NewExtractor([]byte(user.Str))
 		xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
 		if err != nil {
@ -121,8 +130,8 @@ func TestExtractor(t *testing.T) {
 		}

 		followers := strings.ReplaceAll(xp.String(), ",", "")
-
 		followersInt, err := strconv.ParseInt(followers, 10, 64)
+
 		if err != nil {
 			t.Error(err)
 		}
@ -134,6 +143,7 @@ func TestExtractor(t *testing.T) {
 		} else {
 			t.Error(err)
 		}
+		t.Error(source.GetSource())
 		t.Error(anchorName)

 		// c-contents
@ -149,8 +159,8 @@ func TestExtractor(t *testing.T) {
 				t.Error(err)
 			}

-			ai.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true})
-			ai.SetIsShowing(1)
+			clog.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true})
+			clog.SetIsShowing(1)
 		}

 		var givers []interface{}
@ -167,50 +177,69 @@ func TestExtractor(t *testing.T) {
 		giversbytes, err := json.Marshal(givers)
 		if err != nil {
 			t.Error(err)
-			ai.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+			clog.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
 		} else {
-			ai.SetGiver(giversbytes)
+			clog.SetGiver(giversbytes)
 		}

 		// MovieToolbar__Views-g5e6ic-13 iDRGyA
 		livejson := m["user_live"]

-		f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm)
-		if err != nil {
-			panic(err)
-		}
-		f.WriteString(livejson.String())
-		t.Error(livejson)
-		extractor = hunter.NewExtractor([]byte(livejson.Str))
-		xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]")
+		// f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm)
+		// if err != nil {
+		// 	panic(err)
+		// }
+		// f.WriteString(livejson.String())

-		if err != nil {
-			t.Error(err)
-		}
-		t.Error(xr)
-		iter := xr.NodeIter()
-		if iter.Next() {
-			t.Error(iter.Node().TextContent())
-			ai.SetShowTitle(sql.NullString{String: iter.Node().TextContent(), Valid: true})
+		extractor = hunter.NewExtractor([]byte(livejson.Str))
+		// xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]")
+		// if err != nil {
+		// 	t.Error(err)
+		// }
+
+		mathes := regexp.MustCompile("MovieTitle__Title.*>(.+)</h1>").FindStringSubmatch(livejson.Str)
+		if len(mathes) == 2 {
+
+			clog.SetShowTitle(sql.NullString{String: mathes[1], Valid: true})

 			content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content")
 			if err != nil {
 				t.Error(err)
 			}

-			if content.NodeIter().Next() {
-				t.Error(content.String())
+			iter := content.NodeIter()
+			if iter.Next() {
+				tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local)
+				if err != nil {
+					t.Error(err)
+				}
+				clog.SetShowStartTime(sql.NullTime{Time: tm.Local(), Valid: true})
+
+				duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content")
+				if err != nil {
+					t.Error(err)
+				}
+
+				diter := duration.NodeIter()
+				if diter.Next() {
+
+					dt, err := intimate.ParseDuration(diter.Node().NodeValue())
+					if err != nil {
+						log.Println(err)
+					}
+					endtm := tm.Add(dt)
+					clog.SetShowEndTime(sql.NullTime{Time: endtm.Local(), Valid: true})
+				}
 			}
 		}
-		t.Error(xr.String(), xr.NodeIter().Next(), xr.String())

-		ai.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true})
-		ai.SetPlatform(string(intimate.Popenrec))
-		ai.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true})
-		ai.SetAnchorId(source.GetSource().String)
-		ai.SetUpdateTime(source.GetUpdateTime())
+		clog.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true})
+		clog.SetPlatform(string(intimate.Popenrec))
+		clog.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true})
+		clog.SetAnchorId(anchorId)
+		clog.SetUpdateTime(source.GetUpdateTime())

-		collect.InsertCollectLog(ai)
+		collect.InsertCollectLog(clog)

 	} else {
 		t.Error("data is not json:\n", string(sdata))
--- a/go.mod
+++ b/go.mod
@ -7,6 +7,7 @@ require (
 	github.com/474420502/hunter v0.3.0
 	github.com/474420502/requests v1.6.0
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb
 	github.com/tidwall/gjson v1.6.0
 	github.com/tidwall/pretty v1.0.1 // indirect
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
--- a/sql/intimate_extractor.sql
+++ b/sql/intimate_extractor.sql
@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS `anchor_info` (
  `channel`       varchar(128) DEFAULT NULL,
  `show_type`     varchar(255) DEFAULT NULL,
  `ext`           json DEFAULT NULL,
-  `update_time`   timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `update_time`   timestamp   NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`uid`),
  UNIQUE KEY `platform_anchor_id_idx` (`platform`, `anchor_id`),
  KEY `platform_idx` (`platform`),
@ -35,9 +35,9 @@ CREATE TABLE IF NOT EXISTS `collect_log` (
  `gratuity` bigint(11) DEFAULT NULL,

  `show_title` text DEFAULT NULL,
-  `show_start_time` timestamp NULL DEFAULT NULL,
-  `show_end_time` timestamp NULL DEFAULT NULL,
-  `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `show_start_time` timestamp   NULL DEFAULT NULL,
+  `show_end_time` timestamp  NULL DEFAULT NULL,
+  `update_time` timestamp  NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  `ext` json DEFAULT NULL,
  
  `error_msg` text DEFAULT NULL,
--- a/sql/intimate_source.sql
+++ b/sql/intimate_source.sql
@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `source_openrec` (
 	`source` longtext DEFAULT NULL,
 	`ext` json DEFAULT NULL,
 	`pass_gob` blob DEFAULT NULL,
-	`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+	`update_time` timestamp  NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 	`operator` int DEFAULT 0,
 	`error_msg` text DEFAULT NULL,
 	PRIMARY KEY(`uid`),
--- a/store.go
+++ b/store.go
@ -181,9 +181,10 @@ func NewExtractorStore() *ExtractorStore {
 */

 // InsertAnchorInfo AnchorInfo表, 插入数据
-func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) {
+func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) error {
 	_, err := store.db.Exec("insert  into "+AnchorTable+"(platform, anchor_id, anchor_name,  live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt())
 	store.errorAlarm(err)
+	return err
 }

 /*
@ -209,9 +210,10 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) {
 */

 // InsertCollectLog CollectLog表插入数据
-func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) {
+func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) error {
 	_, err := store.db.Exec("insert  into "+CollectLogTable+"(uid, platform, anchor_id,  is_showing, is_error, followers, views, giver, gratuity, show_title, show_start_time, show_end_time, update_time, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
 		isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsShowing(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetShowTitle(), isource.GetShowStartTime(), isource.GetShowEndTime(), isource.GetUpdateTime(), isource.GetExt(), isource.GetErrorMsg(),
 	)
 	store.errorAlarm(err)
+	return err
 }
--- a/tasks/openrec/openrec_task2/task_openrec.go
+++ b/tasks/openrec/openrec_task2/task_openrec.go
@ -35,7 +35,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {

 		source, err := store.Pop(string(intimate.TTOpenrecUser))

-		if source == nil && err != nil {
+		if source == nil || err != nil {
 			log.Println(err)
 			time.Sleep(time.Second * 2)
 			continue
--- a/utils.go
+++ b/utils.go
@ -0,0 +1,28 @@
+package intimate
+
+import (
+	"log"
+	"time"
+)
+
+var zeroTime time.Time
+
+func init() {
+
+	tm, err := time.Parse("15:04:05", "0:00:00")
+	if err != nil {
+		log.Println(err)
+	}
+	zeroTime = tm
+
+}
+
+// ParseDuration time to duration eg: 1:40:00 -> time.Duration
+func ParseDuration(dt string) (time.Duration, error) {
+	tdt, err := time.Parse("15:04:05", dt)
+	if err != nil {
+
+		return time.Duration(0), err
+	}
+	return tdt.Sub(zeroTime), nil
+}