finish: 解析逻辑, 入库正确.

TODO: 整理代码, 让入库提取数据成为 基础库.

extractor/openrec/openrec_test.go

@@ -19,6 +19,20 @@ import (
 	"github.com/tidwall/gjson"
 )
 
+func TestCase0(t *testing.T) {
+	f, err := os.Open("./test.html")
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+
+	matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})</a`).FindAllStringSubmatch(string(data), -1)
+	t.Error(matheslist)
+}
+
 func TestCase1(t *testing.T) {
 	date := "2020-07-13T18:58:24+09:00"
 
@@ -196,7 +210,7 @@ func TestExtractor(t *testing.T) {
 		// 	t.Error(err)
 		// }
 
-		mathes := regexp.MustCompile("MovieTitle__Title.*>(.+)</h1>").FindStringSubmatch(livejson.Str)
+		mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})</h1>").FindStringSubmatch(livejson.Str)
 		if len(mathes) == 2 {
 
 			clog.SetShowTitle(sql.NullString{String: mathes[1], Valid: true})
@@ -232,8 +246,19 @@ func TestExtractor(t *testing.T) {
 			}
 		}
 
-		matheslist := regexp.MustCompile(`TagButton__Button.+>([^<]+)<`).FindAllStringSubmatch(livejson.Str, 0)
-		t.Error(matheslist)
+		var tags []string
+		matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})</a`).FindAllStringSubmatch(livejson.Str, -1)
+		for _, m := range matheslist {
+			tags = append(tags, m[1])
+		}
+		t.Error(tags)
+		tagsBytes, err := json.Marshal(tags)
+		if err != nil {
+			log.Println(err)
+		}
+
+		ai.SetTags(tagsBytes)
+		ai.SetUpdateTime(source.GetUpdateTime())
 
 		LiveUrl := "https://www.openrec.tv/live/" + anchorId
 		ai.SetLiveUrl(sql.NullString{String: LiveUrl, Valid: true})
@@ -245,6 +270,7 @@ func TestExtractor(t *testing.T) {
 		}
 
 		clog.SetUid(Uid)
+		clog.SetTags(tagsBytes)
 		clog.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true})
 		clog.SetPlatform(string(intimate.Popenrec))
 		clog.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true})

extractor_field.go

@@ -2,19 +2,18 @@ package intimate
 
 import (
 	"database/sql"
-	"time"
 )
 
 type ISetAnchorInfo interface {
-	SetUid(int64)              //
-	SetPlatform(string)        //
-	SetAnchorId(string)        //
-	SetAnchorName(string)      //
-	SetLiveUrl(sql.NullString) //
-	SetChannel(sql.NullString) //
-	SetTags(interface{})       //
-	SetExt(interface{})        //
-	SetUpdateTime(time.Time)   //
+	SetUid(int64)               //
+	SetPlatform(string)         //
+	SetAnchorId(string)         //
+	SetAnchorName(string)       //
+	SetLiveUrl(sql.NullString)  //
+	SetChannel(sql.NullString)  //
+	SetTags(interface{})        //
+	SetExt(interface{})         //
+	SetUpdateTime(sql.NullTime) //
 }
 
 type IGetAnchorInfo interface {
@@ -25,8 +24,8 @@ type IGetAnchorInfo interface {
 	GetLiveUrl() sql.NullString //
 	GetChannel() sql.NullString //
 	GetTags() interface{}
-	GetExt() interface{}      //
-	GetUpdateTime() time.Time //
+	GetExt() interface{}         //
+	GetUpdateTime() sql.NullTime //
 }
 
 /*
@@ -59,8 +58,8 @@ type AnchorInfo struct {
 	LiveUrl    sql.NullString //
 	Channel    sql.NullString //
 	Tags       interface{}
-	Ext        interface{} //
-	UpdateTime time.Time   //
+	Ext        interface{}  //
+	UpdateTime sql.NullTime //
 }
 
 // GetTags Get return Tags interface{}
@@ -74,12 +73,12 @@ func (ai *AnchorInfo) SetTags(Tags interface{}) {
 }
 
 // GetUpdateTime Get return UpdateTime time.Time
-func (ai *AnchorInfo) GetUpdateTime() time.Time {
+func (ai *AnchorInfo) GetUpdateTime() sql.NullTime {
 	return ai.UpdateTime
 }
 
 // SetUpdateTime Set UpdateTime time.Time
-func (ai *AnchorInfo) SetUpdateTime(UpdateTime time.Time) {
+func (ai *AnchorInfo) SetUpdateTime(UpdateTime sql.NullTime) {
 	ai.UpdateTime = UpdateTime
 }
 

store.go

@@ -2,7 +2,6 @@ package intimate
 
 import (
 	"database/sql"
-	"errors"
 	"log"
 
 	_ "github.com/go-sql-driver/mysql"
@@ -112,26 +111,16 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou
 		}
 	}()
 
-	if row != nil {
-		s := &Source{}
-
-		// uid, url, target_type, source, ext, operator
-		err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime)
-		s.SetLastOperator(s.Operator)
-
-		if err != nil {
-			log.Println(err, targetType)
-			_, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", err.Error(), OperatorError, s.Uid)
-			if err != nil {
-				log.Println(err)
-			}
-			return nil, err
-		}
-		_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
-		return s, nil
+	s := &Source{}
+	// uid, url, target_type, source, ext, operator
+	err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime)
+	if err != nil {
+		log.Println(err, targetType)
+		return nil, err
 	}
-
-	return nil, errors.New("TaskQueue is nil")
+	s.SetLastOperator(s.Operator)
+	_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
+	return s, nil
 }
 
 // AnchorTable 主播表名称
@@ -189,12 +178,11 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64
 		log.Println(err)
 		return 0, err
 	}
-
+	log.Println(isource.GetPlatform(), isource.GetAnchorId())
 	row := tx.QueryRow(selectSQL+` limit 1 for update`, isource.GetPlatform(), isource.GetAnchorId())
 
-	if row != nil {
-		var uid int64
-		row.Scan(&uid)
+	var uid int64
+	if err = row.Scan(&uid); err == nil {
 		return uid, nil
 	}
 
@@ -213,6 +201,7 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64
 		if err != nil {
 			log.Println(err)
 		}
+		return 0, err
 	}
 
 	return result.LastInsertId()