intimate/extractor/openrec_extractor/openrec_extractor.go

package main

import (
	"database/sql"
	"encoding/json"
	"errors"
	"intimate"
	"log"
	"time"

	"github.com/474420502/extractor"
	"github.com/tidwall/gjson"
)

var estore = intimate.NewStoreExtractor()
var sstore = intimate.NewStoreSource(string(intimate.STOpenrec))

//UserInfo 提取信息的结构体
type UserInfo struct {
	UserName  string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"`
	Followers int64  `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"`
	Views     int64  `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"`
}

//UserLive 提取信息的结构体
type UserLive struct {
	Title         string   `exp:"//h1[contains(@class,'MovieTitle__Title')]"`
	LiveStartTime string   `exp:"//meta[@itemprop='uploadDate']/@content"`
	LiveEndTime   string   `exp:"//meta[@itemprop='duration']/@content"`
	Tags          []string `exp:"//a[contains(@class,'TagButton')]"`
}

// Execute 执行
func Execute() {

	ps := intimate.NewPerfectShutdown()

	var lasterr error = nil
	for !ps.IsClose() {
		var err error

		source, err := sstore.Pop(intimate.TOpenrecUser, 0)
		if err != nil {
			if err != lasterr {
				log.Println(err, lasterr)
				lasterr = err
			}
			time.Sleep(time.Second * 5)
			continue
		}
		lasterr = nil

		sdata := source.Ext.([]byte)
		datamap := gjson.ParseBytes(sdata).Map()

		source.Operator = int32(intimate.OperatorError)
		userId := datamap["var_user_id"].String()

		streamer := &intimate.Streamer{}
		streamer.UserId = userId
		// streamer.Platform = intimate.Popenrec 不需要更新字段

		htmlUser := datamap["html_user"]

		userEtor := extractor.ExtractHtmlString(htmlUser.String())
		ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo)

		htmlLive := datamap["html_live"]

		liveEtor := extractor.ExtractHtmlString(htmlLive.String())
		ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive)

		jsonSupporters := datamap["json_supporters"]
		clog := &intimate.CollectLog{}

		if ok1 {
			clog.Followers = sql.NullInt64{Int64: ui.Followers, Valid: true}
			clog.Views = sql.NullInt64{Int64: ui.Views, Valid: true}
			if ui.Views != 0 {
				clog.IsLiveStreaming = true
			}
			streamer.UserName = sql.NullString{String: ui.UserName, Valid: true}

			giverjson := jsonSupporters
			var givers []interface{}
			var gratuity int64 = 0

			for _, v := range giverjson.Array() {
				giverSource := gjson.Parse(v.String())
				for _, item := range giverSource.Get("data.items").Array() {
					givers = append(givers, item.Map())
					gratuity += item.Get("total_yells").Int()
				}
			}

			giversbytes, err := json.Marshal(givers)
			if err != nil {
				log.Println(err)
				clog.ErrorMsg = sql.NullString{String: err.Error(), Valid: true}
			} else {
				clog.Giver = giversbytes
			}

			clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true}
		} else {
			log.Println("UserInfo may be not exists")
			estore.UpdateError(streamer, errors.New("UserInfo may be not exists"))
			continue
		}

		//log.Println(ul)
		if ok2 {
			clog.LiveTitle = sql.NullString{String: ul.Title, Valid: true}

			startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local)
			if err != nil {
				log.Println(err)
			} else {
				clog.LiveStartTime = sql.NullTime{Time: startTime.Local(), Valid: true}
				duration, err := intimate.ParseDuration(ul.LiveEndTime)
				if err != nil {
					log.Println(err)
				} else {
					endTime := startTime.Add(duration)
					clog.LiveStartTime = sql.NullTime{Time: endTime.Local(), Valid: true}
				}
			}

			if tags, err := json.Marshal(ul.Tags); err == nil {
				clog.Tags = tags
			} else {
				log.Println("json error", ul.Tags, clog.Tags)
			}
		}

		streamer.Uid = source.StreamerId.Int64
		streamer.UpdateTime = source.UpdateTime
		streamer.Tags = clog.Tags

		clog.Platform = intimate.Popenrec
		clog.UserId = userId
		clog.UpdateTime = source.UpdateTime
		clog.StreamerUid = streamer.Uid

		logUid := estore.InsertClog(clog)

		LiveUrl := "https://www.openrec.tv/live/" + userId

		streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true}
		streamer.LatestLogUid = logUid
		// streamer.Operator = 0

		log.Println(streamer.UserId)
		estore.Update(streamer,
			"user_name", streamer.UserName,
			"user_id", streamer.UserId,
			"live_url", streamer.LiveUrl,
			"latest_log_uid", streamer.LatestLogUid,
			"update_time", streamer.UpdateTime,
			"tags", streamer.Tags,
		)

		source.Operator = int32(intimate.OperatorExtractorOK)
		sstore.UpdateOperator(source)
	}

}