intimate/extractor/mirrativ_extractor/mirrativ_extractor.go

package main

import (
	"database/sql"
	"encoding/json"
	"intimate"
	"log"
	"time"

	"github.com/474420502/gcurl"
	"github.com/tidwall/gjson"
)

func main() {

	sessionstr := `
	-H 'authority: www.mirrativ.com' 
	-H 'accept: application/json' 
	-H 'x-timezone: Asia/Shanghai' 
	-H 'x-csrf-token: F3Ojd6RBtApP6YAZzVn-9jWN1of159VxAqOQL1Zn' 
	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' 
	-H 'content-type: application/json' 
	-H 'sec-fetch-site: same-origin' 
	-H 'sec-fetch-mode: cors' 
	-H 'sec-fetch-dest: empty' 
	-H 'referer: https://www.mirrativ.com/live/O5Ia4iX9c5CeZj7DFtg52Q' 
	-H 'accept-language: zh-CN,zh;q=0.9,ja;q=0.8' 
	-H 'cookie: f=A2D75F0E-D218-11EA-A042-452BF6D21CE8; _ga=GA1.2.689947597.1596081392; mr_id=kxb65LddGMZf5C28jkR_tGCZD_ZFOAepD5gfXO7eNjfPMB8EKYvU1Vg_Y29V1lsa; _gid=GA1.2.2116692650.1600139685; lang=ja'`

	ps := intimate.NewPerfectShutdown()
	gprofile := gcurl.Parse(`curl 'https://www.mirrativ.com/api/user/profile?user_id=103383701'` + sessionstr)
	tpProfile := gprofile.CreateTemporary(nil)
	tpProfileUserID := tpProfile.QueryParam("user_id")

	g := gcurl.Parse(`culr 'https://www.mirrativ.com/api/live/live?live_id=O5Ia4iX9c5CeZj7DFtg52Q'` + sessionstr)
	tpLive := g.CreateTemporary(nil)
	tpLiveID := tpLive.QueryParam("live_id")

	var lasterr error
	queue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.PMirrativ))
	for !ps.IsClose() {
		istreamer, err := queue.Pop()
		if err != nil {
			if lasterr != err {
				lasterr = err
				log.Println(err)
			}
			ps.Wait(time.Second * 5)
			continue
		}

		now := &sql.NullTime{Time: time.Now(), Valid: true}
		streamer := istreamer.(*intimate.Streamer)
		streamer.UpdateTime = now
		userid := *streamer.UserId
		log.Println(userid)

		tpProfileUserID.StringSet(userid)
		resp, err := tpProfile.Execute()
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			continue
		}

		clog := &intimate.CollectLog{}
		clog.Platform = intimate.PMirrativ
		clog.UpdateTime = now
		clog.UserId = userid
		clog.StreamerUid = streamer.Uid

		profilejson := gjson.ParseBytes(resp.Content())
		if result := profilejson.Get("follower_num"); result.Exists() {
			clog.Followers = &sql.NullInt64{Int64: result.Int(), Valid: true}
		}

		if result := profilejson.Get("onlive.live_id"); result.Exists() {
			liveID := result.String()
			tpLiveID.StringSet(liveID)
			resp, err = tpLive.Execute()
			if err != nil {
				log.Println(err)
				continue
			}

			livejson := gjson.ParseBytes(resp.Content())
			if result := livejson.Get("total_viewer_num"); result.Exists() {
				clog.Views = &sql.NullInt64{Int64: result.Int(), Valid: true}
			}

			if result := livejson.Get("max_online_viewer_num"); result.Exists() {
				clog.PCU = &sql.NullInt64{Int64: result.Int(), Valid: true}
			}

			if result := livejson.Get("title"); result.Exists() {
				clog.LiveTitle = &sql.NullString{String: result.String(), Valid: true}
			}

			if result := livejson.Get("started_at"); result.Exists() {
				clog.LiveStartTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true}
			}

			if result := livejson.Get("heartbeated_at"); result.Exists() {
				clog.LiveEndTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true}
			}

			if result := livejson.Get("app_id"); result.Exists() {
				streamer.Channel = &sql.NullString{String: result.String(), Valid: true}
			}

			if result := livejson.Get("timeline.#.app.short_title"); result.Exists() {
				for _, tl := range result.Array() {
					var tags []string = []string{tl.String()}
					jtags, _ := json.Marshal(tags)
					streamer.Tags = jtags
					clog.Tags = jtags
					break
				}
			} else {
				log.Println(string(resp.Content()))
				return
			}

			if result := livejson.Get("gift_ranking_url"); result.Exists() {
				// streamer.Channel = &sql.NullString{String: result.String(), Valid: true}
				gifturl := "curl '" + result.String() + "&type=monthly&cursor='" + sessionstr
				ggift := gcurl.Parse(gifturl)
				tp := ggift.CreateTemporary(nil)
				tp.SetURLRawPath("/api/gift/ranking")
				pcursor := tp.QueryParam("cursor")
				var gratuity int64 = 0

				for {
					giftdata, err := tp.Execute()
					giftjson := gjson.ParseBytes(giftdata.Content())
					if err != nil {
						log.Println(err)
					} else {
						for _, rpoint := range giftjson.Get("ranking.#.point").Array() {
							gratuity += rpoint.Int()
						}
					}
					ncursor := giftjson.Get("next_cursor").String()
					if ncursor == "" {
						break
					}
					pcursor.StringSet(ncursor)
				}
				// https://www.mirrativ.com/gift/ranking?live_id=O5Ia4iX9c5CeZj7DFtg52Q&obfuscated_user_id=PgIBEgc6jVc
				clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true}
			}

			cid, err := intimate.TClog.InsertRetAutoID(clog)
			if err != nil {
				log.Println(err)
			}
			streamer.LatestLogUid = cid
		}

		intimate.TStreamer.Update(streamer)
		time.Sleep(time.Second * 2)
	}
}