intimate/tasks/openrec/openrec_task1/task_openrec.go

package main

import (
	"encoding/json"
	"intimate"
	"log"
	"strconv"
	"time"

	"github.com/474420502/gcurl"
	"github.com/tidwall/gjson"
)

// // sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
// var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))

// // estore 解析存储连接实例
// var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()

// Execute 执行方法
func Execute() {

	ps := intimate.NewPerfectShutdown()
	turl := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
	-H 'authority: public.openrec.tv' \
	-H 'accept: application/json, text/javascript, */*; q=0.01' \
	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
	-H 'origin: https://www.openrec.tv' \
	-H 'sec-fetch-site: same-site' \
	-H 'sec-fetch-mode: cors' \
	-H 'sec-fetch-dest: empty' \
	-H 'referer: https://www.openrec.tv/ranking' \
	-H 'accept-language: zh-CN,zh;q=0.9' \
	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
	--compressed`

	g := gcurl.Parse(turl)
	tp := g.Temporary()

	for !ps.IsClose() {

		resp, err := tp.Execute()
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second * 2)
			continue
		}

		content := resp.Content()
		if len(content) <= 200 { //末页时没有内容返回, 末页退出
			finishpoint := time.Now()
			log.Println("任务Ranking UserId结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*120))
			for time.Now().Sub(finishpoint) < time.Minute*120 {
				time.Sleep(time.Second)
				if ps.IsClose() {
					return
				}
			}

			log.Println("获取Ranking UserId启动:", time.Now())
			querys := tp.GetQuery()
			querys.Set("page", strconv.Itoa(1))
			tp.SetQuery(querys)

			continue
		}

		result := gjson.ParseBytes(content)
		if result.IsArray() {
			for _, User := range result.Array() {

				userid := User.Get("channel.id").String()
				streamer := &intimate.Streamer{}
				streamer.UserId = &userid
				streamer.Platform = intimate.Popenrec

				updateUrl := make(map[string]interface{})

				supportersUrl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01'   -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'  -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;'   --compressed"
				updateUrl["supporters"] = supportersUrl
				updateUrl["user"] = "https://www.openrec.tv/user/" + userid
				updateUrl["live"] = "https://www.openrec.tv/live/" + userid

				updateUrlBytes, err := json.Marshal(updateUrl)
				if err != nil {
					intimate.TStreamer.UpdateError(streamer, err)
					continue
				}

				streamer.UpdateUrl = updateUrlBytes
				intimate.TStreamer.Insert(streamer)
			}
		}

		log.Println("streamer count:", len(result.Array()), tp.ParsedURL.String())
		// 修改url query 参数的page递增. 遍历所有页面
		tp.QueryParam("page").IntAdd(1)
		time.Sleep(time.Second * 1)
	}
}