intimate/tasks/openrec/openrec_task1/task_openrec.go

104 lines
2.3 KiB
Go
Raw Normal View History

package main
import (
"database/sql"
"intimate"
"log"
"strconv"
"time"
"github.com/474420502/hunter"
2020-07-09 09:09:46 +00:00
"github.com/tidwall/gjson"
)
var targetTypeRanking = "openrec_ranking"
2020-07-09 09:09:46 +00:00
var targetTypeUser = "openrec_user"
var openrecRanking *OpenrecRanking
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var store *intimate.Store = intimate.NewStore("source_openrec")
func init() {
openrecRanking = &OpenrecRanking{}
openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
-H 'authority: public.openrec.tv' \
-H 'accept: application/json, text/javascript, */*; q=0.01' \
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
-H 'origin: https://www.openrec.tv' \
-H 'sec-fetch-site: same-site' \
-H 'sec-fetch-mode: cors' \
-H 'sec-fetch-dest: empty' \
-H 'referer: https://www.openrec.tv/ranking' \
-H 'accept-language: zh-CN,zh;q=0.9' \
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
--compressed`
}
// OpenrecRanking 获取排名任务
type OpenrecRanking struct {
hunter.PreCurlUrl
}
// Execute 执行方法
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
for {
resp, err := cxt.Hunt()
if err != nil {
log.Println(err)
break
}
wf := cxt.Workflow()
content := resp.Content()
if len(content) <= 200 {
return
}
2020-07-09 09:09:46 +00:00
result := gjson.ParseBytes(content)
if result.IsArray() {
for _, User := range result.Array() {
data := &intimate.Source{}
userid := User.Get("channel.id").String()
data.SetSource(sql.NullString{String: userid, Valid: len(userid) > 0})
data.SetUrl(wf.GetRawURL())
data.SetTargetType(targetTypeUser)
store.Insert(data)
}
}
querys := wf.GetQuery()
page, err := strconv.Atoi(querys.Get("page"))
if err != nil {
log.Println(err)
return
}
page++
querys.Set("page", strconv.Itoa(page))
wf.SetQuery(querys)
time.Sleep(time.Second * 2)
}
}
// OpenrecUser 获取用户信息
type OpenrecUser struct {
hunter.PreCurlUrl
}
// Execute 执行方法
func (or *OpenrecUser) Execute(cxt *hunter.TaskContext) {
resp, err := cxt.Hunt()
if err != nil {
log.Println(err)
return
}
resp.Content()
}