test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

修改直到curl 成功.

Browse Source
This commit is contained in:
eson 2020-07-13 18:10:48 +08:00
parent a305700a69
commit d034647b93
10 changed files with 400 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.go
View File

@ -16,6 +16,8 @@ func init() {
InitConfig = &Config{}
InitConfig.Load()
// storeOpenrec = NewStore()
log.SetFlags(log.Lshortfile | log.Ldate)
}
// Config 配置

86
extractor/openrec/openrec_test.go
View File

@ -1,35 +1,73 @@
package main
import (
"intimate"
"bytes"
"encoding/gob"
"net/http"
"testing"
"github.com/tidwall/gjson"
"github.com/474420502/requests"
)
func TestExtractor(t *testing.T) {
collect := intimate.NewExtractorStore()
store := intimate.NewSourceStore("source_openrec")
source, err := store.Pop("openrec_user", 100)
if source != nil {
defer store.Restore(source)
}
if err != nil {
t.Error(err)
}
sdata := source.GetExt().([]byte)
if gjson.ValidBytes(sdata) {
result := gjson.ParseBytes(sdata)
m := result.Map()
for key := range m {
t.Error(key)
ai := &intimate.CollectLog{}
ai.SetAnchorId("123")
collect.InsertCollectLog(ai)
}
} else {
t.Error("data is not json:\n", string(sdata))
}
ses := requests.NewSession()
wf := ses.Get("https://www.openrec.tv/user/Riowh/supporters")
wf.Execute()
// t.Error(ses.GetCookies(wf.GetParsedURL()))
buf := bytes.Buffer{}
encoder := gob.NewEncoder(&buf)
encoder.Encode(ses.GetCookies(wf.GetParsedURL()))
var cookies []*http.Cookie
decoder := gob.NewDecoder(&buf)
t.Error(decoder.Decode(&cookies))
t.Error(cookies)
// collect := intimate.NewExtractorStore()
// store := intimate.NewSourceStore("source_openrec")
// source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
// if source != nil {
// defer store.Restore(source)
// }
// if err != nil {
// t.Error(err)
// }
// sdata := source.GetExt().([]byte)
// if gjson.ValidBytes(sdata) {
// result := gjson.ParseBytes(sdata)
// m := result.Map()
// user := m["user"]
// ai := &intimate.CollectLog{}
// extractor := hunter.NewExtractor([]byte(user.Str))
// xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
// if err != nil {
// t.Error(err)
// }
// if !xp.NodeIter().Next() {
// t.Error("不存在粉丝数")
// }
// followers := strings.ReplaceAll(xp.String(), ",", "")
// followersInt, err := strconv.Atoi(followers)
// if err != nil {
// t.Error(err)
// }
// ai.SetPlatform(string(intimate.Popenrec))
// ai.SetFollowers(sql.NullInt32{Int32: int32(followersInt), Valid: true})
// ai.SetAnchorId(source.GetSource().String)
// collect.InsertCollectLog(ai)
// } else {
// t.Error("data is not json:\n", string(sdata))
// }
}

2
go.mod
View File

@ -3,7 +3,9 @@ module intimate
go 1.14
require (
github.com/474420502/gcurl v0.0.4
github.com/474420502/hunter v0.1.2
github.com/474420502/requests v1.5.1
github.com/go-sql-driver/mysql v1.5.0
github.com/go-yaml/yaml v2.1.0+incompatible // indirect
github.com/satori/go.uuid v1.2.0

9
platform_list.go Normal file
View File

@ -0,0 +1,9 @@
package intimate
// Platform 源的table列表
type Platform string
const (
// Popenrec openrec源table名称
Popenrec Platform = "openrec"
)

13
source_field.go
View File

@ -11,6 +11,7 @@ type IGetSource interface {
GetUrl() string //
GetTargetType() string //
GetSource() sql.NullString //
GetPassGob() sql.NullString //
GetExt() interface{} //
GetUpdateTime() time.Time //
GetOperator() int32 //
@ -22,6 +23,7 @@ type IUpdateSource interface {
GetLastOperator() int32
SetPassGob(sql.NullString)
SetExt(ext interface{}) //
SetUpdateTime(ut time.Time) //
SetOperator(operator int32) //
@ -34,6 +36,7 @@ type Source struct {
Url string //
TargetType string //
Source sql.NullString //
PassGob sql.NullString //
Ext interface{} //
UpdateTime time.Time //
Operator int32 //
@ -42,6 +45,16 @@ type Source struct {
lastOperator int32
}
// GetPassGob Get return PassGob sql.NullString
func (so *Source) GetPassGob() sql.NullString {
return so.PassGob
}
// SetPassGob Set PassGob sql.NullString
func (so *Source) SetPassGob(PassGob sql.NullString) {
so.PassGob = PassGob
}
// GetLastOperator Get return lastOperator int32
func (so *Source) GetLastOperator() int32 {
return so.lastOperator

2
sql/intimate_source.sql
View File

@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS `source_openrec` (
`target_type` varchar(64) NOT NULL,
`source` longtext DEFAULT NULL,
`ext` json DEFAULT NULL,
`pass_gob` blob DEFAULT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`operator` int DEFAULT 0,
`error_msg` text DEFAULT NULL,

10
store.go
View File

@ -60,10 +60,18 @@ func (store *SourceStore) Insert(isource IGetSource) {
// Update 更新数据
func (store *SourceStore) Update(isource IUpdateSource) {
_, err := store.db.Exec("update "+store.table+" set ext = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
_, err := store.db.Exec("update "+store.table+" set ext = ?, pass_gob = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetPassGob(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
store.errorAlarm(err)
}
// UpdateError 更新错误数据
func (store *SourceStore) UpdateError(isource IUpdateSource, err error) {
isource.SetOperator(int32(OperatorError))
isource.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
_, dberr := store.db.Exec("update "+store.table+" set operator = ?, error_msg = ? where uid = ?", isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
store.errorAlarm(dberr)
}
// Restore 恢复Operator数据状态
func (store *SourceStore) Restore(isource IUpdateSource) {
_, err := store.db.Exec("update "+store.table+" set operator = ? where uid = ?", isource.GetLastOperator(), isource.GetUid())

2
tasks/openrec/openrec_task1/task_openrec.go
View File

@ -76,7 +76,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
log.Println(err)
return
}
return
page++
querys.Set("page", strconv.Itoa(page))
wf.SetQuery(querys)

90
tasks/openrec/openrec_task2/task_openrec.go
View File

@ -1,12 +1,13 @@
package main
import (
"database/sql"
"encoding/json"
"intimate"
"log"
"time"
"github.com/474420502/gcurl"
"github.com/474420502/hunter"
)
@ -30,13 +31,11 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
for {
source, err := store.Pop(string(intimate.TTOpenrecUser))
if err != nil {
log.Println(err)
return
}
if source == nil {
return
if source == nil && err != nil {
log.Println(err)
time.Sleep(time.Second * 2)
continue
}
userSource := &intimate.Source{}
@ -50,32 +49,71 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
store.UpdateError(source, err)
continue
}
cookies := cxt.Session().GetCookies(wf.GetParsedURL())
scurl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
curl := gcurl.ParseRawCURL(scurl)
supportersSession := curl.CreateSession()
supportersSession.SetCookies(wf.GetParsedURL(), cookies)
log.Println(curl.ParsedURL)
workflow := curl.CreateWorkflow(supportersSession)
for {
supportersQuery := workflow.GetQuery()
for _, cookie := range cookies {
if cookie.Name == "uuid" {
supportersQuery.Set("Uuid", cookie.Value)
continue
}
if cookie.Name == "token" {
supportersQuery.Set("Token", cookie.Value)
continue
}
if cookie.Name == "random" {
supportersQuery.Set("Random", cookie.Value)
continue
}
}
supportersQuery.Set("identify_id", source.GetSource().String)
workflow.SetQuery(supportersQuery)
log.Println(workflow.GetRawURL())
resp, err := workflow.Execute()
if err != nil {
log.Println(err)
} else {
log.Println(string(resp.Content()))
}
return
}
// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
ext := make(map[string]interface{})
ext["user"] = string(resp.Content())
wf = cxt.Session().Get(userUrl + "/supporters")
resp, err = wf.Execute()
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
ext["user_supporters"] = string(resp.Content())
// wf = cxt.Session().Get(userUrl + "/supporters")
// resp, err = wf.Execute()
// if err != nil {
// log.Println(err)
// store.UpdateError(source, err)
// continue
// }
// ext["user_supporters"] = string(resp.Content())
wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid)
resp, err = wf.Execute()
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
store.UpdateError(source, err)
continue
}
ext["user_live"] = string(resp.Content())
@ -83,15 +121,19 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
extJsonBytes, err := json.Marshal(ext)
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
store.UpdateError(source, err)
continue
}
// buf := bytes.Buffer{}
// encoder := gob.NewEncoder(&buf)
// encoder.Encode(cookies)
// source.SetPassGob(sql.NullString{String: buf.String(), Valid: true})
source.SetOperator(int32(intimate.OperatorOK))
source.SetExt(string(extJsonBytes))
store.Update(source)
break
}
}

235
testfile/openrec_supporter.json Normal file
View File

@ -0,0 +1,235 @@
{
"status": 0,
"data": {
"items": [
{
"user_id": 655750535,
"user_name": "\u8471(\u306d\u304e)\u3093\u3061\u3087",
"user_icon": null,
"user_key": "NeginCho",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "ebev00Nwej3",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 480,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 37
},
{
"user_id": 97105260,
"user_name": "\u5f71\u306c\u3044",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/971053\/97105260.png?1496661867",
"user_key": "touka1308",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "PDZBatTy0sDvB",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 480,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 38
},
{
"user_id": 45079954,
"user_name": "\u3086\u3063\u3061\u3083",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/450800\/45079954.png?1482929097",
"user_key": "yuccha1444",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "PUBG\u3000\u30b9\u30d7\u30e9\u30c8\u30a5\u30fc\u30f3\u30e1\u30a4\u30f3\u3067\u3059\u3002\n",
"user_type": 2,
"identify_id": "UQIKagVQXfbRX",
"followed_flg": 0,
"movie_total_views": "466",
"onair_flg": 0,
"onair": null,
"update_at": "2020\/7\/12 14:36:11",
"total_yells": 480,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 39
},
{
"user_id": 72882108,
"user_name": "\u3042\u308b\u3050\u308c",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/728822\/72882108.png?1502904526",
"user_key": "foarsiljw10",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "O2d6Htfxc8PvB",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 480,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 40
},
{
"user_id": 709331194,
"user_name": "\u3044\u3044\u3093\u3059\u304b240\u30a8\u30fc\u30eb\uff01\u3042\u3056\u3059\uff01",
"user_icon": null,
"user_key": "oriton",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "b8JQo5PvZAJ",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 480,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 41
},
{
"user_id": 3756302,
"user_name": "\u3048\u3082\u3042\u3044\u3046",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/37564\/3756302.png?1457963374",
"user_key": "nininiku",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "etB8pDT5kl2Gs",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 42
},
{
"user_id": 102189594,
"user_name": "K",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1021896\/102189594.png?1589000488",
"user_key": "sc__pk",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "FejYFKk3wcoGV",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 43
},
{
"user_id": 193073369,
"user_name": "minomushi",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1930734\/193073369.png?1519733979",
"user_key": "daitaidaitai",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "v_QBD_XBpWoAS",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 44
},
{
"user_id": 126454208,
"user_name": "\u7d76\u671b\u304a\u3063\u3071\u3043\u3061\u3083\u3093",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1264543\/126454208.png?1575873318",
"user_key": "chiyongo",
"user_bg": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/1264543\/126454208_cover.png?1557337359",
"background_file": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/1264543\/126454208_cover.png?1557337359",
"user_introduce": "\u56fd\u6307\u5b9a\u306e\u96e3\u75c5\u3068\u6226\u3063\u3066\u3044\u307e\u3059\u3002\n\u3067\u3082\u6c17\u6301\u3061\u7684\u306b\u306f\u6bce\u65e5\u5143\u6c17\uff01\uff01\uff01\n\u6bce\u65e5\u30b2\u30fc\u30e0\u3059\u308b\u6642\u9593\u304c\u8db3\u308a\u306a\u3044\u60b2\u3057\u307f\u3002\nTwitter\u219288tico88",
"user_type": 2,
"identify_id": "qWbsDP_t1WjER",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 45
},
{
"user_id": 608946713,
"user_name": "\u91ce\u826f\u30aa\u30af\u30bf\u30f3",
"user_icon": null,
"user_key": "ryryryryryryo",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "jAnaeQ11WOg",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 46
},
{
"user_id": 34933103,
"user_name": "\u306f\u308b\u3061",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/349332\/34933103.png?1580674220",
"user_key": "renaryu07131003",
"user_bg": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/349332\/34933103_cover.png?1580674220",
"background_file": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/349332\/34933103_cover.png?1580674220",
"user_introduce": "\u3053\u3093\u306b\u3061\u306f\u3002",
"user_type": 2,
"identify_id": "idqwub_qCyhxd",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 47
},
{
"user_id": 651299052,
"user_name": "\u3084\u3080\u304f\u3093",
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/6512991\/651299052.png?1581381215",
"user_key": "kangsangsoo",
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
"user_introduce": "",
"user_type": 2,
"identify_id": "3XvQmvpxLgL",
"followed_flg": 0,
"movie_total_views": null,
"onair_flg": 0,
"onair": null,
"update_at": null,
"total_yells": 400,
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
"supporter_rank": 48
}
]
}
}