diff --git a/config.yaml b/config.yaml index e55d371..5a5df4a 100644 --- a/config.yaml +++ b/config.yaml @@ -1,3 +1,3 @@ database: - source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source" - extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor" \ No newline at end of file + source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true" + extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true" \ No newline at end of file diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index b4921f4..bb73e3c 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -1,73 +1,219 @@ package main import ( - "bytes" - "encoding/gob" - "net/http" + "database/sql" + "encoding/json" + "intimate" + "log" + "os" + "regexp" + "strconv" + "strings" "testing" + "github.com/474420502/hunter" "github.com/474420502/requests" + "github.com/tidwall/gjson" ) +func preNUm(data byte) int { + var mask byte = 0x80 + var num int = 0 + //8bit中首个0bit前有多少个1bits + for i := 0; i < 8; i++ { + if (data & mask) == mask { + num++ + mask = mask >> 1 + } else { + break + } + } + return num +} +func isUtf8(data []byte) bool { + i := 0 + for i < len(data) { + if (data[i] & 0x80) == 0x00 { + // 0XXX_XXXX + i++ + continue + } else if num := preNUm(data[i]); num > 2 { + // 110X_XXXX 10XX_XXXX + // 1110_XXXX 10XX_XXXX 10XX_XXXX + // 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数 + i++ + for j := 0; j < num-1; j++ { + //判断后面的 num - 1 个字节是不是都是10开头 + if (data[i] & 0xc0) != 0x80 { + return false + } + i++ + } + } else { + //其他情况说明不是utf-8 + return false + } + } + return true +} + +func isGBK(data []byte) bool { + length := len(data) + var i int = 0 + for i < length { + if data[i] <= 0x7f { + //编码0~127,只有一个字节的编码，兼容ASCII码 + i++ + continue + } else { + //大于127的使用双字节编码，落在gbk编码范围内的字符 + if data[i] >= 0x81 && + data[i] <= 0xfe && + data[i+1] >= 0x40 && + data[i+1] <= 0xfe && + data[i+1] != 0xf7 { + i += 2 + continue + } else { + return false + } + } + } + return true +} + func TestExtractor(t *testing.T) { ses := requests.NewSession() - wf := ses.Get("https://www.openrec.tv/user/Riowh/supporters") - wf.Execute() + tp := ses.Get("https://www.openrec.tv/user/Riowh/supporters") + tp.Execute() // t.Error(ses.GetCookies(wf.GetParsedURL())) - buf := bytes.Buffer{} - encoder := gob.NewEncoder(&buf) - encoder.Encode(ses.GetCookies(wf.GetParsedURL())) - - var cookies []*http.Cookie - decoder := gob.NewDecoder(&buf) - t.Error(decoder.Decode(&cookies)) - t.Error(cookies) - - // collect := intimate.NewExtractorStore() - // store := intimate.NewSourceStore("source_openrec") - // source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + collect := intimate.NewExtractorStore() + store := intimate.NewSourceStore("source_openrec") + source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) // if source != nil { // defer store.Restore(source) // } - // if err != nil { - // t.Error(err) - // } - // sdata := source.GetExt().([]byte) + if err != nil { + t.Error(err) + } + sdata := source.GetExt().([]byte) - // if gjson.ValidBytes(sdata) { - // result := gjson.ParseBytes(sdata) - // m := result.Map() + if gjson.ValidBytes(sdata) { + result := gjson.ParseBytes(sdata) + m := result.Map() - // user := m["user"] + user := m["user"] - // ai := &intimate.CollectLog{} - // extractor := hunter.NewExtractor([]byte(user.Str)) - // xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") - // if err != nil { - // t.Error(err) - // } - // if !xp.NodeIter().Next() { - // t.Error("不存在粉丝数") - // } + ai := &intimate.CollectLog{} + extractor := hunter.NewExtractor([]byte(user.Str)) + xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") + if err != nil { + t.Error(err) + } + if !xp.NodeIter().Next() { + t.Error("不存在粉丝数") + } - // followers := strings.ReplaceAll(xp.String(), ",", "") + followers := strings.ReplaceAll(xp.String(), ",", "") - // followersInt, err := strconv.Atoi(followers) - // if err != nil { - // t.Error(err) - // } + followersInt, err := strconv.ParseInt(followers, 10, 64) + if err != nil { + t.Error(err) + } - // ai.SetPlatform(string(intimate.Popenrec)) - // ai.SetFollowers(sql.NullInt32{Int32: int32(followersInt), Valid: true}) - // ai.SetAnchorId(source.GetSource().String) + var anchorName string + xp, err = extractor.XPathResult("//p[@class='c-global__user__profile__list__name__text official-icon--after']/text()") + if xp.NodeIter().Next() { + anchorName = xp.String() + } else { + t.Error(err) + } + t.Error(anchorName) - // collect.InsertCollectLog(ai) + // c-contents + xp, err = extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()") + if err != nil { + log.Println(err) + } + if xp.NodeIter().Next() { + views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String()) + views = strings.ReplaceAll(views, ",", "") + viewsint, err := strconv.Atoi(views) + if err != nil { + t.Error(err) + } - // } else { - // t.Error("data is not json:\n", string(sdata)) - // } + ai.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true}) + ai.SetIsShowing(1) + } + + var givers []interface{} + var gratuity int64 = 0 + giverjson := m["supporters"] + for _, v := range giverjson.Array() { + giverSource := gjson.Parse(v.String()) + for _, item := range giverSource.Get("data.items").Array() { + givers = append(givers, item.Map()) + gratuity += item.Get("total_yells").Int() + } + } + + giversbytes, err := json.Marshal(givers) + if err != nil { + t.Error(err) + ai.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + } else { + ai.SetGiver(giversbytes) + } + + // MovieToolbar__Views-g5e6ic-13 iDRGyA + livejson := m["user_live"] + + f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm) + if err != nil { + panic(err) + } + f.WriteString(livejson.String()) + t.Error(livejson) + extractor = hunter.NewExtractor([]byte(livejson.Str)) + xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]") + + if err != nil { + t.Error(err) + } + t.Error(xr) + iter := xr.NodeIter() + if iter.Next() { + t.Error(iter.Node().TextContent()) + ai.SetShowTitle(sql.NullString{String: iter.Node().TextContent(), Valid: true}) + + content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content") + if err != nil { + t.Error(err) + } + + if content.NodeIter().Next() { + t.Error(content.String()) + } + } + t.Error(xr.String(), xr.NodeIter().Next(), xr.String()) + + ai.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true}) + ai.SetPlatform(string(intimate.Popenrec)) + ai.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true}) + ai.SetAnchorId(source.GetSource().String) + ai.SetUpdateTime(source.GetUpdateTime()) + + collect.InsertCollectLog(ai) + + } else { + t.Error("data is not json:\n", string(sdata)) + } } diff --git a/extractor_field.go b/extractor_field.go index 9aa9cbd..ad5127e 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -159,14 +159,14 @@ type IGetCollectLog interface { GetAnchorId() string // GetIsShowing() int32 // GetIsError() int32 // - GetFollowers() sql.NullInt32 // - GetViews() sql.NullInt32 // + GetFollowers() sql.NullInt64 // + GetViews() sql.NullInt64 // GetGiver() interface{} // - GetGratuity() sql.NullInt32 // + GetGratuity() sql.NullInt64 // GetShowTitle() sql.NullString // GetShowStartTime() sql.NullTime // GetShowEndTime() sql.NullTime // - GetUpdateTime() time.Time // + GetUpdateTime() sql.NullTime // GetExt() interface{} // GetErrorMsg() sql.NullString // } @@ -177,14 +177,14 @@ type ISetCollectLog interface { SetAnchorId(string) // SetIsShowing(int32) // SetIsError(int32) // - SetFollowers(sql.NullInt32) // - SetViews(sql.NullInt32) // + SetFollowers(sql.NullInt64) // + SetViews(sql.NullInt64) // SetGiver(interface{}) // - SetGratuity(sql.NullInt32) // + SetGratuity(sql.NullInt64) // SetShowTitle(sql.NullString) // SetShowStartTime(sql.NullTime) // SetShowEndTime(sql.NullTime) // - SetUpdateTime(time.Time) // + SetUpdateTime(sql.NullTime) // SetExt(interface{}) // SetErrorMsg(sql.NullString) // } @@ -229,14 +229,14 @@ type CollectLog struct { AnchorId string // IsShowing int32 // IsError int32 // - Followers sql.NullInt32 // - Views sql.NullInt32 // + Followers sql.NullInt64 // + Views sql.NullInt64 // Giver interface{} // - Gratuity sql.NullInt32 // + Gratuity sql.NullInt64 // ShowTitle sql.NullString // ShowStartTime sql.NullTime // ShowEndTime sql.NullTime // - UpdateTime time.Time // + UpdateTime sql.NullTime // Ext interface{} // ErrorMsg sql.NullString // } @@ -262,12 +262,12 @@ func (cl *CollectLog) SetExt(Ext interface{}) { } // GetUpdateTime Get return UpdateTime time.Time -func (cl *CollectLog) GetUpdateTime() time.Time { +func (cl *CollectLog) GetUpdateTime() sql.NullTime { return cl.UpdateTime } // SetUpdateTime Set UpdateTime time.Time -func (cl *CollectLog) SetUpdateTime(UpdateTime time.Time) { +func (cl *CollectLog) SetUpdateTime(UpdateTime sql.NullTime) { cl.UpdateTime = UpdateTime } @@ -302,12 +302,12 @@ func (cl *CollectLog) SetShowTitle(ShowTitle sql.NullString) { } // GetGratuity Get return Gratuity sql.NullInt32 -func (cl *CollectLog) GetGratuity() sql.NullInt32 { +func (cl *CollectLog) GetGratuity() sql.NullInt64 { return cl.Gratuity } // SetGratuity Set Gratuity sql.NullInt32 -func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt32) { +func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt64) { cl.Gratuity = Gratuity } @@ -321,23 +321,23 @@ func (cl *CollectLog) SetGiver(Giver interface{}) { cl.Giver = Giver } -// GetViews Get return Views sql.NullInt32 -func (cl *CollectLog) GetViews() sql.NullInt32 { +// GetViews Get return Views sql.NullInt64 +func (cl *CollectLog) GetViews() sql.NullInt64 { return cl.Views } -// SetViews Set Views sql.NullInt32 -func (cl *CollectLog) SetViews(Views sql.NullInt32) { +// SetViews Set Views sql.NullInt64 +func (cl *CollectLog) SetViews(Views sql.NullInt64) { cl.Views = Views } -// GetFollowers Get return Followers sql.NullInt32 -func (cl *CollectLog) GetFollowers() sql.NullInt32 { +// GetFollowers Get return Followers sql.NullInt64 +func (cl *CollectLog) GetFollowers() sql.NullInt64 { return cl.Followers } // SetFollowers Set Followers sql.NullInt32 -func (cl *CollectLog) SetFollowers(Followers sql.NullInt32) { +func (cl *CollectLog) SetFollowers(Followers sql.NullInt64) { cl.Followers = Followers } diff --git a/go.mod b/go.mod index 5d32e9d..6575876 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,12 @@ module intimate go 1.14 require ( - github.com/474420502/gcurl v0.0.4 - github.com/474420502/hunter v0.1.2 - github.com/474420502/requests v1.5.1 + github.com/474420502/gcurl v0.1.2 + github.com/474420502/hunter v0.3.0 + github.com/474420502/requests v1.6.0 github.com/go-sql-driver/mysql v1.5.0 - github.com/go-yaml/yaml v2.1.0+incompatible // indirect - github.com/satori/go.uuid v1.2.0 github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.1 // indirect - golang.org/x/tools v0.0.0-20200708003708-134513de8882 // indirect - gopkg.in/yaml.v2 v2.2.2 - honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a + golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect + gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 942a89d..7a07e84 100644 --- a/go.sum +++ b/go.sum @@ -2,21 +2,29 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/focus v0.9.0 h1:Y/TjSdSdIgegO78OJixphiPl1wVqhK9AbcXjiDDugo4= -github.com/474420502/focus v0.9.0/go.mod h1:jrDXvK1CnUJ3PCR3ZJVYinbS2Yz5kM8OoAbCLe6AF7Y= -github.com/474420502/gcurl v0.0.4 h1:eR1BNXvQ4T245dotWpjDzAMWch+FTTfScqzsdq93JK0= -github.com/474420502/gcurl v0.0.4/go.mod h1:qtCzAZZbVRIsBt0lNUh2I0qDniU9T3E21aSsVUYo7Hc= -github.com/474420502/hunter v0.1.2 h1:1dY2C9IJvh81+04Xx8OuTPcUw6WNlKCiOaGmJdgX83Y= -github.com/474420502/hunter v0.1.2/go.mod h1:41DpZWSsGBWsFwb/liapbT1uH58Yvl+BpW4SJwLC2Fw= -github.com/474420502/requests v1.5.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= -github.com/474420502/requests v1.5.1 h1:miv6O4RMbZ8I0ZdUTLf/EU5Dmewc/4IL/DmUMwtuv8M= -github.com/474420502/requests v1.5.1/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= +github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= +github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= +github.com/474420502/gcurl v0.1.0 h1:aKPP27qAnofntTqqVF8rjejHBWVlYWzFEZGdqjBiMgw= +github.com/474420502/gcurl v0.1.0/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= +github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= +github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= +github.com/474420502/hunter v0.2.0 h1:pH7xIWzas2IuLdCJL/HtcouHcZQq0XIP/FswY7yF+pA= +github.com/474420502/hunter v0.2.0/go.mod h1:c1+92qUtjNzYw6Mzl6Qkb2kMALAXMlYQk3kJdoOqnmY= +github.com/474420502/hunter v0.3.0 h1:0VPi1MInxjHOta3da4v0ALWK0y3/X4/6nUSLFvdbiFU= +github.com/474420502/hunter v0.3.0/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA= +github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= +github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA= github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k= github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP3VrhEcx8= github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= +github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -24,12 +32,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY= github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -39,6 +48,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -54,11 +64,11 @@ github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb h1:qqNmX9V9n4b github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb/go.mod h1:fy/ZVbgyB83mtricxwSW3zqIRXWOVpKG2PvdUDFeC58= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w= github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc= @@ -71,12 +81,10 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -87,7 +95,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -100,8 +107,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -110,7 +117,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -122,6 +128,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -136,12 +143,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJYds2YJwZRNw4zs7wSnNhk= golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200708003708-134513de8882 h1:x4Two2lSwHxTqR+eal4lB4ydUnTvmDDpPQeL92ZHDgA= -golang.org/x/tools v0.0.0-20200708003708-134513de8882/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -159,10 +160,18 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7 h1:zibSPXbkfB1Dwl76rJgLa68xcdHu42qmFTe6vAnU4wA= +gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7/go.mod h1:wo0SW5T6XqIKCCAge330Cd5sm+7VI6v85OrQHIk50KM= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +launchpad.net/xmlpath v0.0.0-20130614043138-000000000004 h1:B8nNZBUrx8YufDCAJjvO/lVs4GxXMQHyrjwJdJzXMFg= +launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/source_field.go b/source_field.go index 433736f..b8d601c 100644 --- a/source_field.go +++ b/source_field.go @@ -2,7 +2,6 @@ package intimate import ( "database/sql" - "time" ) // IGetSource 源接口结构 @@ -13,7 +12,7 @@ type IGetSource interface { GetSource() sql.NullString // GetPassGob() sql.NullString // GetExt() interface{} // - GetUpdateTime() time.Time // + GetUpdateTime() sql.NullTime // GetOperator() int32 // GetErrorMsg() sql.NullString // } @@ -25,7 +24,7 @@ type IUpdateSource interface { SetPassGob(sql.NullString) SetExt(ext interface{}) // - SetUpdateTime(ut time.Time) // + SetUpdateTime(ut sql.NullTime) // SetOperator(operator int32) // SetErrorMsg(emsg sql.NullString) // } @@ -38,7 +37,7 @@ type Source struct { Source sql.NullString // PassGob sql.NullString // Ext interface{} // - UpdateTime time.Time // + UpdateTime sql.NullTime // Operator int32 // ErrorMsg sql.NullString // @@ -86,12 +85,12 @@ func (so *Source) SetOperator(Operator int32) { } // GetUpdateTime Get return UpdateTime time.Time -func (so *Source) GetUpdateTime() time.Time { +func (so *Source) GetUpdateTime() sql.NullTime { return so.UpdateTime } // SetUpdateTime Set UpdateTime time.Time -func (so *Source) SetUpdateTime(UpdateTime time.Time) { +func (so *Source) SetUpdateTime(UpdateTime sql.NullTime) { so.UpdateTime = UpdateTime } diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index 8979744..cb1d1e5 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -29,10 +29,10 @@ CREATE TABLE IF NOT EXISTS `collect_log` ( `is_showing` tinyint(1) DEFAULT 0, `is_error` tinyint(1) DEFAULT 0, - `followers` int(11) DEFAULT NULL, - `views` int(11) DEFAULT NULL, + `followers` bigint(11) DEFAULT NULL, + `views` bigint(11) DEFAULT NULL, `giver` json DEFAULT NULL, - `gratuity` int(11) DEFAULT NULL, + `gratuity` bigint(11) DEFAULT NULL, `show_title` text DEFAULT NULL, `show_start_time` timestamp NULL DEFAULT NULL, diff --git a/store.go b/store.go index bfa0596..4810691 100644 --- a/store.go +++ b/store.go @@ -87,7 +87,7 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou return nil, err } var args = []interface{}{targetType} - selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?` + selectSQL := `select uid, url, target_type, source, ext, operator, update_time from ` + store.table + ` where target_type = ?` if len(operators) == 0 { selectSQL += " and operator = ?" args = append(args, 0) @@ -116,12 +116,12 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou s := &Source{} // uid, url, target_type, source, ext, operator - err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator) + err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime) s.SetLastOperator(s.Operator) if err != nil { log.Println(err, targetType) - _, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid) + _, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", err.Error(), OperatorError, s.Uid) if err != nil { log.Println(err) } diff --git a/tasks/openrec/openrec_task1/source_openrec_test.go b/tasks/openrec/openrec_task1/source_openrec_test.go index 8eddfd6..b21c538 100644 --- a/tasks/openrec/openrec_task1/source_openrec_test.go +++ b/tasks/openrec/openrec_task1/source_openrec_test.go @@ -29,8 +29,8 @@ func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) { t.Error("rank is error. result raw is ", result.Raw) } - if cxt.Workflow().GetQuery().Get("page") != "1" { - t.Error("workflow page error") + if cxt.Temporary().GetQuery().Get("page") != "1" { + t.Error("Temporary page error") } // t.Error(string(resp.Content())) } diff --git a/tasks/openrec/openrec_task1/task_openrec.go b/tasks/openrec/openrec_task1/task_openrec.go index 1aeee12..7976b96 100644 --- a/tasks/openrec/openrec_task1/task_openrec.go +++ b/tasks/openrec/openrec_task1/task_openrec.go @@ -50,7 +50,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { break } - wf := cxt.Workflow() + wf := cxt.Temporary() content := resp.Content() if len(content) <= 200 { diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index defa483..6610541 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -1,12 +1,15 @@ package main import ( + "database/sql" "encoding/json" "intimate" "log" + "strconv" "time" "github.com/474420502/gcurl" + "github.com/tidwall/gjson" "github.com/474420502/hunter" ) @@ -45,7 +48,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { wf := cxt.Session().Get(userUrl) resp, err := wf.Execute() - source.SetUpdateTime(time.Now()) + source.SetUpdateTime(sql.NullTime{Time: time.Now(), Valid: true}) if err != nil { log.Println(err) @@ -55,16 +58,16 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { cookies := cxt.Session().GetCookies(wf.GetParsedURL()) - scurl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed" + scurl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed" curl := gcurl.ParseRawCURL(scurl) supportersSession := curl.CreateSession() - log.Println(curl.ParsedURL) - workflow := curl.CreateWorkflow(supportersSession) - supportersSession.SetCookies(workflow.GetParsedURL(), cookies) + temporary := curl.CreateTemporary(supportersSession) + supportersSession.SetCookies(temporary.GetParsedURL(), cookies) + var supporters []string for { - supportersQuery := workflow.GetQuery() + supportersQuery := temporary.GetQuery() for _, cookie := range cookies { if cookie.Name == "uuid" { @@ -84,34 +87,37 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { } supportersQuery.Set("identify_id", source.GetSource().String) + temporary.SetQuery(supportersQuery) - workflow.SetQuery(supportersQuery) - // workflow.AddCookies(cookies) - log.Println(workflow.GetRawURL()) - log.Println(supportersSession.GetCookies(workflow.GetParsedURL())) - resp, err := workflow.Execute() + resp, err := temporary.Execute() if err != nil { log.Println(err) - } else { - log.Println(string(resp.Content())) } - return - } - // cookies := cxt.Session().GetCookies(wf.GetParsedURL()) + supporterjson := gjson.ParseBytes(resp.Content()) + supporterdata := supporterjson.Get("data") + if supporterdata.Type == gjson.Null { + break + } + supporters = append(supporters, string(resp.Content())) + page := supportersQuery.Get("page_number") + pageint, err := strconv.Atoi(page) + if err != nil { + log.Println(err) + break + } + pageint++ + page = strconv.Itoa(pageint) + supportersQuery.Set("page_number", page) + temporary.SetQuery(supportersQuery) + } + + // cookies := cxt.Session().GetCookies(wf.GetParsedURL()) ext := make(map[string]interface{}) + ext["supporters"] = supporters ext["user"] = string(resp.Content()) - // wf = cxt.Session().Get(userUrl + "/supporters") - // resp, err = wf.Execute() - // if err != nil { - // log.Println(err) - // store.UpdateError(source, err) - // continue - // } - // ext["user_supporters"] = string(resp.Content()) - wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid) resp, err = wf.Execute() if err != nil { @@ -128,15 +134,9 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { continue } - // buf := bytes.Buffer{} - // encoder := gob.NewEncoder(&buf) - // encoder.Encode(cookies) - - // source.SetPassGob(sql.NullString{String: buf.String(), Valid: true}) source.SetOperator(int32(intimate.OperatorOK)) source.SetExt(string(extJsonBytes)) store.Update(source) - break } } diff --git a/testfile/openrec_user_supporters.html b/testfile/openrec_user_supporters.html deleted file mode 100755 index 19d322e..0000000 --- a/testfile/openrec_user_supporters.html +++ /dev/null @@ -1,1935 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 手越ちゃんねる | OPENREC.tv - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- -
- - - - - - - -
-
- -
- -
- - - -
-
-
- - -
-
-
-
-
-
-
-
Trends
-
-

ピューロ

-
-
-

APEX LEGENDS

-
-
-

FORTNITE

-
-
-

VALORANT

-
-
-

あつまれ どうぶつの森

-
-
-
-
History
-
-
-
-
関連ゲームタイトル
-
-
    -
-
-
-
-
関連チャンネル
-
-
    -
-
-
-
-
関連キーワード
-
-
    -
-
-
-
- -
-
-
-
-
- -
- -
- - - - - Level up to
Premium Account! - -
- -
- - - - ダークモード -
- - -
-
- -
-
- - - - - -
-
- -
-
- - - - - -
- -
- -
-
-

-

- - -

-
-
- -
- -
    -
  • -
    - - - - ダークモード -
    - - -
    -
    -
    • - - -
-
- - - -
-

Important Notice -

-
-
    -
- See more -
- - - -

Notice -

-
-
    -
-
-
- - - - -
- - -
- -
- -
-
-
    -
  • - - Login -
    • -
  • -
    - - - - -
    -
    • -
  • -
    - - -
    -
    • -
  • -
    - Received on agreement to the Terms of Use,
    Privacy Policy of OPENREC, then you proceed please .
    -
    • - -
  • -
    -
    - Upgrade to Premium Membership -
    - - - There are a bunch of perks that will allow you to enjoy OPENREC.tv even more.
    -
    -
    • -
- -
-
- -
-
- - -
-
- - -
-
-
- -
- -
-
- - -
-
-
    -
  • - Provisional registration completion -
    • -
  • - We've sent a verification e-mail to the registered e-mail address. Verify your registration by confirming the message.
    • -
  • - If you do not receive the verification email, please check the following .
    • -
-
    -
  • Or there was no error in - your input our e-mail address
    • -
  • Or e-mail from the " @ openrec.jp " domain has not been classified as junk folder
    • -
  • Or e-mail from the " @ openrec.jp " domain is not being blocked by domain rejection setting
-
- - -
- -
-
- - -
-
-
    -
  • - Reregistration your password -
    • -
  • - Enter the email address you've registered.
    -We'll send you an email to reset the password.
    • -
  • - -
    • -
  • -
  • - -
    • -
- -
-
- - -
-
-
-
- Reregistration your password
-
- We've sent you an email to reset the password.
-
- -
- -
-
- - -
-
-
    -
  • - Reregistration your password -
    • -
  • - Highly secure password is a combination of alphanumeric characters and symbols .
    • -
  • - -
    • -
  • - -
    • -
  • -

    - Reregistration your password

    -
    • -
- -
-
- - -
-
-
    -
  • - Report as a Inappropriate movie -
    • -
  • - Types of problem -
    • -
  • - -
    • -
  • - Details of problem -
    • -
  • - -
    • -
  • - We will check if reported contents is violating terms of service
    - To use this function immoderately is also violating terms of service.
    • -
  • -

    - Send

    -
    • -
- -
-
- - -
-
-
    -
  • - Report as a spam -
    • -
  • - Details of problem -
    • -
  • - -
    • -
  • - Details of problem -
    • -
  • - -
    • -
  • - We will check if reported contents is violating terms of service
    - To use this function immoderately is also violating terms of service.
    • -
  • -

    - Send

    -
    • -
- -
-
- - -
-
-
    -
  • - Report as a Inappropriate movie -
    • -
  • - Thank you for reporting
    - We will check if reported contents is violating terms of service
    -
    • -
- -
-
- - -
-
-
    -
  • - Report as a spam -
    • -
  • - Thank you for reporting
    - We will check if reported contents is violating terms of service
    -
    • -
- -
-
- -
-
-
    -
  • - For playing video -
    • -
  • - Playing video, it is required to install Flash Player. If you can't play video please download the latest version of Flash Player from here.
    • -
- -
-
- - -
-
-
-
- Your request is pending now.
Please wait for the result.
-
- Your request got denied.
-
- You can not apply with this account.
-
- -
-
- - - - -
-
-
- - - -
-
-
    -
  • - To watch a previous live broadcast -
    • -
  • -
    You can get various privileges by upgrading to premium account
    -
    - Chat/comment stamp -
    -
    - Unlimited viewing of archived videos -
    -
    - Special Premium Member only broadcasts -
    -
    • -
  • - Register in only 3 steps!Easy ragestration.¥550/month
    • -
  • - Become premium member -
    • -
- -
-
- - -
-
-
    -
  • - 超低遅延モードで視聴するためには、
    プレミアム会員への入会が必要です。     -
    • -
  • -
    You can get various privileges by upgrading to premium account
    -
    - Chat/comment stamp -
    -
    - Unlimited viewing of archived videos -
    -
    - Special Premium Member only broadcasts -
    -
    • -
  • - Register in only 3 steps!Easy ragestration.¥550/month
    • -
  • - Become premium member -
    • -
- -
-
- - -
-
-
    -
  • - You can set 200 forbidden words
    if you upgrade to premium account     -
    • -
  • -
    You can get various privileges by upgrading to premium account
    -
    - Chat/comment stamp -
    -
    - Unlimited viewing of archived videos -
    -
    - Special Premium Member only broadcasts -
    -
    • -
  • - Register in only 3 steps!Easy ragestration.¥550/month
    • -
  • - Become premium member -
    • -
- -
-
- - -
-
-
    -
  • - Premium user can pop out a chat. -
    • -
  • -
    You can get various privileges by upgrading to premium account
    -
    - Chat/comment stamp -
    -
    - Unlimited viewing of archived videos -
    -
    - Special Premium Member only broadcasts -
    -
    • -
  • - Register in only 3 steps!Easy ragestration.¥550/month
    • -
  • - Become premium member -
    • -
- -
-
- -
-
-
-
- This video has been finished
-
- -
- -
-
- - -
-
-
-
- Do you want to remove it from the blacklist?
-
- - -
- -
-
- - -
-
-
-
- Are you sure you want to add to the black list?
-
- - -
- -
-
- - -
-
-
-
- Do you want to repoart as a spam user?
-
- - -
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
- -
-
-

Videos

-

5

-
-
-

Views

-

732,990

-
-
-

Followers

-

64,978

-
-
-

Supporter

-

281

-
-
- - -
- -
-
- -
-
-
-
-
- 手越ちゃんねる
-
-
-
-
-
-
- -
- -
-
-
-

Channels

 - -

- Capture

- -

Follow

 -

Follower

 -

Supporter

 -

Your team

 -

Playlist

 -
-
-
- - - - - -
-
-
-
-
-
-

-
- -
-
-
-
- Are you sure to add to the team members
-
- - -
- -
-
-
-
-
-
- Are you sure to remove from the team members
-
- - -
- -
-
- -
-
-
- - diff --git a/testfile/openrec_user_supporters.json b/testfile/openrec_user_supporters.json new file mode 100755 index 0000000..7bbb2a8 --- /dev/null +++ b/testfile/openrec_user_supporters.json @@ -0,0 +1,7 @@ +{ + "supporters": [ + "{\"status\":0,\"data\":{\"items\":[{\"user_id\":228479866,\"user_name\":\"Dayeee\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/2284799\\/228479866.png?1590723285\",\"user_key\":\"shengyq1993\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\\u3074\\u3087\\u3093\\u3055\\u3093\\u306e\\u30d5\\u30a1\\u30f3\\u3067\\u3059\",\"user_type\":2,\"identify_id\":\"B_fON_LCTQ80x\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":10600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/26\\/496dc6c54b4004437d44784f4c06921387d6e39c.gif\",\"supporter_rank\":1},{\"user_id\":180754796,\"user_name\":\"\\ud835\\udc24\\ud835\\udc1a\\ud835\\udc28\\ud835\\udc2b\\ud835\\udc22\\u2661\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1807548\\/180754796.png?1559139579\",\"user_key\":\"k11a18o\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1807548\\/180754796_cover.png?1584442115\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1807548\\/180754796_cover.png?1584442115\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"vmYmTocjdxMTK\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":7680,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/25\\/3496e3d762840443ccd55dd2e5809df0075f5173.gif\",\"supporter_rank\":2},{\"user_id\":84826105,\"user_name\":\"\\u3055\\u3044\\u3068\\u30fc\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/848262\\/84826105.png?1513746953\",\"user_key\":\"saitoaaaaaaaa\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"XljOMaqesQ6ux\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":6800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/25\\/3496e3d762840443ccd55dd2e5809df0075f5173.gif\",\"supporter_rank\":3},{\"user_id\":119385260,\"user_name\":\"\\u30a8\\u30c9\\u592a\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1193853\\/119385260.png?1593078041\",\"user_key\":\"yugata89\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1193853\\/119385260_cover.png?1535988951\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1193853\\/119385260_cover.png?1535988951\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"A5IawMLVQ8v2o\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":5840,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/25\\/3496e3d762840443ccd55dd2e5809df0075f5173.gif\",\"supporter_rank\":4},{\"user_id\":704857697,\"user_name\":\"H a n a\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7048577\\/704857697.png?1588750687\",\"user_key\":\"H_a_n_a_______\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"jBp4GE9X5BY\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":3000,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":5},{\"user_id\":194458029,\"user_name\":\"\\u3081\\u308b\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1944581\\/194458029.png?1564990460\",\"user_key\":\"pipi_oO\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"fUi4v1klHFnlg\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2434,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":6},{\"user_id\":46792908,\"user_name\":\"\\u3074\\u3074\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/467930\\/46792908.png?1512380327\",\"user_key\":\"youatto\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"NDg1hCUn8vzpP\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2434,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":7},{\"user_id\":295982388,\"user_name\":\"\\u3053\\u3082\\u3082\\u3093\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/2959824\\/295982388.png?1535135708\",\"user_key\":\"komomo0702\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/2959824\\/295982388_cover.png?1555259860\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/2959824\\/295982388_cover.png?1555259860\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"BaXhPdbhLdz_W\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":8},{\"user_id\":4435320,\"user_name\":\"nono\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/44354\\/4435320.png?1585839429\",\"user_key\":\"game19268033\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"ENgQDYpS1iyCR\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2200,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":9},{\"user_id\":721579501,\"user_name\":\"\\u3046\\u307f\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7215796\\/721579501.png?1593699619\",\"user_key\":\"yamamiu\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"egl2RPDrqbx\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2200,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":10},{\"user_id\":3930736,\"user_name\":\"\\u305f\\u3044\\u3061\\u3083\\u3093\\u306d\\u308b\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/39308\\/3930736.png?1493878431\",\"user_key\":\"Yaritaiji\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/39308\\/3930736_202005152205.png\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/39308\\/3930736_202005152205.png\",\"user_introduce\":\"\\u30b9\\u30d7\\u30e9\\u30c8\\u30a5\\u30fc\\u30f3\\u3068\\u304b\\u30b2\\u30fc\\u30e0\\u914d\\u4fe1\\u3057\\u307e\\u3059\\n\\nhttps:\\/\\/twitter.com\\/taijich0324\",\"user_type\":1,\"identify_id\":\"o3xBdEYzl37pC\",\"followed_flg\":0,\"movie_total_views\":\"83406087\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/13 21:37:29\",\"total_yells\":1600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":11},{\"user_id\":692929798,\"user_name\":\"aoi\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6929298\\/692929798.png?1592410193\",\"user_key\":\"aoi_1005_0723\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"VbMYjJv6vbv\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":12}]}}", + "{\"status\":0,\"data\":{\"items\":[{\"user_id\":498764657,\"user_name\":\"\\u3067\\u30fc\\u308b\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/4987647\\/498764657.png?1588513790\",\"user_key\":\"chipdale33s\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"jAnaWaEJXOg\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":13},{\"user_id\":707167706,\"user_name\":\"\\u3072\\u308d\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7071678\\/707167706.png?1589063190\",\"user_key\":\"hiro_22\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"Ml3oVZy6rlD\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1480,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":14},{\"user_id\":244359955,\"user_name\":\"chiko\",\"user_icon\":null,\"user_key\":\"c_sugar5\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"9suM8ASMHRsHA\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1200,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":15},{\"user_id\":715018720,\"user_name\":\"\\u306a\\u306e\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7150188\\/715018720.png?1593430058\",\"user_key\":\"itigomusume\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"9X22kGnEAXx\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1040,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":16},{\"user_id\":700811078,\"user_name\":\"\\u307f\\u30fc\\u30fc\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7008111\\/700811078.png?1588158172\",\"user_key\":\"adgjmptw_258\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"ygG9Qn8dYbv\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1040,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":17},{\"user_id\":689988739,\"user_name\":\"\\u30de\\u30d5\\u30a3\\u30f3\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6899888\\/689988739.png?1594472502\",\"user_key\":\"wnwn_wdrdy\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"q19R7lwV9g8\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1000,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":18},{\"user_id\":694279147,\"user_name\":\"\\u3042\\u3044\\u3061\\u3083\\u3093\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6942792\\/694279147.png?1589038772\",\"user_key\":\"sasasama1112\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\\u9326\",\"user_type\":2,\"identify_id\":\"7oMWnqR9VJx\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1000,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":19},{\"user_id\":160426237,\"user_name\":\"\\u3086\\u304d\\u9178\",\"user_icon\":null,\"user_key\":\"yukisan62\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"sjobSgml8yFAU\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":20},{\"user_id\":431849798,\"user_name\":\"\\u3048\\u307f\\u308a\\u3093\\u3068\\u3059\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/4318498\\/431849798.png?1554369655\",\"user_key\":\"emirintoshhh\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"nOdm6o10DAQ\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":21},{\"user_id\":745366562,\"user_name\":\"\\u30eb\\u30ab\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7453666\\/745366562.png?1593870385\",\"user_key\":\"1988rose\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":null,\"user_type\":2,\"identify_id\":\"dgOWMJeGBg5\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":22},{\"user_id\":195436432,\"user_name\":\"\\u306a\\u3063\\u3061\\u3083\\u3093\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1954365\\/195436432.png?1525131618\",\"user_key\":\"nacchaaaan\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"k0tYrSVxKKQpp\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":23},{\"user_id\":154805243,\"user_name\":\"\\u3051\\u3044\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1548053\\/154805243.png?1550334800\",\"user_key\":\"keiko1018\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"voLRaZeS5o7we\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":24}]}}", + "{\"status\":0,\"data\":{\"items\":[{\"user_id\":142527800,\"user_name\":\"\\u308b\\u3059\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1425278\\/142527800.png?1591854905\",\"user_key\":\"rusuninaritai\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1425278\\/142527800.png?1584450166\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1425278\\/142527800.png?1584450166\",\"user_introduce\":\"[\\u5927\\u4f1a\\u5b9f\\u7e3e]\\n\\u7b2c13\\u56deglorycup \\u30d9\\u30b9\\u30c84\\n\\u7b2c2\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c84\\n\\u7b2c8\\u56de\\u30a8\\u30ea\\u30a2\\u676f\\uff0b \\u30d9\\u30b9\\u30c84\\n\\u7b2c11\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c84\\n\\u7b2c13\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c83\\n\\u7b2c14\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c16\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c84\\n\\u7b2c18\\u56de\\u30a8\\u30ea\\u30a2\\u676f\\uff0b \\u30d9\\u30b9\\u30c83\\n\\u7b2c5\\u56de\\u30ca\\u30ef\\u30d0\\u30ea\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c6\\u56de\\u30ca\\u30ef\\u30d0\\u30ea\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c8\\u56de\\u30ca\\u30ef\\u30d0\\u30ea\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c15\\u56de\\u7121\\u6575\\u7981\\u6b62\\u676f \\u512a\\u52dd\\n\\u7b2c1\\u56de\\u4e0a\\u4f4dtier\\u6b66\\u5668\\u7981\\u6b62\\u676f \\u512a\\u52dd\\n\\u30c0\\u30a4\\u30ca\\u30e2XP2974\",\"user_type\":1,\"identify_id\":\"Ntnpbs_bcU8l8\",\"followed_flg\":0,\"movie_total_views\":\"1685613\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/14 02:45:27\",\"total_yells\":640,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":25},{\"user_id\":536290679,\"user_name\":\"\\u3055\\u3093\\u3059\\u3051*\",\"user_icon\":null,\"user_key\":\"tktk-01234\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":null,\"user_type\":2,\"identify_id\":\"98aLYLD27Ab\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":480,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":26},{\"user_id\":670887609,\"user_name\":\"\\u3061\\u3043\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6708877\\/670887609.png?1583749066\",\"user_key\":\"chii_1\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/6708877\\/670887609_cover.png?1594455043\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/6708877\\/670887609_cover.png?1594455043\",\"user_introduce\":\"\\u914d\\u4fe1\\u3059\\u308b\\u304b\\u3057\\u306a\\u3044\\u304b\\u306f\\u74b0\\u5883\\u51fa\\u6765\\u3066\\u304b\\u3089\\u6c7a\\u3081\\u307e\\u3059\",\"user_type\":2,\"identify_id\":\"Ng4M4O6YNXq\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":480,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":27},{\"user_id\":3658258,\"user_name\":\"\\u3072\\u3044\\u3089\\u304e\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/36583\\/3658258.png?1591548658\",\"user_key\":\"hiragispla\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/36583\\/3658258_cover.png?1591547045\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/36583\\/3658258_cover.png?1591547045\",\"user_introduce\":\"Twitter\\u2192https:\\/\\/twitter.com\\/hiragi0223spla2\",\"user_type\":1,\"identify_id\":\"q_nlRPYAl4NOQ\",\"followed_flg\":0,\"movie_total_views\":\"997973\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/13 18:16:58\",\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":28},{\"user_id\":113903074,\"user_name\":\"\\u9060\\u85e4\\u3061\\u304b\\u3057\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1139031\\/113903074.png?1593980363\",\"user_key\":\"yutorin_916\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1139031\\/113903074_cover.png?1590494029\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1139031\\/113903074_cover.png?1590494029\",\"user_introduce\":\"Twitter @splar_ \\n\\u30a4\\u30f3\\u30b9\\u30bf aimyoncamper\\n\\u30e8\\u30ed\\u30b7\\u30af\\u30cd\",\"user_type\":1,\"identify_id\":\"h7yjbZhS4VIsb\",\"followed_flg\":0,\"movie_total_views\":\"244973\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/13 15:04:01\",\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":29},{\"user_id\":641638275,\"user_name\":\"\\u3059\\u30fc\\u3061\\u3083\\u3093\",\"user_icon\":null,\"user_key\":\"suchan_\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"yOorwM5YrBE\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":30},{\"user_id\":110696888,\"user_name\":\"PANKO\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1106969\\/110696888.png?1587648021\",\"user_key\":\"sakupan06\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"hMYG9tGa0o1JG\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":31},{\"user_id\":676511312,\"user_name\":\"\\u307b \\u306a\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6765114\\/676511312.png?1584621136\",\"user_key\":\"hona_126\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"4Xr22JKr2Xp\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":32}]}}" + ] +} \ No newline at end of file