TODO: 解决编码问题
This commit is contained in:
eson
parent
ad02a5b4d9
commit
0d8b456f41
|
|
@ -1,3 +1,3 @@
|
|||
database:
|
||||
source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source"
|
||||
extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor"
|
||||
source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true"
|
||||
extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true"
|
||||
|
|
@ -1,73 +1,219 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"net/http"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
"github.com/474420502/requests"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
func preNUm(data byte) int {
|
||||
var mask byte = 0x80
|
||||
var num int = 0
|
||||
//8bit中首个0bit前有多少个1bits
|
||||
for i := 0; i < 8; i++ {
|
||||
if (data & mask) == mask {
|
||||
num++
|
||||
mask = mask >> 1
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return num
|
||||
}
|
||||
func isUtf8(data []byte) bool {
|
||||
i := 0
|
||||
for i < len(data) {
|
||||
if (data[i] & 0x80) == 0x00 {
|
||||
// 0XXX_XXXX
|
||||
i++
|
||||
continue
|
||||
} else if num := preNUm(data[i]); num > 2 {
|
||||
// 110X_XXXX 10XX_XXXX
|
||||
// 1110_XXXX 10XX_XXXX 10XX_XXXX
|
||||
// 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
|
||||
// 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
|
||||
// 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
|
||||
// preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数
|
||||
i++
|
||||
for j := 0; j < num-1; j++ {
|
||||
//判断后面的 num - 1 个字节是不是都是10开头
|
||||
if (data[i] & 0xc0) != 0x80 {
|
||||
return false
|
||||
}
|
||||
i++
|
||||
}
|
||||
} else {
|
||||
//其他情况说明不是utf-8
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isGBK(data []byte) bool {
|
||||
length := len(data)
|
||||
var i int = 0
|
||||
for i < length {
|
||||
if data[i] <= 0x7f {
|
||||
//编码0~127,只有一个字节的编码，兼容ASCII码
|
||||
i++
|
||||
continue
|
||||
} else {
|
||||
//大于127的使用双字节编码，落在gbk编码范围内的字符
|
||||
if data[i] >= 0x81 &&
|
||||
data[i] <= 0xfe &&
|
||||
data[i+1] >= 0x40 &&
|
||||
data[i+1] <= 0xfe &&
|
||||
data[i+1] != 0xf7 {
|
||||
i += 2
|
||||
continue
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestExtractor(t *testing.T) {
|
||||
|
||||
ses := requests.NewSession()
|
||||
wf := ses.Get("https://www.openrec.tv/user/Riowh/supporters")
|
||||
wf.Execute()
|
||||
tp := ses.Get("https://www.openrec.tv/user/Riowh/supporters")
|
||||
tp.Execute()
|
||||
|
||||
// t.Error(ses.GetCookies(wf.GetParsedURL()))
|
||||
|
||||
buf := bytes.Buffer{}
|
||||
encoder := gob.NewEncoder(&buf)
|
||||
encoder.Encode(ses.GetCookies(wf.GetParsedURL()))
|
||||
|
||||
var cookies []*http.Cookie
|
||||
decoder := gob.NewDecoder(&buf)
|
||||
t.Error(decoder.Decode(&cookies))
|
||||
t.Error(cookies)
|
||||
|
||||
// collect := intimate.NewExtractorStore()
|
||||
// store := intimate.NewSourceStore("source_openrec")
|
||||
// source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
|
||||
collect := intimate.NewExtractorStore()
|
||||
store := intimate.NewSourceStore("source_openrec")
|
||||
source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
|
||||
// if source != nil {
|
||||
// defer store.Restore(source)
|
||||
// }
|
||||
// if err != nil {
|
||||
// t.Error(err)
|
||||
// }
|
||||
// sdata := source.GetExt().([]byte)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
sdata := source.GetExt().([]byte)
|
||||
|
||||
// if gjson.ValidBytes(sdata) {
|
||||
// result := gjson.ParseBytes(sdata)
|
||||
// m := result.Map()
|
||||
if gjson.ValidBytes(sdata) {
|
||||
result := gjson.ParseBytes(sdata)
|
||||
m := result.Map()
|
||||
|
||||
// user := m["user"]
|
||||
user := m["user"]
|
||||
|
||||
// ai := &intimate.CollectLog{}
|
||||
// extractor := hunter.NewExtractor([]byte(user.Str))
|
||||
// xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
|
||||
// if err != nil {
|
||||
// t.Error(err)
|
||||
// }
|
||||
// if !xp.NodeIter().Next() {
|
||||
// t.Error("不存在粉丝数")
|
||||
// }
|
||||
ai := &intimate.CollectLog{}
|
||||
extractor := hunter.NewExtractor([]byte(user.Str))
|
||||
xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !xp.NodeIter().Next() {
|
||||
t.Error("不存在粉丝数")
|
||||
}
|
||||
|
||||
// followers := strings.ReplaceAll(xp.String(), ",", "")
|
||||
followers := strings.ReplaceAll(xp.String(), ",", "")
|
||||
|
||||
// followersInt, err := strconv.Atoi(followers)
|
||||
// if err != nil {
|
||||
// t.Error(err)
|
||||
// }
|
||||
followersInt, err := strconv.ParseInt(followers, 10, 64)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// ai.SetPlatform(string(intimate.Popenrec))
|
||||
// ai.SetFollowers(sql.NullInt32{Int32: int32(followersInt), Valid: true})
|
||||
// ai.SetAnchorId(source.GetSource().String)
|
||||
var anchorName string
|
||||
xp, err = extractor.XPathResult("//p[@class='c-global__user__profile__list__name__text official-icon--after']/text()")
|
||||
if xp.NodeIter().Next() {
|
||||
anchorName = xp.String()
|
||||
} else {
|
||||
t.Error(err)
|
||||
}
|
||||
t.Error(anchorName)
|
||||
|
||||
// collect.InsertCollectLog(ai)
|
||||
// c-contents
|
||||
xp, err = extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if xp.NodeIter().Next() {
|
||||
views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String())
|
||||
views = strings.ReplaceAll(views, ",", "")
|
||||
viewsint, err := strconv.Atoi(views)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// } else {
|
||||
// t.Error("data is not json:\n", string(sdata))
|
||||
// }
|
||||
ai.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true})
|
||||
ai.SetIsShowing(1)
|
||||
}
|
||||
|
||||
var givers []interface{}
|
||||
var gratuity int64 = 0
|
||||
giverjson := m["supporters"]
|
||||
for _, v := range giverjson.Array() {
|
||||
giverSource := gjson.Parse(v.String())
|
||||
for _, item := range giverSource.Get("data.items").Array() {
|
||||
givers = append(givers, item.Map())
|
||||
gratuity += item.Get("total_yells").Int()
|
||||
}
|
||||
}
|
||||
|
||||
giversbytes, err := json.Marshal(givers)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
ai.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
} else {
|
||||
ai.SetGiver(giversbytes)
|
||||
}
|
||||
|
||||
// MovieToolbar__Views-g5e6ic-13 iDRGyA
|
||||
livejson := m["user_live"]
|
||||
|
||||
f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
f.WriteString(livejson.String())
|
||||
t.Error(livejson)
|
||||
extractor = hunter.NewExtractor([]byte(livejson.Str))
|
||||
xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]")
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
t.Error(xr)
|
||||
iter := xr.NodeIter()
|
||||
if iter.Next() {
|
||||
t.Error(iter.Node().TextContent())
|
||||
ai.SetShowTitle(sql.NullString{String: iter.Node().TextContent(), Valid: true})
|
||||
|
||||
content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if content.NodeIter().Next() {
|
||||
t.Error(content.String())
|
||||
}
|
||||
}
|
||||
t.Error(xr.String(), xr.NodeIter().Next(), xr.String())
|
||||
|
||||
ai.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true})
|
||||
ai.SetPlatform(string(intimate.Popenrec))
|
||||
ai.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true})
|
||||
ai.SetAnchorId(source.GetSource().String)
|
||||
ai.SetUpdateTime(source.GetUpdateTime())
|
||||
|
||||
collect.InsertCollectLog(ai)
|
||||
|
||||
} else {
|
||||
t.Error("data is not json:\n", string(sdata))
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -159,14 +159,14 @@ type IGetCollectLog interface {
|
|||
GetAnchorId() string //
|
||||
GetIsShowing() int32 //
|
||||
GetIsError() int32 //
|
||||
GetFollowers() sql.NullInt32 //
|
||||
GetViews() sql.NullInt32 //
|
||||
GetFollowers() sql.NullInt64 //
|
||||
GetViews() sql.NullInt64 //
|
||||
GetGiver() interface{} //
|
||||
GetGratuity() sql.NullInt32 //
|
||||
GetGratuity() sql.NullInt64 //
|
||||
GetShowTitle() sql.NullString //
|
||||
GetShowStartTime() sql.NullTime //
|
||||
GetShowEndTime() sql.NullTime //
|
||||
GetUpdateTime() time.Time //
|
||||
GetUpdateTime() sql.NullTime //
|
||||
GetExt() interface{} //
|
||||
GetErrorMsg() sql.NullString //
|
||||
}
|
||||
|
|
@ -177,14 +177,14 @@ type ISetCollectLog interface {
|
|||
SetAnchorId(string) //
|
||||
SetIsShowing(int32) //
|
||||
SetIsError(int32) //
|
||||
SetFollowers(sql.NullInt32) //
|
||||
SetViews(sql.NullInt32) //
|
||||
SetFollowers(sql.NullInt64) //
|
||||
SetViews(sql.NullInt64) //
|
||||
SetGiver(interface{}) //
|
||||
SetGratuity(sql.NullInt32) //
|
||||
SetGratuity(sql.NullInt64) //
|
||||
SetShowTitle(sql.NullString) //
|
||||
SetShowStartTime(sql.NullTime) //
|
||||
SetShowEndTime(sql.NullTime) //
|
||||
SetUpdateTime(time.Time) //
|
||||
SetUpdateTime(sql.NullTime) //
|
||||
SetExt(interface{}) //
|
||||
SetErrorMsg(sql.NullString) //
|
||||
}
|
||||
|
|
@ -229,14 +229,14 @@ type CollectLog struct {
|
|||
AnchorId string //
|
||||
IsShowing int32 //
|
||||
IsError int32 //
|
||||
Followers sql.NullInt32 //
|
||||
Views sql.NullInt32 //
|
||||
Followers sql.NullInt64 //
|
||||
Views sql.NullInt64 //
|
||||
Giver interface{} //
|
||||
Gratuity sql.NullInt32 //
|
||||
Gratuity sql.NullInt64 //
|
||||
ShowTitle sql.NullString //
|
||||
ShowStartTime sql.NullTime //
|
||||
ShowEndTime sql.NullTime //
|
||||
UpdateTime time.Time //
|
||||
UpdateTime sql.NullTime //
|
||||
Ext interface{} //
|
||||
ErrorMsg sql.NullString //
|
||||
}
|
||||
|
|
@ -262,12 +262,12 @@ func (cl *CollectLog) SetExt(Ext interface{}) {
|
|||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (cl *CollectLog) GetUpdateTime() time.Time {
|
||||
func (cl *CollectLog) GetUpdateTime() sql.NullTime {
|
||||
return cl.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (cl *CollectLog) SetUpdateTime(UpdateTime time.Time) {
|
||||
func (cl *CollectLog) SetUpdateTime(UpdateTime sql.NullTime) {
|
||||
cl.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
|
|
@ -302,12 +302,12 @@ func (cl *CollectLog) SetShowTitle(ShowTitle sql.NullString) {
|
|||
}
|
||||
|
||||
// GetGratuity Get return Gratuity sql.NullInt32
|
||||
func (cl *CollectLog) GetGratuity() sql.NullInt32 {
|
||||
func (cl *CollectLog) GetGratuity() sql.NullInt64 {
|
||||
return cl.Gratuity
|
||||
}
|
||||
|
||||
// SetGratuity Set Gratuity sql.NullInt32
|
||||
func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt32) {
|
||||
func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt64) {
|
||||
cl.Gratuity = Gratuity
|
||||
}
|
||||
|
||||
|
|
@ -321,23 +321,23 @@ func (cl *CollectLog) SetGiver(Giver interface{}) {
|
|||
cl.Giver = Giver
|
||||
}
|
||||
|
||||
// GetViews Get return Views sql.NullInt32
|
||||
func (cl *CollectLog) GetViews() sql.NullInt32 {
|
||||
// GetViews Get return Views sql.NullInt64
|
||||
func (cl *CollectLog) GetViews() sql.NullInt64 {
|
||||
return cl.Views
|
||||
}
|
||||
|
||||
// SetViews Set Views sql.NullInt32
|
||||
func (cl *CollectLog) SetViews(Views sql.NullInt32) {
|
||||
// SetViews Set Views sql.NullInt64
|
||||
func (cl *CollectLog) SetViews(Views sql.NullInt64) {
|
||||
cl.Views = Views
|
||||
}
|
||||
|
||||
// GetFollowers Get return Followers sql.NullInt32
|
||||
func (cl *CollectLog) GetFollowers() sql.NullInt32 {
|
||||
// GetFollowers Get return Followers sql.NullInt64
|
||||
func (cl *CollectLog) GetFollowers() sql.NullInt64 {
|
||||
return cl.Followers
|
||||
}
|
||||
|
||||
// SetFollowers Set Followers sql.NullInt32
|
||||
func (cl *CollectLog) SetFollowers(Followers sql.NullInt32) {
|
||||
func (cl *CollectLog) SetFollowers(Followers sql.NullInt64) {
|
||||
cl.Followers = Followers
|
||||
}
|
||||
|
||||
|
|
|
|||
13
go.mod
13
go.mod
|
|
@ -3,15 +3,12 @@ module intimate
|
|||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/474420502/gcurl v0.0.4
|
||||
github.com/474420502/hunter v0.1.2
|
||||
github.com/474420502/requests v1.5.1
|
||||
github.com/474420502/gcurl v0.1.2
|
||||
github.com/474420502/hunter v0.3.0
|
||||
github.com/474420502/requests v1.6.0
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible // indirect
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/tidwall/gjson v1.6.0
|
||||
github.com/tidwall/pretty v1.0.1 // indirect
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
)
|
||||
|
|
|
|||
59
go.sum
59
go.sum
|
|
@ -2,21 +2,29 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
|||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
|
||||
github.com/474420502/focus v0.9.0 h1:Y/TjSdSdIgegO78OJixphiPl1wVqhK9AbcXjiDDugo4=
|
||||
github.com/474420502/focus v0.9.0/go.mod h1:jrDXvK1CnUJ3PCR3ZJVYinbS2Yz5kM8OoAbCLe6AF7Y=
|
||||
github.com/474420502/gcurl v0.0.4 h1:eR1BNXvQ4T245dotWpjDzAMWch+FTTfScqzsdq93JK0=
|
||||
github.com/474420502/gcurl v0.0.4/go.mod h1:qtCzAZZbVRIsBt0lNUh2I0qDniU9T3E21aSsVUYo7Hc=
|
||||
github.com/474420502/hunter v0.1.2 h1:1dY2C9IJvh81+04Xx8OuTPcUw6WNlKCiOaGmJdgX83Y=
|
||||
github.com/474420502/hunter v0.1.2/go.mod h1:41DpZWSsGBWsFwb/liapbT1uH58Yvl+BpW4SJwLC2Fw=
|
||||
github.com/474420502/requests v1.5.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/474420502/requests v1.5.1 h1:miv6O4RMbZ8I0ZdUTLf/EU5Dmewc/4IL/DmUMwtuv8M=
|
||||
github.com/474420502/requests v1.5.1/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo=
|
||||
github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
|
||||
github.com/474420502/gcurl v0.1.0 h1:aKPP27qAnofntTqqVF8rjejHBWVlYWzFEZGdqjBiMgw=
|
||||
github.com/474420502/gcurl v0.1.0/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM=
|
||||
github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg=
|
||||
github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM=
|
||||
github.com/474420502/hunter v0.2.0 h1:pH7xIWzas2IuLdCJL/HtcouHcZQq0XIP/FswY7yF+pA=
|
||||
github.com/474420502/hunter v0.2.0/go.mod h1:c1+92qUtjNzYw6Mzl6Qkb2kMALAXMlYQk3kJdoOqnmY=
|
||||
github.com/474420502/hunter v0.3.0 h1:0VPi1MInxjHOta3da4v0ALWK0y3/X4/6nUSLFvdbiFU=
|
||||
github.com/474420502/hunter v0.3.0/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA=
|
||||
github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY=
|
||||
github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA=
|
||||
github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k=
|
||||
github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP3VrhEcx8=
|
||||
github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
|
||||
github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
|
||||
github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
|
|
@ -24,12 +32,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY=
|
||||
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
|
|
@ -39,6 +48,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
|
|||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
|
|
@ -54,11 +64,11 @@ github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb h1:qqNmX9V9n4b
|
|||
github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb/go.mod h1:fy/ZVbgyB83mtricxwSW3zqIRXWOVpKG2PvdUDFeC58=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w=
|
||||
github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc=
|
||||
|
|
@ -71,12 +81,10 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
|||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8=
|
||||
github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -87,7 +95,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
|
|||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -100,8 +107,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
|
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -110,7 +117,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -122,6 +128,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -136,12 +143,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
|
|||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJYds2YJwZRNw4zs7wSnNhk=
|
||||
golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882 h1:x4Two2lSwHxTqR+eal4lB4ydUnTvmDDpPQeL92ZHDgA=
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
|
|
@ -159,10 +160,18 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
|
|||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7 h1:zibSPXbkfB1Dwl76rJgLa68xcdHu42qmFTe6vAnU4wA=
|
||||
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7/go.mod h1:wo0SW5T6XqIKCCAge330Cd5sm+7VI6v85OrQHIk50KM=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54=
|
||||
launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
|
||||
launchpad.net/xmlpath v0.0.0-20130614043138-000000000004 h1:B8nNZBUrx8YufDCAJjvO/lVs4GxXMQHyrjwJdJzXMFg=
|
||||
launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package intimate
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IGetSource 源接口结构
|
||||
|
|
@ -13,7 +12,7 @@ type IGetSource interface {
|
|||
GetSource() sql.NullString //
|
||||
GetPassGob() sql.NullString //
|
||||
GetExt() interface{} //
|
||||
GetUpdateTime() time.Time //
|
||||
GetUpdateTime() sql.NullTime //
|
||||
GetOperator() int32 //
|
||||
GetErrorMsg() sql.NullString //
|
||||
}
|
||||
|
|
@ -25,7 +24,7 @@ type IUpdateSource interface {
|
|||
|
||||
SetPassGob(sql.NullString)
|
||||
SetExt(ext interface{}) //
|
||||
SetUpdateTime(ut time.Time) //
|
||||
SetUpdateTime(ut sql.NullTime) //
|
||||
SetOperator(operator int32) //
|
||||
SetErrorMsg(emsg sql.NullString) //
|
||||
}
|
||||
|
|
@ -38,7 +37,7 @@ type Source struct {
|
|||
Source sql.NullString //
|
||||
PassGob sql.NullString //
|
||||
Ext interface{} //
|
||||
UpdateTime time.Time //
|
||||
UpdateTime sql.NullTime //
|
||||
Operator int32 //
|
||||
ErrorMsg sql.NullString //
|
||||
|
||||
|
|
@ -86,12 +85,12 @@ func (so *Source) SetOperator(Operator int32) {
|
|||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (so *Source) GetUpdateTime() time.Time {
|
||||
func (so *Source) GetUpdateTime() sql.NullTime {
|
||||
return so.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (so *Source) SetUpdateTime(UpdateTime time.Time) {
|
||||
func (so *Source) SetUpdateTime(UpdateTime sql.NullTime) {
|
||||
so.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,10 +29,10 @@ CREATE TABLE IF NOT EXISTS `collect_log` (
|
|||
`is_showing` tinyint(1) DEFAULT 0,
|
||||
`is_error` tinyint(1) DEFAULT 0,
|
||||
|
||||
`followers` int(11) DEFAULT NULL,
|
||||
`views` int(11) DEFAULT NULL,
|
||||
`followers` bigint(11) DEFAULT NULL,
|
||||
`views` bigint(11) DEFAULT NULL,
|
||||
`giver` json DEFAULT NULL,
|
||||
`gratuity` int(11) DEFAULT NULL,
|
||||
`gratuity` bigint(11) DEFAULT NULL,
|
||||
|
||||
`show_title` text DEFAULT NULL,
|
||||
`show_start_time` timestamp NULL DEFAULT NULL,
|
||||
|
|
|
|||
6
store.go
6
store.go
|
|
@ -87,7 +87,7 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou
|
|||
return nil, err
|
||||
}
|
||||
var args = []interface{}{targetType}
|
||||
selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?`
|
||||
selectSQL := `select uid, url, target_type, source, ext, operator, update_time from ` + store.table + ` where target_type = ?`
|
||||
if len(operators) == 0 {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, 0)
|
||||
|
|
@ -116,12 +116,12 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou
|
|||
s := &Source{}
|
||||
|
||||
// uid, url, target_type, source, ext, operator
|
||||
err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator)
|
||||
err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime)
|
||||
s.SetLastOperator(s.Operator)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err, targetType)
|
||||
_, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid)
|
||||
_, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", err.Error(), OperatorError, s.Uid)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) {
|
|||
t.Error("rank is error. result raw is ", result.Raw)
|
||||
}
|
||||
|
||||
if cxt.Workflow().GetQuery().Get("page") != "1" {
|
||||
t.Error("workflow page error")
|
||||
if cxt.Temporary().GetQuery().Get("page") != "1" {
|
||||
t.Error("Temporary page error")
|
||||
}
|
||||
// t.Error(string(resp.Content()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
|||
break
|
||||
}
|
||||
|
||||
wf := cxt.Workflow()
|
||||
wf := cxt.Temporary()
|
||||
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 {
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
|
@ -45,7 +48,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
|||
|
||||
wf := cxt.Session().Get(userUrl)
|
||||
resp, err := wf.Execute()
|
||||
source.SetUpdateTime(time.Now())
|
||||
source.SetUpdateTime(sql.NullTime{Time: time.Now(), Valid: true})
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
|
|
@ -55,16 +58,16 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
|||
|
||||
cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
|
||||
scurl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
|
||||
scurl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
|
||||
curl := gcurl.ParseRawCURL(scurl)
|
||||
supportersSession := curl.CreateSession()
|
||||
|
||||
log.Println(curl.ParsedURL)
|
||||
workflow := curl.CreateWorkflow(supportersSession)
|
||||
supportersSession.SetCookies(workflow.GetParsedURL(), cookies)
|
||||
temporary := curl.CreateTemporary(supportersSession)
|
||||
supportersSession.SetCookies(temporary.GetParsedURL(), cookies)
|
||||
var supporters []string
|
||||
for {
|
||||
|
||||
supportersQuery := workflow.GetQuery()
|
||||
supportersQuery := temporary.GetQuery()
|
||||
|
||||
for _, cookie := range cookies {
|
||||
if cookie.Name == "uuid" {
|
||||
|
|
@ -84,34 +87,37 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
|||
}
|
||||
|
||||
supportersQuery.Set("identify_id", source.GetSource().String)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
|
||||
workflow.SetQuery(supportersQuery)
|
||||
// workflow.AddCookies(cookies)
|
||||
log.Println(workflow.GetRawURL())
|
||||
log.Println(supportersSession.GetCookies(workflow.GetParsedURL()))
|
||||
resp, err := workflow.Execute()
|
||||
resp, err := temporary.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
log.Println(string(resp.Content()))
|
||||
}
|
||||
return
|
||||
}
|
||||
// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
supporterjson := gjson.ParseBytes(resp.Content())
|
||||
supporterdata := supporterjson.Get("data")
|
||||
if supporterdata.Type == gjson.Null {
|
||||
break
|
||||
}
|
||||
supporters = append(supporters, string(resp.Content()))
|
||||
|
||||
page := supportersQuery.Get("page_number")
|
||||
pageint, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
pageint++
|
||||
page = strconv.Itoa(pageint)
|
||||
supportersQuery.Set("page_number", page)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
}
|
||||
|
||||
// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
ext := make(map[string]interface{})
|
||||
|
||||
ext["supporters"] = supporters
|
||||
ext["user"] = string(resp.Content())
|
||||
|
||||
// wf = cxt.Session().Get(userUrl + "/supporters")
|
||||
// resp, err = wf.Execute()
|
||||
// if err != nil {
|
||||
// log.Println(err)
|
||||
// store.UpdateError(source, err)
|
||||
// continue
|
||||
// }
|
||||
// ext["user_supporters"] = string(resp.Content())
|
||||
|
||||
wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid)
|
||||
resp, err = wf.Execute()
|
||||
if err != nil {
|
||||
|
|
@ -128,15 +134,9 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
|||
continue
|
||||
}
|
||||
|
||||
// buf := bytes.Buffer{}
|
||||
// encoder := gob.NewEncoder(&buf)
|
||||
// encoder.Encode(cookies)
|
||||
|
||||
// source.SetPassGob(sql.NullString{String: buf.String(), Valid: true})
|
||||
source.SetOperator(int32(intimate.OperatorOK))
|
||||
source.SetExt(string(extJsonBytes))
|
||||
store.Update(source)
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
7
testfile/openrec_user_supporters.json
Executable file
7
testfile/openrec_user_supporters.json
Executable file
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user