add: Pop 队列. 以mysql 表的标志做一个队列.
This commit is contained in:
eson
parent
0b5425088f
commit
5cbb17d6d4
1
go.mod
1
go.mod
|
|
@ -9,6 +9,7 @@ require (
|
|||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/tidwall/gjson v1.6.0
|
||||
github.com/tidwall/pretty v1.0.1 // indirect
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a
|
||||
)
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -71,10 +71,13 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
|||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8=
|
||||
github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
|
|
@ -84,6 +87,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
|
|||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -96,6 +100,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
|
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -104,6 +110,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -127,7 +134,14 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
|
|||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJYds2YJwZRNw4zs7wSnNhk=
|
||||
golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882 h1:x4Two2lSwHxTqR+eal4lB4ydUnTvmDDpPQeL92ZHDgA=
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
|
|
|
|||
98
source.go
Normal file
98
source.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package intimate
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Source 的结构体
|
||||
type Source struct {
|
||||
Uid sql.NullInt64 //
|
||||
Url string //
|
||||
TargetType string //
|
||||
Source sql.NullString //
|
||||
Ext interface{} //
|
||||
UpdateTime time.Time //
|
||||
Operator int32 //
|
||||
ErrorMsg sql.NullString //
|
||||
}
|
||||
|
||||
// GetErrorMsg Get return ErrorMsg sql.NullString
|
||||
func (so *Source) GetErrorMsg() sql.NullString {
|
||||
return so.ErrorMsg
|
||||
}
|
||||
|
||||
// SetErrorMsg Set ErrorMsg sql.NullString
|
||||
func (so *Source) SetErrorMsg(ErrorMsg sql.NullString) {
|
||||
so.ErrorMsg = ErrorMsg
|
||||
}
|
||||
|
||||
// GetOperator Get return Operator sql.NullInt32
|
||||
func (so *Source) GetOperator() int32 {
|
||||
return so.Operator
|
||||
}
|
||||
|
||||
// SetOperator Set Operator sql.NullInt32
|
||||
func (so *Source) SetOperator(Operator int32) {
|
||||
so.Operator = Operator
|
||||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (so *Source) GetUpdateTime() time.Time {
|
||||
return so.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (so *Source) SetUpdateTime(UpdateTime time.Time) {
|
||||
so.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
// GetExt Get return Ext interface{}
|
||||
func (so *Source) GetExt() interface{} {
|
||||
return so.Ext
|
||||
}
|
||||
|
||||
// SetExt Set Ext interface{}
|
||||
func (so *Source) SetExt(Ext interface{}) {
|
||||
so.Ext = Ext
|
||||
}
|
||||
|
||||
// GetSource Get return Source sql.NullString
|
||||
func (so *Source) GetSource() sql.NullString {
|
||||
return so.Source
|
||||
}
|
||||
|
||||
// SetSource Set Source sql.NullString
|
||||
func (so *Source) SetSource(Source sql.NullString) {
|
||||
so.Source = Source
|
||||
}
|
||||
|
||||
// GetTargetType Get return TargetType string
|
||||
func (so *Source) GetTargetType() string {
|
||||
return so.TargetType
|
||||
}
|
||||
|
||||
// SetTargetType Set TargetType string
|
||||
func (so *Source) SetTargetType(TargetType string) {
|
||||
so.TargetType = TargetType
|
||||
}
|
||||
|
||||
// GetUrl Get return Url string
|
||||
func (so *Source) GetUrl() string {
|
||||
return so.Url
|
||||
}
|
||||
|
||||
// SetUrl Set Url string
|
||||
func (so *Source) SetUrl(Url string) {
|
||||
so.Url = Url
|
||||
}
|
||||
|
||||
// GetUid Get return Uid sql.NullInt64
|
||||
func (so *Source) GetUid() sql.NullInt64 {
|
||||
return so.Uid
|
||||
}
|
||||
|
||||
// SetUid Set Uid sql.NullInt64
|
||||
func (so *Source) SetUid(Uid sql.NullInt64) {
|
||||
so.Uid = Uid
|
||||
}
|
||||
|
|
@ -17,7 +17,6 @@ CREATE TABLE IF NOT EXISTS `anchor_info` (
|
|||
KEY `channel_idx` (`channel`),
|
||||
KEY `show_type_idx` (`show_type`),
|
||||
KEY `update_time_idx` (`update_time`)
|
||||
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `show_log` (
|
||||
|
|
|
|||
89
store.go
89
store.go
|
|
@ -2,14 +2,15 @@ package intimate
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// ISource 源接口结构
|
||||
type ISource interface {
|
||||
// IGetSource 源接口结构
|
||||
type IGetSource interface {
|
||||
GetUrl() string //
|
||||
GetTargetType() string //
|
||||
GetSource() sql.NullString //
|
||||
|
|
@ -17,27 +18,99 @@ type ISource interface {
|
|||
GetUpdateTime() time.Time //
|
||||
GetOperator() int32 //
|
||||
GetErrorMsg() sql.NullString //
|
||||
|
||||
}
|
||||
|
||||
type IUpdateSource interface {
|
||||
IGetSource
|
||||
|
||||
SetExt(ext interface{}) //
|
||||
SetUpdateTime(ut time.Time) //
|
||||
SetOperator(operator int32) //
|
||||
SetErrorMsg(emsg sql.NullString) //
|
||||
}
|
||||
|
||||
// OperatorFlag 标志
|
||||
type OperatorFlag int32
|
||||
|
||||
const (
|
||||
// OperatorWait 等待被处理
|
||||
OperatorWait OperatorFlag = 1000
|
||||
// OperatorError 错误标志
|
||||
OperatorError OperatorFlag = 10000
|
||||
)
|
||||
|
||||
// Store 储存
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
table string
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewStore 创建一个存储实例
|
||||
func NewStore() *Store {
|
||||
func NewStore(table string) *Store {
|
||||
db, err := sql.Open("mysql", InitConfig.Database.URI)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Store{db: db}
|
||||
return &Store{table: table, db: db}
|
||||
}
|
||||
|
||||
// Save 储存数据
|
||||
func (store *Store) Save(isource ISource) {
|
||||
_, err := store.db.Exec("insert into `platform_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg())
|
||||
func (store *Store) Save(isource IGetSource) {
|
||||
_, err := store.db.Exec("insert into `source_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg())
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Pop 储存数据
|
||||
func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, error) {
|
||||
|
||||
tx, err := store.db.Begin()
|
||||
if err != nil {
|
||||
log.Println(err, targetType)
|
||||
return nil, err
|
||||
}
|
||||
var args = []interface{}{targetType}
|
||||
selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?`
|
||||
if len(operators) == 0 {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, 0)
|
||||
} else {
|
||||
for _, operator := range operators {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, operator)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println(selectSQL + ` limit 1 for update`)
|
||||
row := tx.QueryRow(selectSQL, args...)
|
||||
|
||||
defer func() {
|
||||
err := tx.Commit()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if row != nil {
|
||||
s := &Source{}
|
||||
// uid, url, target_type, source, ext, operator
|
||||
err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator)
|
||||
if err != nil {
|
||||
log.Println(err, targetType)
|
||||
_, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
return nil, errors.New("TaskQueue is nil")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,3 +22,8 @@ func TestStoreInsertCase1(t *testing.T) {
|
|||
// t.Error(err, len(uuid.NewV4().String()))
|
||||
|
||||
}
|
||||
|
||||
func TestStorePopCase1(t *testing.T) {
|
||||
store := NewStore("source_openrec")
|
||||
t.Error(store.Pop("openrec_ranking"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,5 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"intimate"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
var targetTypeRanking = "openrec_ranking"
|
||||
var openrecRanking *OpenrecRanking
|
||||
|
||||
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var store *intimate.Store = intimate.NewStore()
|
||||
|
||||
func init() {
|
||||
|
||||
openrecRanking = &OpenrecRanking{}
|
||||
openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
-H 'authority: public.openrec.tv' \
|
||||
-H 'accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
|
||||
-H 'origin: https://www.openrec.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.openrec.tv/ranking' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
|
||||
--compressed`
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
CREATE TABLE `source_openrec` (
|
||||
uid bigint AUTO_INCREMENT,
|
||||
|
|
@ -52,147 +17,3 @@ CREATE TABLE `source_openrec` (
|
|||
KEY `target_type_idx` (`target_type`)
|
||||
);
|
||||
*/
|
||||
|
||||
type SourceOpenrec struct {
|
||||
Uid sql.NullInt64 //
|
||||
Url string //
|
||||
TargetType string //
|
||||
Source sql.NullString //
|
||||
Ext interface{} //
|
||||
UpdateTime time.Time //
|
||||
Operator int32 //
|
||||
ErrorMsg sql.NullString //
|
||||
}
|
||||
|
||||
// GetErrorMsg Get return ErrorMsg sql.NullString
|
||||
func (so *SourceOpenrec) GetErrorMsg() sql.NullString {
|
||||
return so.ErrorMsg
|
||||
}
|
||||
|
||||
// SetErrorMsg Set ErrorMsg sql.NullString
|
||||
func (so *SourceOpenrec) SetErrorMsg(ErrorMsg sql.NullString) {
|
||||
so.ErrorMsg = ErrorMsg
|
||||
}
|
||||
|
||||
// GetOperator Get return Operator sql.NullInt32
|
||||
func (so *SourceOpenrec) GetOperator() int32 {
|
||||
return so.Operator
|
||||
}
|
||||
|
||||
// SetOperator Set Operator sql.NullInt32
|
||||
func (so *SourceOpenrec) SetOperator(Operator int32) {
|
||||
so.Operator = Operator
|
||||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (so *SourceOpenrec) GetUpdateTime() time.Time {
|
||||
return so.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (so *SourceOpenrec) SetUpdateTime(UpdateTime time.Time) {
|
||||
so.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
// GetExt Get return Ext interface{}
|
||||
func (so *SourceOpenrec) GetExt() interface{} {
|
||||
return so.Ext
|
||||
}
|
||||
|
||||
// SetExt Set Ext interface{}
|
||||
func (so *SourceOpenrec) SetExt(Ext interface{}) {
|
||||
so.Ext = Ext
|
||||
}
|
||||
|
||||
// GetSource Get return Source sql.NullString
|
||||
func (so *SourceOpenrec) GetSource() sql.NullString {
|
||||
return so.Source
|
||||
}
|
||||
|
||||
// SetSource Set Source sql.NullString
|
||||
func (so *SourceOpenrec) SetSource(Source sql.NullString) {
|
||||
so.Source = Source
|
||||
}
|
||||
|
||||
// GetTargetType Get return TargetType string
|
||||
func (so *SourceOpenrec) GetTargetType() string {
|
||||
return so.TargetType
|
||||
}
|
||||
|
||||
// SetTargetType Set TargetType string
|
||||
func (so *SourceOpenrec) SetTargetType(TargetType string) {
|
||||
so.TargetType = TargetType
|
||||
}
|
||||
|
||||
// GetUrl Get return Url string
|
||||
func (so *SourceOpenrec) GetUrl() string {
|
||||
return so.Url
|
||||
}
|
||||
|
||||
// SetUrl Set Url string
|
||||
func (so *SourceOpenrec) SetUrl(Url string) {
|
||||
so.Url = Url
|
||||
}
|
||||
|
||||
// GetUid Get return Uid sql.NullInt64
|
||||
func (so *SourceOpenrec) GetUid() sql.NullInt64 {
|
||||
return so.Uid
|
||||
}
|
||||
|
||||
// SetUid Set Uid sql.NullInt64
|
||||
func (so *SourceOpenrec) SetUid(Uid sql.NullInt64) {
|
||||
so.Uid = Uid
|
||||
}
|
||||
|
||||
// OpenrecRanking 获取排名任务
|
||||
type OpenrecRanking struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
for {
|
||||
|
||||
errorMsg := sql.NullString{Valid: false}
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
|
||||
wf := cxt.Workflow()
|
||||
|
||||
data := &SourceOpenrec{}
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 {
|
||||
return
|
||||
}
|
||||
|
||||
data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
|
||||
data.SetUrl(wf.GetRawURL())
|
||||
data.SetTargetType(targetTypeRanking)
|
||||
|
||||
querys := wf.GetQuery()
|
||||
page, err := strconv.Atoi(querys.Get("page"))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
errorMsg.String = err.Error()
|
||||
errorMsg.Valid = true
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
data.SetOperator(10000)
|
||||
store.Save(data)
|
||||
return
|
||||
}
|
||||
|
||||
page++
|
||||
querys.Set("page", strconv.Itoa(page))
|
||||
wf.SetQuery(querys)
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
store.Save(data)
|
||||
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
104
tasks/openrec/task_openrec.go
Normal file
104
tasks/openrec/task_openrec.go
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"intimate"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
var targetTypeRanking = "openrec_ranking"
|
||||
var openrecRanking *OpenrecRanking
|
||||
|
||||
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var store *intimate.Store = intimate.NewStore("source_openrec")
|
||||
|
||||
func init() {
|
||||
|
||||
openrecRanking = &OpenrecRanking{}
|
||||
openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
-H 'authority: public.openrec.tv' \
|
||||
-H 'accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
|
||||
-H 'origin: https://www.openrec.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.openrec.tv/ranking' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
|
||||
--compressed`
|
||||
|
||||
}
|
||||
|
||||
// OpenrecRanking 获取排名任务
|
||||
type OpenrecRanking struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
for {
|
||||
|
||||
errorMsg := sql.NullString{Valid: false}
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
|
||||
wf := cxt.Workflow()
|
||||
|
||||
data := &intimate.Source{}
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 {
|
||||
return
|
||||
}
|
||||
|
||||
data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
|
||||
data.SetUrl(wf.GetRawURL())
|
||||
data.SetTargetType(targetTypeRanking)
|
||||
|
||||
querys := wf.GetQuery()
|
||||
page, err := strconv.Atoi(querys.Get("page"))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
errorMsg.String = err.Error()
|
||||
errorMsg.Valid = true
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
data.SetOperator(10000)
|
||||
store.Save(data)
|
||||
return
|
||||
}
|
||||
|
||||
page++
|
||||
querys.Set("page", strconv.Itoa(page))
|
||||
wf.SetQuery(querys)
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
store.Save(data)
|
||||
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
}
|
||||
|
||||
// OpenrecUser 获取用户信息
|
||||
type OpenrecUser struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecUser) Execute(cxt *hunter.TaskContext) {
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
resp.Content()
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user