From eba36b0c95b0774450708d621641664fab16a882 Mon Sep 17 00:00:00 2001 From: eson Date: Tue, 7 Jul 2020 18:14:14 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=20openrec=20=E7=9A=84?= =?UTF-8?q?=E6=8E=92=E5=90=8D=E8=8E=B7=E5=8F=96=E4=B8=BB=E6=92=AD=E4=B8=BB?= =?UTF-8?q?=E9=A1=B5=E7=9A=84=E5=8A=9F=E8=83=BD.=20TODO:=20=E5=88=86?= =?UTF-8?q?=E8=A7=A3=E6=88=90=E5=A4=9A=E4=BB=BB=E5=8A=A1=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E6=9C=BA=E5=88=B6.=20=E8=AE=A9=E6=AF=8F=E4=B8=AA=E5=AD=90?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=8B=AC=E7=AB=8B=E4=BA=92=E4=B8=8D=E5=B9=B2?= =?UTF-8?q?=E6=89=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform_openrec.go | 134 -------------------------- platform_openrec_test.go | 51 ---------- source_openrec.go | 196 +++++++++++++++++++++++++++++++++++++++ source_openrec_test.go | 57 ++++++++++++ sql/intimate_source.sql | 2 +- store.go | 15 +-- store_test.go | 6 +- 7 files changed, 265 insertions(+), 196 deletions(-) delete mode 100644 platform_openrec.go delete mode 100644 platform_openrec_test.go create mode 100644 source_openrec.go create mode 100644 source_openrec_test.go diff --git a/platform_openrec.go b/platform_openrec.go deleted file mode 100644 index dc2d780..0000000 --- a/platform_openrec.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "database/sql" - "time" - - "github.com/474420502/hunter" -) - -var targetTypeRanking = "openrec_ranking" -var openrecRanking *OpenrecRanking - -// storeOpenrec 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql -var storeOpenrec *Store - -func init() { - - openrecRanking = &OpenrecRanking{} - openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ - -H 'authority: public.openrec.tv' \ - -H 'accept: application/json, text/javascript, */*; q=0.01' \ - -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ - -H 'origin: https://www.openrec.tv' \ - -H 'sec-fetch-site: same-site' \ - -H 'sec-fetch-mode: cors' \ - -H 'sec-fetch-dest: empty' \ - -H 'referer: https://www.openrec.tv/ranking' \ - -H 'accept-language: zh-CN,zh;q=0.9' \ - -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ - --compressed` -} - -/* -CREATE TABLE `platform_openrec` ( - `url` text NOT NULL, - `target_type` varchar(64) NOT NULL, - `source` longtext DEFAULT NULL, - `ext` json DEFAULT NULL, - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `operator` tinyint NOT NULL, - KEY `operator_idx` (`operator`), - KEY `update_time_idx` (`update_time`), - KEY `target_type_idx` (`target_type`) - ); -*/ - -type PlatformOpenrec struct { - Url string // - TargetType string // - Source sql.NullString // - Ext interface{} // - UpdateTime time.Time // - Operator int32 // -} - -// GetOperator Get return Operator int32 -func (po *PlatformOpenrec) GetOperator() int32 { - return po.Operator -} - -// SetOperator Set Operator int32 -func (po *PlatformOpenrec) SetOperator(Operator int32) { - po.Operator = Operator -} - -// GetUpdateTime Get return UpdateTime time.Time -func (po *PlatformOpenrec) GetUpdateTime() time.Time { - return po.UpdateTime -} - -// SetUpdateTime Set UpdateTime time.Time -func (po *PlatformOpenrec) SetUpdateTime(UpdateTime time.Time) { - po.UpdateTime = UpdateTime -} - -// GetExt Get return Ext interface{} -func (po *PlatformOpenrec) GetExt() interface{} { - return po.Ext -} - -// SetExt Set Ext interface{} -func (po *PlatformOpenrec) SetExt(Ext interface{}) { - po.Ext = Ext -} - -// GetSource Get return Source sql.NullString -func (po *PlatformOpenrec) GetSource() sql.NullString { - return po.Source -} - -// SetSource Set Source sql.NullString -func (po *PlatformOpenrec) SetSource(Source sql.NullString) { - po.Source = Source -} - -// GetTargetType Get return TargetType string -func (po *PlatformOpenrec) GetTargetType() string { - return po.TargetType -} - -// SetTargetType Set TargetType string -func (po *PlatformOpenrec) SetTargetType(TargetType string) { - po.TargetType = TargetType -} - -// GetUrl Get return Url string -func (po *PlatformOpenrec) GetUrl() string { - return po.Url -} - -// SetUrl Set Url string -func (po *PlatformOpenrec) SetUrl(Url string) { - po.Url = Url -} - -// OpenrecRanking 获取排名任务 -type OpenrecRanking struct { - hunter.PreCurlUrl -} - -// Execute 执行方法 -func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { - resp, err := cxt.Hunt() - if err != nil { - panic(err) - } - - data := &PlatformOpenrec{} - content := resp.Content() - data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0}) - data.SetUrl(cxt.Workflow().GetRawURL()) - data.SetTargetType(targetTypeRanking) - storeOpenrec.Save(data) -} diff --git a/platform_openrec_test.go b/platform_openrec_test.go deleted file mode 100644 index 71371cd..0000000 --- a/platform_openrec_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "testing" - - "github.com/tidwall/gjson" - - "github.com/474420502/hunter" -) - -// OpenrecRanking 获取排名任务 -type OpenrecRankingTest struct { - hunter.PreCurlUrl -} - -// Execute 执行方法 -func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) { - - resp, err := cxt.Hunt() - if err != nil { - panic(err) - } - t := cxt.GetShare("test").(*testing.T) - if !gjson.ValidBytes(resp.Content()) { - t.Error("source is not json format.") - } - result := gjson.GetBytes(resp.Content(), "0.rank") - if result.Int() != 1 { - t.Error("rank is error. result raw is ", result.Raw) - } - // t.Error(string(resp.Content())) -} - -func TestRanking(t *testing.T) { - // curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ - // -H 'authority: public.openrec.tv' \ - // -H 'accept: application/json, text/javascript, */*; q=0.01' \ - // -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ - // -H 'origin: https://www.openrec.tv' \ - // -H 'sec-fetch-site: same-site' \ - // -H 'sec-fetch-mode: cors' \ - // -H 'sec-fetch-dest: empty' \ - // -H 'referer: https://www.openrec.tv/ranking' \ - // -H 'accept-language: zh-CN,zh;q=0.9' \ - // -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ - // --compressed` - - ht := hunter.NewHunter(openrecRanking) - ht.SetShare("test", t) - ht.Execute() -} diff --git a/source_openrec.go b/source_openrec.go new file mode 100644 index 0000000..771e90a --- /dev/null +++ b/source_openrec.go @@ -0,0 +1,196 @@ +package main + +import ( + "database/sql" + "log" + "strconv" + "time" + + "github.com/474420502/hunter" +) + +var targetTypeRanking = "openrec_ranking" +var openrecRanking *OpenrecRanking + +// storeOpenrec 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql +var storeOpenrec *Store + +func init() { + + openrecRanking = &OpenrecRanking{} + openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ + -H 'authority: public.openrec.tv' \ + -H 'accept: application/json, text/javascript, */*; q=0.01' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ + -H 'origin: https://www.openrec.tv' \ + -H 'sec-fetch-site: same-site' \ + -H 'sec-fetch-mode: cors' \ + -H 'sec-fetch-dest: empty' \ + -H 'referer: https://www.openrec.tv/ranking' \ + -H 'accept-language: zh-CN,zh;q=0.9' \ + -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ + --compressed` +} + +/* +CREATE TABLE `source_openrec` ( + uid bigint AUTO_INCREMENT, + `url` text NOT NULL, + `target_type` varchar(64) NOT NULL, + `source` longtext DEFAULT NULL, + `ext` json DEFAULT NULL, + + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `operator` int DEFAULT 0, + `error_msg` text DEFAULT NULL, + PRIMARY KEY(`uid`), + KEY `operator_idx` (`operator`), + KEY `update_time_idx` (`update_time`), + KEY `target_type_idx` (`target_type`) + ); +*/ + +type SourceOpenrec struct { + Uid sql.NullInt64 // + Url string // + TargetType string // + Source sql.NullString // + Ext interface{} // + UpdateTime time.Time // + Operator int32 // + ErrorMsg sql.NullString // +} + +// GetErrorMsg Get return ErrorMsg sql.NullString +func (so *SourceOpenrec) GetErrorMsg() sql.NullString { + return so.ErrorMsg +} + +// SetErrorMsg Set ErrorMsg sql.NullString +func (so *SourceOpenrec) SetErrorMsg(ErrorMsg sql.NullString) { + so.ErrorMsg = ErrorMsg +} + +// GetOperator Get return Operator sql.NullInt32 +func (so *SourceOpenrec) GetOperator() int32 { + return so.Operator +} + +// SetOperator Set Operator sql.NullInt32 +func (so *SourceOpenrec) SetOperator(Operator int32) { + so.Operator = Operator +} + +// GetUpdateTime Get return UpdateTime time.Time +func (so *SourceOpenrec) GetUpdateTime() time.Time { + return so.UpdateTime +} + +// SetUpdateTime Set UpdateTime time.Time +func (so *SourceOpenrec) SetUpdateTime(UpdateTime time.Time) { + so.UpdateTime = UpdateTime +} + +// GetExt Get return Ext interface{} +func (so *SourceOpenrec) GetExt() interface{} { + return so.Ext +} + +// SetExt Set Ext interface{} +func (so *SourceOpenrec) SetExt(Ext interface{}) { + so.Ext = Ext +} + +// GetSource Get return Source sql.NullString +func (so *SourceOpenrec) GetSource() sql.NullString { + return so.Source +} + +// SetSource Set Source sql.NullString +func (so *SourceOpenrec) SetSource(Source sql.NullString) { + so.Source = Source +} + +// GetTargetType Get return TargetType string +func (so *SourceOpenrec) GetTargetType() string { + return so.TargetType +} + +// SetTargetType Set TargetType string +func (so *SourceOpenrec) SetTargetType(TargetType string) { + so.TargetType = TargetType +} + +// GetUrl Get return Url string +func (so *SourceOpenrec) GetUrl() string { + return so.Url +} + +// SetUrl Set Url string +func (so *SourceOpenrec) SetUrl(Url string) { + so.Url = Url +} + +// GetUid Get return Uid sql.NullInt64 +func (so *SourceOpenrec) GetUid() sql.NullInt64 { + return so.Uid +} + +// SetUid Set Uid sql.NullInt64 +func (so *SourceOpenrec) SetUid(Uid sql.NullInt64) { + so.Uid = Uid +} + +// OpenrecRanking 获取排名任务 +type OpenrecRanking struct { + hunter.PreCurlUrl +} + +// Execute 执行方法 +func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { + + for { + + errorMsg := sql.NullString{Valid: false} + resp, err := cxt.Hunt() + if err != nil { + log.Println(err) + break + } + + wf := cxt.Workflow() + + data := &SourceOpenrec{} + content := resp.Content() + log.Println(len(content)) + if len(content) <= 300 { + return + } + + data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0}) + data.SetUrl(wf.GetRawURL()) + data.SetTargetType(targetTypeRanking) + + querys := wf.GetQuery() + page, err := strconv.Atoi(querys.Get("page")) + if err != nil { + log.Println(err) + errorMsg.String = err.Error() + errorMsg.Valid = true + + data.SetErrorMsg(errorMsg) + data.SetOperator(10000) + storeOpenrec.Save(data) + return + } + + page++ + querys.Set("page", strconv.Itoa(page)) + wf.SetQuery(querys) + + data.SetErrorMsg(errorMsg) + storeOpenrec.Save(data) + + time.Sleep(time.Second * 2) + } +} diff --git a/source_openrec_test.go b/source_openrec_test.go new file mode 100644 index 0000000..6ef3628 --- /dev/null +++ b/source_openrec_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "testing" + + "github.com/tidwall/gjson" + + "github.com/474420502/hunter" +) + +// OpenrecRanking 获取排名任务 +type OpenrecRankingTest struct { + hunter.PreCurlUrl +} + +// Execute 执行方法 +func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) { + + resp, err := cxt.Hunt() + if err != nil { + panic(err) + } + t := cxt.GetShare("test").(*testing.T) + if !gjson.ValidBytes(resp.Content()) { + t.Error("source is not json format.") + } + result := gjson.GetBytes(resp.Content(), "0.rank") + if result.Int() != 1 { + t.Error("rank is error. result raw is ", result.Raw) + } + + if cxt.Workflow().GetQuery().Get("page") != "1" { + t.Error("workflow page error") + } + // t.Error(string(resp.Content())) +} + +func TestRanking(t *testing.T) { + curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ + -H 'authority: public.openrec.tv' \ + -H 'accept: application/json, text/javascript, */*; q=0.01' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ + -H 'origin: https://www.openrec.tv' \ + -H 'sec-fetch-site: same-site' \ + -H 'sec-fetch-mode: cors' \ + -H 'sec-fetch-dest: empty' \ + -H 'referer: https://www.openrec.tv/ranking' \ + -H 'accept-language: zh-CN,zh;q=0.9' \ + -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ + --compressed` + + ort := &OpenrecRankingTest{} + ort.PreCurlUrl = hunter.PreCurlUrl(curlBash) + ht := hunter.NewHunter(ort) + ht.SetShare("test", t) + ht.Execute() +} diff --git a/sql/intimate_source.sql b/sql/intimate_source.sql index 3e3aadd..8e18af3 100644 --- a/sql/intimate_source.sql +++ b/sql/intimate_source.sql @@ -1,7 +1,7 @@ create database if not exists `intimate_source`; use intimate_source; -CREATE TABLE IF NOT EXISTS `platform_openrec` ( +CREATE TABLE IF NOT EXISTS `source_openrec` ( uid bigint AUTO_INCREMENT, `url` text NOT NULL, `target_type` varchar(64) NOT NULL, diff --git a/store.go b/store.go index ef9d1b8..dcec84c 100644 --- a/store.go +++ b/store.go @@ -10,11 +10,14 @@ import ( // ISource 源接口结构 type ISource interface { - GetUrl() string // - GetTargetType() string // - GetSource() sql.NullString // - GetExt() interface{} // - GetUpdateTime() time.Time // + GetUrl() string // + GetTargetType() string // + GetSource() sql.NullString // + GetExt() interface{} // + GetUpdateTime() time.Time // + GetOperator() int32 // + GetErrorMsg() sql.NullString // + } // Store 储存 @@ -33,7 +36,7 @@ func NewStore() *Store { // Save 储存数据 func (store *Store) Save(isource ISource) { - _, err := store.db.Exec("insert into `platform_openrec`(url, target_type, source, ext) values(?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt()) + _, err := store.db.Exec("insert into `platform_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg()) if err != nil { log.Fatalln(err) } diff --git a/store_test.go b/store_test.go index 80a9664..cc68934 100644 --- a/store_test.go +++ b/store_test.go @@ -2,13 +2,11 @@ package main import ( "testing" - - "github.com/474420502/hunter" ) func TestStoreInsert(t *testing.T) { - ht := hunter.NewHunter(openrecRanking) - ht.Execute() + // ht := hunter.NewHunter(openrecRanking) + // ht.Execute() } func TestStoreInsertCase1(t *testing.T) {