From 9d7c2e1e549c7f9b0c57e7208bf5e3f8abf9b092 Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 17 Aug 2020 14:38:00 +0800 Subject: [PATCH] =?UTF-8?q?1.=E4=BF=AE=E6=94=B9build.sh=202.=E5=88=A0?= =?UTF-8?q?=E9=99=A4hunter=E5=8C=85=E5=BC=95=E7=94=A8=E5=92=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + build.sh | 8 ++- extractor/openrec_extractor/openrec_test.go | 23 -------- extractor_field.go | 36 ------------- go.mod | 5 +- go.sum | 12 +---- store_test.go | 3 +- .../openrec_task1/task_openrec_test.go | 52 ------------------- 8 files changed, 12 insertions(+), 128 deletions(-) diff --git a/.gitignore b/.gitignore index 657f5f1..23a8cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +bin *.log log screenlog.* diff --git a/build.sh b/build.sh index 6bdb63f..f92ccd4 100644 --- a/build.sh +++ b/build.sh @@ -7,13 +7,17 @@ src=`pwd` for path in `ls -d $source_tasks` do echo $path - cd $path && go build + projectname=${path##*/} + projectworkspace=$src/bin/$projectname + cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname cd $src done for path in `ls -d $extractor_tasks` do echo $path - cd $path && go build + projectname=${path##*/} + projectworkspace=$src/bin/$projectname + cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname cd $src done diff --git a/extractor/openrec_extractor/openrec_test.go b/extractor/openrec_extractor/openrec_test.go index e9bc878..2620803 100644 --- a/extractor/openrec_extractor/openrec_test.go +++ b/extractor/openrec_extractor/openrec_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/474420502/hunter" "github.com/lestrrat-go/libxml2" ) @@ -91,28 +90,6 @@ func TestCase(t *testing.T) { t.Error(xr) } -func TestUserName(t *testing.T) { - f, err := os.Open("test.html") - if err != nil { - panic(err) - } - data, err := ioutil.ReadAll(f) - if err != nil { - panic(err) - } - - extractor := hunter.NewExtractor(data) - xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()") - if err != nil { - t.Error(err) - } else { - if xp.NodeIter().Next() { - userName := xp.String() - t.Error(userName) - } - } -} - func TestExtractor(t *testing.T) { Execute() } diff --git a/extractor_field.go b/extractor_field.go index 2977941..1adda5e 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -4,9 +4,6 @@ import ( "database/sql" "reflect" "time" - - "github.com/474420502/hunter" - "github.com/tidwall/gjson" ) type GetSet struct { @@ -104,36 +101,3 @@ func (cl *CollectLog) Get(field string) interface{} { func (cl *CollectLog) Set(field string, value interface{}) { reflect.ValueOf(cl).Elem().FieldByName(field).Set(reflect.ValueOf(value)) } - -type ExtractorSource struct { - source *gjson.Result - extractor *hunter.Extractor -} - -func NewExtractorSource(gr *gjson.Result) *ExtractorSource { - es := &ExtractorSource{} - es.SetSource(gr) - return es -} - -func (es *ExtractorSource) SetSource(gr *gjson.Result) { - es.source = gr - es.extractor = nil -} - -func (es *ExtractorSource) Clear() { - es.source = nil - es.extractor = nil -} - -func (es *ExtractorSource) CreateExtractor() { - es.extractor = hunter.NewExtractor([]byte(es.source.String())) -} - -func (es *ExtractorSource) GetSource() *gjson.Result { - return es.source -} - -func (es *ExtractorSource) GetExtractor() *hunter.Extractor { - return es.extractor -} diff --git a/go.mod b/go.mod index e58f8be..312edd1 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,11 @@ module intimate -go 1.14 +go 1.15 require ( - github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 + github.com/474420502/extractor v0.9.5 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.2.0 - github.com/474420502/hunter v0.3.4 github.com/474420502/requests v1.7.0 github.com/go-sql-driver/mysql v1.5.0 github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb diff --git a/go.sum b/go.sum index e71679a..65f21fa 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/extractor v0.9.2 h1:pDDLpxq4bsWjwU3H8epBxDifnmMVBdT0onye2MXCJ8A= -github.com/474420502/extractor v0.9.2/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= -github.com/474420502/extractor v0.9.3 h1:Cjri64DbgWQQ64EjPiBSQfUH9l0cYlzU8py0PQu8pAs= -github.com/474420502/extractor v0.9.3/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= -github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 h1:cxgsTQwRJSiML4yBL40n/0pD/FbEqkCIXE7qq6hJyLg= -github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= -github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 h1:6g4sPgooFdsVAdxNMhP6sqKQ0Z5EPBb4tGj9/absPoY= -github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= +github.com/474420502/extractor v0.9.5 h1:nM3/1tNL8BHS3PN9eXvm8Ve3hqTReKNB+ba8VpdL8bw= +github.com/474420502/extractor v0.9.5/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= @@ -18,8 +12,6 @@ github.com/474420502/gcurl v0.2.0 h1:m6+vw4NX4f5Tfp7c3nuaIgHUE/7zTX6K3xK+pTCBoCo github.com/474420502/gcurl v0.2.0/go.mod h1:kJZDbgXn5wbAaR+hhBi4Sbw44P4igJ7qYXC6mejLuhQ= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU= -github.com/474420502/hunter v0.3.4 h1:fyLAgI84jWe3IcqsISC53j1w3CXI1FERxX//Potns0M= -github.com/474420502/hunter v0.3.4/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA= github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw= diff --git a/store_test.go b/store_test.go index 7d169ff..d9c6858 100644 --- a/store_test.go +++ b/store_test.go @@ -5,8 +5,7 @@ import ( ) func TestStoreInsert(t *testing.T) { - // ht := hunter.NewHunter(openrecRanking) - // ht.Execute() + } func TestStoreInsertCase1(t *testing.T) { diff --git a/tasks/openrec/openrec_task1/task_openrec_test.go b/tasks/openrec/openrec_task1/task_openrec_test.go index db355eb..9e649d2 100644 --- a/tasks/openrec/openrec_task1/task_openrec_test.go +++ b/tasks/openrec/openrec_task1/task_openrec_test.go @@ -3,60 +3,8 @@ package main import ( "testing" "time" - - "github.com/tidwall/gjson" - - "github.com/474420502/hunter" ) -// OpenrecRanking 获取排名任务 -type OpenrecRankingTest struct { - hunter.PreCurlUrl -} - -// Execute 执行方法 -func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) { - - resp, err := cxt.Hunt() - if err != nil { - panic(err) - } - t := cxt.GetShare("test").(*testing.T) - if !gjson.ValidBytes(resp.Content()) { - t.Error("source is not json format.") - } - result := gjson.GetBytes(resp.Content(), "0.rank") - if result.Int() != 1 { - t.Error("rank is error. result raw is ", result.Raw) - } - - if cxt.Temporary().GetQuery().Get("page") != "1" { - t.Error("Temporary page error") - } - // t.Error(string(resp.Content())) -} - -func TestRanking(t *testing.T) { - curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ - -H 'authority: public.openrec.tv' \ - -H 'accept: application/json, text/javascript, */*; q=0.01' \ - -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ - -H 'origin: https://www.openrec.tv' \ - -H 'sec-fetch-site: same-site' \ - -H 'sec-fetch-mode: cors' \ - -H 'sec-fetch-dest: empty' \ - -H 'referer: https://www.openrec.tv/ranking' \ - -H 'accept-language: zh-CN,zh;q=0.9' \ - -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ - --compressed` - - ort := &OpenrecRankingTest{} - ort.PreCurlUrl = hunter.PreCurlUrl(curlBash) - ht := hunter.NewHunter(ort) - ht.SetShare("test", t) - ht.Execute() -} - func TestTimeAdd(t *testing.T) { finishpoint := time.Now() time.Sleep(time.Second * 2)