test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.修改build.sh

Browse Source 
2.删除hunter包引用和使用
This commit is contained in:
eson 2020-08-17 14:38:00 +08:00
parent d1298dc3f3
commit 9d7c2e1e54
8 changed files with 12 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
bin
*.log
log
screenlog.*

8
build.sh
View File

@ -7,13 +7,17 @@ src=`pwd`
for path in `ls -d $source_tasks`
do
echo $path
cd $path && go build
projectname=${path##*/}
projectworkspace=$src/bin/$projectname
cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname
cd $src
done
for path in `ls -d $extractor_tasks`
do
echo $path
cd $path && go build
projectname=${path##*/}
projectworkspace=$src/bin/$projectname
cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname
cd $src
done

23
extractor/openrec_extractor/openrec_test.go
View File

@ -7,7 +7,6 @@ import (
"testing"
"time"
"github.com/474420502/hunter"
"github.com/lestrrat-go/libxml2"
)
@ -91,28 +90,6 @@ func TestCase(t *testing.T) {
t.Error(xr)
}
func TestUserName(t *testing.T) {
f, err := os.Open("test.html")
if err != nil {
panic(err)
}
data, err := ioutil.ReadAll(f)
if err != nil {
panic(err)
}
extractor := hunter.NewExtractor(data)
xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()")
if err != nil {
t.Error(err)
} else {
if xp.NodeIter().Next() {
userName := xp.String()
t.Error(userName)
}
}
}
func TestExtractor(t *testing.T) {
Execute()
}

36
extractor_field.go
View File

@ -4,9 +4,6 @@ import (
"database/sql"
"reflect"
"time"
"github.com/474420502/hunter"
"github.com/tidwall/gjson"
)
type GetSet struct {
@ -104,36 +101,3 @@ func (cl *CollectLog) Get(field string) interface{} {
func (cl *CollectLog) Set(field string, value interface{}) {
reflect.ValueOf(cl).Elem().FieldByName(field).Set(reflect.ValueOf(value))
}
type ExtractorSource struct {
source *gjson.Result
extractor *hunter.Extractor
}
func NewExtractorSource(gr *gjson.Result) *ExtractorSource {
es := &ExtractorSource{}
es.SetSource(gr)
return es
}
func (es *ExtractorSource) SetSource(gr *gjson.Result) {
es.source = gr
es.extractor = nil
}
func (es *ExtractorSource) Clear() {
es.source = nil
es.extractor = nil
}
func (es *ExtractorSource) CreateExtractor() {
es.extractor = hunter.NewExtractor([]byte(es.source.String()))
}
func (es *ExtractorSource) GetSource() *gjson.Result {
return es.source
}
func (es *ExtractorSource) GetExtractor() *hunter.Extractor {
return es.extractor
}

5
go.mod
View File

@ -1,12 +1,11 @@
module intimate
go 1.14
go 1.15
require (
github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89
github.com/474420502/extractor v0.9.5
github.com/474420502/focus v0.12.0
github.com/474420502/gcurl v0.2.0
github.com/474420502/hunter v0.3.4
github.com/474420502/requests v1.7.0
github.com/go-sql-driver/mysql v1.5.0
github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb

12
go.sum
View File

@ -2,14 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
github.com/474420502/extractor v0.9.2 h1:pDDLpxq4bsWjwU3H8epBxDifnmMVBdT0onye2MXCJ8A=
github.com/474420502/extractor v0.9.2/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4=
github.com/474420502/extractor v0.9.3 h1:Cjri64DbgWQQ64EjPiBSQfUH9l0cYlzU8py0PQu8pAs=
github.com/474420502/extractor v0.9.3/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4=
github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 h1:cxgsTQwRJSiML4yBL40n/0pD/FbEqkCIXE7qq6hJyLg=
github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4=
github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 h1:6g4sPgooFdsVAdxNMhP6sqKQ0Z5EPBb4tGj9/absPoY=
github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4=
github.com/474420502/extractor v0.9.5 h1:nM3/1tNL8BHS3PN9eXvm8Ve3hqTReKNB+ba8VpdL8bw=
github.com/474420502/extractor v0.9.5/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY=
github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo=
github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg=
@ -18,8 +12,6 @@ github.com/474420502/gcurl v0.2.0 h1:m6+vw4NX4f5Tfp7c3nuaIgHUE/7zTX6K3xK+pTCBoCo
github.com/474420502/gcurl v0.2.0/go.mod h1:kJZDbgXn5wbAaR+hhBi4Sbw44P4igJ7qYXC6mejLuhQ=
github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc=
github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU=
github.com/474420502/hunter v0.3.4 h1:fyLAgI84jWe3IcqsISC53j1w3CXI1FERxX//Potns0M=
github.com/474420502/hunter v0.3.4/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA=
github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY=
github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw=

3
store_test.go
View File

@ -5,8 +5,7 @@ import (
)
func TestStoreInsert(t *testing.T) {
// ht := hunter.NewHunter(openrecRanking)
// ht.Execute()
}
func TestStoreInsertCase1(t *testing.T) {

52
tasks/openrec/openrec_task1/task_openrec_test.go
View File

@ -3,60 +3,8 @@ package main
import (
"testing"
"time"
"github.com/tidwall/gjson"
"github.com/474420502/hunter"
)
// OpenrecRanking 获取排名任务
type OpenrecRankingTest struct {
hunter.PreCurlUrl
}
// Execute 执行方法
func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) {
resp, err := cxt.Hunt()
if err != nil {
panic(err)
}
t := cxt.GetShare("test").(*testing.T)
if !gjson.ValidBytes(resp.Content()) {
t.Error("source is not json format.")
}
result := gjson.GetBytes(resp.Content(), "0.rank")
if result.Int() != 1 {
t.Error("rank is error. result raw is ", result.Raw)
}
if cxt.Temporary().GetQuery().Get("page") != "1" {
t.Error("Temporary page error")
}
// t.Error(string(resp.Content()))
}
func TestRanking(t *testing.T) {
curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
-H 'authority: public.openrec.tv' \
-H 'accept: application/json, text/javascript, */*; q=0.01' \
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
-H 'origin: https://www.openrec.tv' \
-H 'sec-fetch-site: same-site' \
-H 'sec-fetch-mode: cors' \
-H 'sec-fetch-dest: empty' \
-H 'referer: https://www.openrec.tv/ranking' \
-H 'accept-language: zh-CN,zh;q=0.9' \
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
--compressed`
ort := &OpenrecRankingTest{}
ort.PreCurlUrl = hunter.PreCurlUrl(curlBash)
ht := hunter.NewHunter(ort)
ht.SetShare("test", t)
ht.Execute()
}
func TestTimeAdd(t *testing.T) {
finishpoint := time.Now()
time.Sleep(time.Second * 2)