add proxy
This commit is contained in:
parent
4083abe141
commit
5019e46d08
12
person.go
12
person.go
|
|
@ -97,9 +97,10 @@ func splitTasks(conf *Config) []ITask {
|
|||
initTask(conf, task, curl)
|
||||
for _, cnode := range proxies.GetLoopValues() {
|
||||
proxy := cnode.GetValue().(string)
|
||||
log.Println(proxy)
|
||||
task.AddProxies(proxy)
|
||||
}
|
||||
|
||||
log.Println(task.GetProxies())
|
||||
tasks = append(tasks, task)
|
||||
case 1:
|
||||
|
||||
|
|
@ -118,7 +119,7 @@ func splitTasks(conf *Config) []ITask {
|
|||
initTask(conf, ptask, ncurl)
|
||||
|
||||
ptask.AddProxies(proxy)
|
||||
tasks = append(tasks, task)
|
||||
tasks = append(tasks, ptask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -166,16 +167,19 @@ func ExecuteOnPlan(task ITask, result chan string) {
|
|||
urlname := task.GetCurl().Name
|
||||
|
||||
for {
|
||||
interval := time.Second * 2
|
||||
if task.TimeUp() {
|
||||
rtask = task.Execute(data) // 事件 在这里变化
|
||||
ntime := task.NextTime()
|
||||
interval = ntime.Sub(time.Now())
|
||||
if rtask == nil {
|
||||
// log.Println("rtask is nil")
|
||||
result <- fmt.Sprintf("rtask is nil, the first task = %s, name = %s", taskname, urlname)
|
||||
return
|
||||
}
|
||||
task = rtask
|
||||
|
||||
}
|
||||
interval := task.NextTime().Sub(time.Now())
|
||||
time.Sleep(interval)
|
||||
task = rtask
|
||||
}
|
||||
}
|
||||
|
|
|
|||
28
task.go
28
task.go
|
|
@ -44,10 +44,7 @@ func (task *Task) GetCurl() *curl2info.CURL {
|
|||
|
||||
// AddProxies 添加代理集合
|
||||
func (task *Task) AddProxies(proxy string) {
|
||||
for _, cnode := range task.proxies.GetLoopValues() {
|
||||
proxy := cnode.GetValue().(string)
|
||||
task.proxies.Append(proxy)
|
||||
}
|
||||
task.proxies.Append(proxy)
|
||||
}
|
||||
|
||||
// GetProxies 获取代理的字符串
|
||||
|
|
@ -78,23 +75,25 @@ func (task *Task) NextTime() time.Time {
|
|||
// Workflow 根据persistent 是否返回现有session(or 新建 session),创建Workflow的信息, 便于设置或者更改参数, Session持久化
|
||||
func (task *Task) Workflow(persistent bool) *requests.Workflow {
|
||||
|
||||
task.workflow = task.curl.CreateWorkflow(task.Session())
|
||||
if persistent {
|
||||
task.workflow = task.curl.CreateWorkflow(task.Session())
|
||||
return task.workflow
|
||||
}
|
||||
|
||||
return task.curl.CreateWorkflow(nil)
|
||||
proxies := task.GetProxies()
|
||||
ses := task.curl.CreateSession()
|
||||
if proxies.Size() > 0 {
|
||||
proxy := proxies.Cursor().GetValue().(string)
|
||||
ses.SetConfig(requests.CProxy, proxy)
|
||||
proxies.MoveNext()
|
||||
}
|
||||
|
||||
return task.curl.CreateWorkflow(ses)
|
||||
}
|
||||
|
||||
// Request 根据curl信息执行,没持久化
|
||||
func (task *Task) Request() (*requests.Response, error) {
|
||||
proxies := task.GetProxies()
|
||||
ses := task.curl.CreateSession()
|
||||
if proxies.Size() > 0 {
|
||||
ses.SetConfig(requests.CProxy, proxies.Cursor().GetValue().(string))
|
||||
proxies.MoveNext()
|
||||
}
|
||||
return task.curl.CreateWorkflow(ses).Execute()
|
||||
return task.Workflow(false).Execute()
|
||||
}
|
||||
|
||||
// Session 获取Session的信息 只保留session的数据和url参数.
|
||||
|
|
@ -103,7 +102,8 @@ func (task *Task) Session() *requests.Session {
|
|||
task.session = task.curl.CreateSession()
|
||||
}
|
||||
if task.proxies.Size() > 0 {
|
||||
task.session.SetConfig(requests.CProxy, task.proxies.Cursor().GetValue().(string))
|
||||
proxy := task.proxies.Cursor().GetValue().(string)
|
||||
task.session.SetConfig(requests.CProxy, proxy)
|
||||
task.proxies.MoveNext()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ type Toutiao struct {
|
|||
|
||||
func (tt *Toutiao) Execute(data interface{}) ITask {
|
||||
resp, err := tt.Request()
|
||||
log.Println(resp, err)
|
||||
log.Println(resp.Content()[0:20], err)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
curl --name "tt-information" 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' --task toutiao -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'
|
||||
curl --name "cip.cc" 'http://cip.cc'
|
||||
curl --name "ag-cn" 'https://appgrowing.cn/' -H 'authority: appgrowing.cn' -H 'cache-control: max-age=0' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: zh' -H 'cookie: _ga=GA1.2.1371058419.1533104518; _gid=GA1.2.1726457685.1544092110; _gat_gtag_UA_4002880_19=1' -H 'if-none-match: W/"5c088f5a-ca6"' -H 'if-modified-since: Thu, 06 Dec 2018 02:54:18 GMT' --compressed
|
||||
|
|
@ -18,4 +18,4 @@ media : 55
|
|||
spider_id : 73
|
||||
catch_account_id : 123
|
||||
|
||||
crontab: "2s"
|
||||
crontab: "1s"
|
||||
1
toutiao.curl
Normal file
1
toutiao.curl
Normal file
|
|
@ -0,0 +1 @@
|
|||
curl --name "tt-information" 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' --task toutiao -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'
|
||||
Loading…
Reference in New Issue
Block a user