test completed! TODO: Proxy

2018-12-07 18:21:23 +08:00 · 2018-12-07 18:21:23 +08:00 · addc6bfa6b
commit addc6bfa6b
parent d08f6c8e68
12 changed files with 120 additions and 70 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,4 @@
 *.pyc
 *.vscode
 *.test
+imitater
--- a/base.go
+++ b/base.go
@ -1 +1 @@
-package imitate
+package imitater
--- a/config.go
+++ b/config.go
@ -1,4 +1,4 @@
-package imitate
+package imitater

 import (
 	"errors"
@ -25,10 +25,15 @@ func (curls *YamlCurls) UnmarshalYAML(unmarshal func(interface{}) error) error {

 	switch tbuf := buf.(type) {
 	case string:
-		*curls = append(*curls, parseCurl(tbuf))
+		for _, curlinfo := range parseCurl(tbuf) {
+			*curls = append(*curls, curlinfo)
+		}
+
 	case []interface{}:
 		for _, ifa := range tbuf {
-			*curls = append(*curls, parseCurl(ifa.(string)))
+			for _, curlinfo := range parseCurl(ifa.(string)) {
+				*curls = append(*curls, curlinfo)
+			}
 		}
 	default:
 		return errors.New("read curls is error, " + reflect.TypeOf(buf).String())
@ -97,6 +102,7 @@ type Config struct {
 	Curls    YamlCurls   `yaml:"curls"`

 	Crontab string `yaml:"crontab"`
+	ITask   string `yaml:"task"`

 	Device         string `yaml:"device"`
 	Platform       string `yaml:"platform"`
@ -145,7 +151,8 @@ func NewConfig(p string) *Config {
 	return conf
 }

-func parseCurl(curl string) string {
+func parseCurl(curl string) []string {
+	var result []string
 	switch curl[0] {
 	case '@':
 		curlfile, err := os.Open(curl[1:])
@ -153,8 +160,12 @@ func parseCurl(curl string) string {
 		if err != nil {
 			panic(err)
 		}
+
 		curldata, err := ioutil.ReadAll(curlfile)
-		return strings.Trim(string(curldata), "\r\n ")
+		for _, curlinfo := range strings.Split(string(curldata), "\n") {
+			result = append(result, strings.Trim(curlinfo, "\r\n "))
+		}
+
 	case '#':
 		resp, err := http.Get(curl[1:])
 		if err != nil {
@ -164,8 +175,10 @@ func parseCurl(curl string) string {
 		if err != nil {
 			panic(err)
 		}
-		return strings.Trim(string(curldata), "\r\n ")
+		for _, curlinfo := range strings.Split(string(curldata), "\n") {
+			result = append(result, strings.Trim(curlinfo, "\r\n "))
+		}
 	}

-	return strings.Trim(curl, "\r\n ")
+	return result
 }
--- a/config_test.go
+++ b/config_test.go
@ -1,4 +1,4 @@
-package imitate
+package imitater

 import (
 	"regexp"
--- a/go.mod
+++ b/go.mod
@ -1,8 +0,0 @@
-module 474420502.top/eson/imitater
-
-require (
-	gopkg.in/yaml.v2 v2.2.2
-	github.com/davecgh/go-spew/spew v1.1.1
-	474420502.top/eson/curl2info v1.0.0
-	474420502.top/eson/requests v1.0.0
-)
--- a/person.go
+++ b/person.go
@ -1,6 +1,7 @@
-package imitate
+package imitater

 import (
+	"fmt"
 	"log"
 	"reflect"
 	"time"
@ -10,14 +11,16 @@ import (

 // ITask 继承这个接口的类
 type ITask interface {
-	Execute()
+	Execute(data interface{}) ITask

 	SetCrontab(cron string)
-
 	SetLastStatus(status bool)

+	SetName(name string)
+	GetName() string
+
 	SetCurl(Curl *curl2info.CURL)
-	//GetCurl() *curl2info.CURL
+	GetCurl() *curl2info.CURL

 	GetProxies() []string
 	AppendProxies(proxies ...string)
@ -29,16 +32,17 @@ type ITask interface {
 var register = make(map[string]reflect.Type)

 func init() {
-
+	log.SetFlags(log.Llongfile)
 }

 // Register 注册 类型 ITask为样本的类型
-func Register(name string, itype interface{}) {
-	register[name] = reflect.TypeOf(itype)
+func Register(name string, itask ITask) {
+	register[name] = reflect.TypeOf(itask).Elem()
 }

-func makeRegisterType(name string) interface{} {
-	result := reflect.New(register[name]).Interface()
+func makeRegisterType(name string) ITask {
+	result := reflect.New(register[name]).Interface().(ITask)
+	result.SetName(name)
 	return result
 }

@ -81,18 +85,15 @@ func splitTasks(conf *Config) []ITask {
 			panic(err)
 		}

-		task := makeRegisterType(curl.ITask).(ITask)
+		if curl.ITask == "" {
+			curl.ITask = conf.ITask
+		}
+		task := makeRegisterType(curl.ITask)

 		switch conf.Mode {
 		case 0:

-			if curl.Crontab != "" {
-				task.SetCrontab(curl.Crontab)
-			} else {
-				log.Println(conf.Crontab)
-				task.SetCrontab(conf.Crontab)
-			}
-			task.SetCurl(curl)
+			initTask(conf, task, curl)
 			task.AppendProxies(conf.Proxies...)
 			tasks = append(tasks, task)

@ -103,13 +104,11 @@ func splitTasks(conf *Config) []ITask {
 					panic(err)
 				}

-				ptask := makeRegisterType(ncurl.ITask).(ITask)
-				if ncurl.Crontab != "" {
-					ptask.SetCrontab(ncurl.Crontab)
-				} else {
-					ptask.SetCrontab(conf.Crontab)
+				if curl.ITask == "" {
+					curl.ITask = conf.ITask
 				}
-				InitTask(ptask, ncurl)
+				ptask := makeRegisterType(ncurl.ITask).(ITask)
+				initTask(conf, ptask, ncurl)

 				ptask.AppendProxies(proxy)
 				tasks = append(tasks, task)
@ -121,7 +120,55 @@ func splitTasks(conf *Config) []ITask {

 // Execute 人的执行所有任务
 func (person *Person) Execute() {
+
+	taskLen := len(person.Tasks)
+
+	result := make(chan string, 1)
 	for _, task := range person.Tasks {
-		ExecuteOnPlan(task)
+		go ExecuteOnPlan(task, result)
+	}
+
+	for t := range result {
+		log.Println(t)
+		taskLen--
+		if taskLen <= 0 {
+			close(result)
+		}
+	}
+
+	// engine := gin.Default()
+	// engine.Run()
+}
+
+// initTask 生成一个新任务
+func initTask(conf *Config, task ITask, curl *curl2info.CURL) {
+	if curl.Crontab != "" {
+		task.SetCrontab(curl.Crontab)
+	} else {
+		task.SetCrontab(conf.Crontab)
+	}
+
+	task.SetCurl(curl)
+}
+
+// ExecuteOnPlan 按照计划执行任务并返回结果
+func ExecuteOnPlan(task ITask, result chan string) {
+	data := make(map[string]interface{})
+	var rtask ITask
+	taskname := task.GetName()
+	urlname := task.GetCurl().Name
+
+	for {
+		if task.TimeUp() {
+			rtask = task.Execute(data) // 事件 在这里变化
+			if rtask == nil {
+				// log.Println("rtask is nil")
+				result <- fmt.Sprintf("rtask is nil, the first task = %s, name = %s", taskname, urlname)
+				return
+			}
+		}
+		interval := task.NextTime().Sub(time.Now())
+		time.Sleep(interval)
+		task = rtask
 	}
 }
--- a/structure.go
+++ b/structure.go
@ -1 +1 @@
-package imitate
+package imitater
--- a/structure_test.go
+++ b/structure_test.go
@ -1 +1 @@
-package imitate
+package imitater
--- a/task.go
+++ b/task.go
@ -1,9 +1,9 @@
-package imitate
+package imitater

 import (
 	"time"

-	crontab "474420502.top/eson/crontabex"
+	"474420502.top/eson/crontabex"
 	"474420502.top/eson/curl2info"
 	"474420502.top/eson/requests"
 )
@ -12,6 +12,7 @@ import (
 type Task struct {
 	ITask

+	name     string
 	crontab  *crontab.Crontab
 	curl     *curl2info.CURL
 	workflow *requests.Workflow
@ -19,15 +20,25 @@ type Task struct {
 	proxies  YamlProxies
 }

+// SetName 任务的名字
+func (task *Task) SetName(name string) {
+	task.name = name
+}
+
+// GetName 获取任务的名字
+func (task *Task) GetName() string {
+	return task.name
+}
+
 // SetCurl 设置任务的curl信息类
 func (task *Task) SetCurl(curl *curl2info.CURL) {
 	task.curl = curl
 }

 // GetCurl 获取任务的curl信息类
-// func (task *Task) GetCurl() *curl2info.CURL {
-// 	return task.curl
-// }
+func (task *Task) GetCurl() *curl2info.CURL {
+	return task.curl
+}

 // AppendProxies 添加代理集合
 func (task *Task) AppendProxies(proxies ...string) {
@ -61,24 +72,7 @@ func (task *Task) NextTime() time.Time {
 	return task.crontab.NextTime()
 }

-// InitTask 生成一个新任务
-func InitTask(task ITask, Curl *curl2info.CURL) {
-	task.SetCurl(Curl)
-}
-
-// ExecuteOnPlan 按照计划执行任务并返回结果
-func ExecuteOnPlan(task ITask) {
-	for {
-		if task.TimeUp() {
-			task.Execute() // 事件 在这里变化
-		}
-		interval := task.NextTime().Sub(time.Now())
-		time.Sleep(interval)
-	}
-
-}
-
-// Workflow 根据现有session获取Workflow的信息, 便于设置或者更改参数, Session持久化
+// Workflow 根据persistent 是否返回现有session(or 新建 session),创建Workflow的信息, 便于设置或者更改参数, Session持久化
 func (task *Task) Workflow(persistent bool) *requests.Workflow {
 	if task.workflow == nil {
 		task.workflow = task.curl.CreateWorkflow(task.Session())
--- a/task_test.go
+++ b/task_test.go
@ -1,4 +1,4 @@
-package imitate
+package imitater

 import (
 	"log"
@ -27,14 +27,15 @@ type Toutiao struct {
 	Task
 }

-func (tt *Toutiao) Execute() {
+func (tt *Toutiao) Execute(data interface{}) ITask {
 	resp, err := tt.Request()
 	log.Println(resp, err)
+	return nil
 }

 func TestExecutePlan(t *testing.T) {

-	Register("toutiao", Toutiao{})
+	Register("toutiao", &Toutiao{})
 	person := NewPerson()
 	person.LoadConfig("test.yaml")
 	person.Execute()
--- a/test.curl
+++ b/test.curl
@ -1 +1,2 @@
-curl 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' --task toutiao -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'
+curl --name "tt-information" 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' --task toutiao -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'
+curl --name "ag-cn" 'https://appgrowing.cn/' -H 'authority: appgrowing.cn' -H 'cache-control: max-age=0' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: zh' -H 'cookie: _ga=GA1.2.1371058419.1533104518; _gid=GA1.2.1726457685.1544092110; _gat_gtag_UA_4002880_19=1' -H 'if-none-match: W/"5c088f5a-ca6"' -H 'if-modified-since: Thu, 06 Dec 2018 02:54:18 GMT' --compressed
--- a/test.yaml
+++ b/test.yaml
@ -7,6 +7,7 @@ priority : 10000

 # curls: "@test.curl" // 支持, 列表 与 单项字符串
 curls : "@test.curl"
+task: "toutiao"
 # next_do : "doothers"

 device : "eson-OnePlus"