yaml test success!
This commit is contained in:
huangsimin
commit
c0a81a3c5f
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
*.pyc
|
||||
*.vscode
|
||||
*.test
|
||||
96
config.go
Normal file
96
config.go
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
package imitate
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// Config 任务加载的默认配置
|
||||
type Config struct {
|
||||
Session int `yaml:"session"`
|
||||
Mode int `yaml:"mode"`
|
||||
Proxies []string `yaml:"proxies"`
|
||||
Retry int `yaml:"retry"`
|
||||
Priority int `yaml:"priority"`
|
||||
Curl string `yaml:"curl"`
|
||||
|
||||
ExecuteInterval `yaml:"execute_interval"`
|
||||
ExecuteAt `yaml:"execute_at"`
|
||||
|
||||
Device string `yaml:"device"`
|
||||
Platform string `yaml:"platform"`
|
||||
AreaCC string `yaml:"area_cc"`
|
||||
Channel int `yaml:"channel"`
|
||||
Media int `yaml:"media"`
|
||||
SpiderID int `yaml:"spider_id"`
|
||||
CatchAccountID int `yaml:"catch_account_id"`
|
||||
}
|
||||
|
||||
// newDefaultConfig create a default config
|
||||
func newDefaultConfig() *Config {
|
||||
conf := &Config{
|
||||
Session: 1,
|
||||
Mode: 0,
|
||||
Retry: 0,
|
||||
Priority: 10000,
|
||||
|
||||
ExecuteInterval: ExecuteInterval{
|
||||
TimeInterval: -1,
|
||||
},
|
||||
|
||||
ExecuteAt: ExecuteAt{
|
||||
Year: -1,
|
||||
Month: -1,
|
||||
Day: -1,
|
||||
Hour: -1,
|
||||
Min: -1,
|
||||
Sec: -1,
|
||||
},
|
||||
|
||||
Device: "",
|
||||
Platform: "",
|
||||
AreaCC: "",
|
||||
Channel: -1,
|
||||
Media: -1,
|
||||
SpiderID: -1,
|
||||
CatchAccountID: -1,
|
||||
}
|
||||
return conf
|
||||
}
|
||||
|
||||
// NewConfig 加载并返回Config
|
||||
func NewConfig(p string) *Config {
|
||||
f, err := os.Open(p)
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
conf := newDefaultConfig()
|
||||
|
||||
err = yaml.NewDecoder(f).Decode(conf)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// spew.Dump(conf)
|
||||
|
||||
if conf.Curl == "" {
|
||||
log.Println("the path ", p, "curl is \"\"")
|
||||
} else {
|
||||
if conf.Curl[0] == '@' {
|
||||
curlfile, err := os.Open(conf.Curl[1:])
|
||||
defer curlfile.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
curldata, err := ioutil.ReadAll(curlfile)
|
||||
conf.Curl = strings.Trim(string(curldata), "\r\n ")
|
||||
}
|
||||
}
|
||||
|
||||
return conf
|
||||
}
|
||||
89
execute_at.go
Normal file
89
execute_at.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
package imitate
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ExecuteAt 特定的时间任务 接口源自于 IExecute
|
||||
type ExecuteAt struct {
|
||||
Year int `yaml:"year"`
|
||||
Month int `yaml:"month"`
|
||||
Day int `yaml:"day"`
|
||||
Hour int `yaml:"hour"`
|
||||
Min int `yaml:"min"`
|
||||
Sec int `yaml:"sec"`
|
||||
|
||||
TriggerTime int64 // 下次的触发时间点
|
||||
StartStatus bool // 一个值判断这个时间表是否有效
|
||||
}
|
||||
|
||||
// SetStartStatus 设置执行计划是否生效
|
||||
func (ea *ExecuteAt) SetStartStatus(status bool) {
|
||||
ea.StartStatus = status
|
||||
}
|
||||
|
||||
// GetTriggerTime 获取计划的触发时间
|
||||
func (ea *ExecuteAt) GetTriggerTime() int64 {
|
||||
if ea.StartStatus {
|
||||
return ea.TriggerTime
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// TimeTo 是否到了该触发的时间
|
||||
func (ea *ExecuteAt) TimeTo() int64 {
|
||||
return time.Now().Unix() - ea.TriggerTime
|
||||
}
|
||||
|
||||
// GetStartStatus 获取计划的触发时间是否在生效
|
||||
func (ea *ExecuteAt) GetStartStatus() bool {
|
||||
return ea.StartStatus
|
||||
}
|
||||
|
||||
// CalculateTrigger 计算触发特定时间任务的时间点 执行后 可以通过GetTriggerTime确认触发时间
|
||||
func (ea *ExecuteAt) CalculateTrigger() int64 {
|
||||
now := time.Now()
|
||||
|
||||
year := ea.Year
|
||||
if ea.Year <= 0 {
|
||||
year = now.Year()
|
||||
}
|
||||
|
||||
month := time.Month(ea.Month)
|
||||
if ea.Month <= 0 {
|
||||
month = now.Month()
|
||||
}
|
||||
|
||||
day := ea.Day
|
||||
if ea.Day <= 0 {
|
||||
day = now.Day()
|
||||
}
|
||||
|
||||
hour := ea.Hour
|
||||
if ea.Hour < 0 {
|
||||
hour = now.Hour()
|
||||
}
|
||||
|
||||
min := ea.Min
|
||||
if ea.Min < 0 {
|
||||
min = now.Minute()
|
||||
}
|
||||
|
||||
sec := ea.Sec
|
||||
if ea.Sec < 0 {
|
||||
sec = now.Second()
|
||||
}
|
||||
|
||||
ea.TriggerTime = time.Date(year, time.Month(month), day, hour, min, sec, 0, time.Local).Unix()
|
||||
return ea.TriggerTime
|
||||
}
|
||||
|
||||
// FromValues 从数值 里获取执行时间表的结构
|
||||
func (ea *ExecuteAt) FromValues(year int, month int, day int, hour int, min int, sec int) {
|
||||
ea.Year = year
|
||||
ea.Month = month
|
||||
ea.Day = day
|
||||
ea.Hour = hour
|
||||
ea.Min = min
|
||||
ea.Sec = sec
|
||||
}
|
||||
48
execute_interval.go
Normal file
48
execute_interval.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package imitate
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ExecuteInterval 时间间隔的类型
|
||||
type ExecuteInterval struct {
|
||||
TimeInterval int64 `yaml:"sec"` // 时间间隔
|
||||
|
||||
TriggerTime int64 // 执行时间间隔触发时间
|
||||
StartStatus bool // 判断是否按照时间间隔执行
|
||||
}
|
||||
|
||||
// SetStartStatus 设置执行计划是否生效
|
||||
func (ei *ExecuteInterval) SetStartStatus(status bool) {
|
||||
ei.StartStatus = status
|
||||
}
|
||||
|
||||
// GetTriggerTime 获取计划的触发时间
|
||||
func (ei *ExecuteInterval) GetTriggerTime() int64 {
|
||||
if ei.StartStatus {
|
||||
return ei.TriggerTime
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// TimeTo 是否到了该触发的时间
|
||||
func (ei *ExecuteInterval) TimeTo() int64 {
|
||||
return time.Now().Unix() - ei.TriggerTime
|
||||
}
|
||||
|
||||
// GetStartStatus 获取计划的触发时间是否在生效
|
||||
func (ei *ExecuteInterval) GetStartStatus() bool {
|
||||
return ei.StartStatus
|
||||
}
|
||||
|
||||
// CalculateTrigger 计算触发特定时间任务的时间点
|
||||
func (ei *ExecuteInterval) CalculateTrigger() int64 {
|
||||
now := time.Now()
|
||||
ei.TriggerTime = now.Unix() + ei.TimeInterval
|
||||
return ei.TriggerTime
|
||||
}
|
||||
|
||||
// FromValue 生成计划表
|
||||
func (ei *ExecuteInterval) FromValue(vsleep int64) {
|
||||
ei.TimeInterval = vsleep
|
||||
}
|
||||
50
execute_plan.go
Normal file
50
execute_plan.go
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
package imitate
|
||||
|
||||
import (
|
||||
"github.com/474420502/requests"
|
||||
)
|
||||
|
||||
// type ExecuteRecord struct {
|
||||
// IsSuccess bool
|
||||
// FailCount int
|
||||
// Message []string
|
||||
// }
|
||||
|
||||
// IExecute 计划执行的时间接口
|
||||
type IExecute interface {
|
||||
SetStartStatus(status bool) // SetStart 设置执行计划是否生效
|
||||
GetStartStatus() bool // IsStart 获取计划的触发时间是否在生效
|
||||
GetTriggerTime() int64 // GetTriggerTime 获取计划的触发时间
|
||||
|
||||
TimeTo() int64 // TimeTo 是否到了该触发的时间
|
||||
|
||||
// SetSuccessStatus(status bool) // SetSuccessStatus 设置成功的状态 将记录于历史
|
||||
// History() []ExecuteRecord // History 记录一些历史, 可能会持久到数据库. 暂时不要
|
||||
CalculateTrigger() int64 // CalculateTrigger 计算触发特定时间任务的时间点
|
||||
}
|
||||
|
||||
// ExecutePlan 执行时间的计划表
|
||||
type ExecutePlan struct {
|
||||
ExecuteQueue []IExecute
|
||||
}
|
||||
|
||||
// PlanResult 执行计划表的结果
|
||||
type PlanResult struct {
|
||||
Exec IExecute
|
||||
Resp *requests.Response
|
||||
}
|
||||
|
||||
// AppendIExecute 添加执行计划任务
|
||||
func (ep *ExecutePlan) AppendIExecute(e IExecute) {
|
||||
ep.ExecuteQueue = append(ep.ExecuteQueue, e)
|
||||
}
|
||||
|
||||
// ClearIExecute 清除执行计划任务
|
||||
func (ep *ExecutePlan) ClearIExecute() {
|
||||
ep.ExecuteQueue = []IExecute{}
|
||||
}
|
||||
|
||||
// CountIExecute 清除执行计划任务
|
||||
func (ep *ExecutePlan) CountIExecute() int {
|
||||
return len(ep.ExecuteQueue)
|
||||
}
|
||||
43
execute_test.go
Normal file
43
execute_test.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package imitate
|
||||
|
||||
import (
|
||||
"log"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
|
||||
"474420502.top/eson/curl2info"
|
||||
)
|
||||
|
||||
type Person struct {
|
||||
Tasks []Task
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
GURL *curl2info.CURL
|
||||
Plan *ExecutePlan
|
||||
}
|
||||
|
||||
func TestNewYaml(t *testing.T) {
|
||||
data := spew.Sdump(NewConfig("test.yaml"))
|
||||
if !(regexp.MustCompile(`Device: \(string\) \(len=12\) "eson-OnePlus"`).MatchString(data) && regexp.MustCompile(`Sec: \(int\) 30`).MatchString(data)) {
|
||||
t.Error(data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecute(t *testing.T) {
|
||||
curl := `curl 'https://re.csdn.net/csdnbi' -H 'Origin: https://blog.csdn.net' -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: zh' -H 'User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' -H 'Content-Type: text/plain;charset=UTF-8' -H 'Accept: */*' -H 'Referer: https://blog.csdn.net/levon2018/article/details/80558108' -H 'Cookie: uuid_tt_dd=10_30744014340-1533110459408-320601; smidV2=2018082914272667751d1cac24793525732f74f62cbe9a0099f94a08e9b7de0; dc_session_id=10_1536825785870.531052; ARK_ID=JSa3a7b5f009df2bf82fa457adf0741aaaa3a7; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1538100545,1538118577,1538119202,1538121038; dc_tos=pfrclv; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1538124692' -H 'Connection: keep-alive' --data-binary '[{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fxixi880928%2Farticle%2Fdetails%2F78339157%2CBlogCommendFromBaidu_2&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fchaowanghn%2Farticle%2Fdetails%2F54601024%2CBlogCommendFromBaidu_3&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Flanchunhui%2Farticle%2Fdetails%2F78877514%2CBlogCommendFromBaidu_4&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fsaw1990%2Farticle%2Fdetails%2F39287631%2CBlogCommendFromBaidu_5&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fu011044684%2Farticle%2Fdetails%2F45287575%2CBlogCommendFromBaidu_6&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fnimade511%2Farticle%2Fdetails%2F52540437%2CBlogCommendFromBaidu_7&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fpipisorry%2Farticle%2Fdetails%2F47948065%2CBlogCommendFromBaidu_8&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fyudiyanwang%2Farticle%2Fdetails%2F71843633%2CBlogCommendFromBaidu_9&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"}]'`
|
||||
|
||||
u, err := curl2info.ParseRawCURL(curl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
resp, err := u.CreateWorkflow(nil).Execute()
|
||||
if err != nil {
|
||||
t.Error("TestExecute")
|
||||
} else {
|
||||
log.Println(resp.Content())
|
||||
}
|
||||
}
|
||||
1
test.curl
Normal file
1
test.curl
Normal file
|
|
@ -0,0 +1 @@
|
|||
curl 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'
|
||||
29
test.yaml
Normal file
29
test.yaml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
session : 1
|
||||
mode : 0
|
||||
proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8089", "socks5://10.10.10.1:8090"]
|
||||
retry : 0
|
||||
priority : 10000
|
||||
|
||||
curl : "@test.curl"
|
||||
# next_do : "doothers"
|
||||
|
||||
|
||||
|
||||
device : "eson-OnePlus"
|
||||
platform : "Android"
|
||||
area_cc : 4401
|
||||
channel : 105
|
||||
media : 55
|
||||
spider_id : 73
|
||||
catch_account_id : 123
|
||||
|
||||
execute_interval: # 时间间隔执行
|
||||
sec: 2
|
||||
|
||||
execute_at : # (-1, -1, -1, -1, -1, 30) 时间 每30秒执行 min 设置20的时候 = 每小时的20分钟30秒执行
|
||||
year : -1
|
||||
month : -1
|
||||
day : -1
|
||||
hour : -1
|
||||
min : -1
|
||||
sec : 30
|
||||
Loading…
Reference in New Issue
Block a user