Compare commits

..

No commits in common. "master" and "develop" have entirely different histories.

254 changed files with 112 additions and 186708 deletions

1
.gitignore vendored
View File

@ -1 +0,0 @@
screenlog.*

173
Gopkg.lock generated
View File

@ -1,173 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:652aa16a70faa85c967dc8b720bb700857036101cfa0dba21bdeaff9dfcf4db6"
name = "474420502.top/eson/crontabex"
packages = ["."]
pruneopts = "UT"
revision = "f83fd9f635474c304964ccdbe3acad5575fa4011"
version = "v2.0.2"
[[projects]]
digest = "1:90d03ae541869192608680de2cb266a29c6570a7d4fcbadfbf5274aca735b46a"
name = "474420502.top/eson/curl2info"
packages = ["."]
pruneopts = "UT"
revision = "478ceec906c3c3a36503d317369d45b0527da3d5"
version = "v1.0.3"
[[projects]]
branch = "eson"
digest = "1:f41d5b32a2870c5881df40a083f9d3ded497b23db137530005910cb2d3370c43"
name = "474420502.top/eson/gjson"
packages = ["."]
pruneopts = "UT"
revision = "ea73b4a9b9dcdd7cea42b43172e0ad6e09b1f599"
[[projects]]
digest = "1:30d6e71057faffc6696eeb803c3b0650bd99b1cbaaba339b2c12a78df1b62462"
name = "474420502.top/eson/imitater"
packages = ["."]
pruneopts = "UT"
revision = "69ea336e44c26c301aae3fe9e25746b6d29df864"
version = "v1.0.0"
[[projects]]
digest = "1:d46c560fc51485079413fc65e561adf4f96cd5e8e5c791f7ce5567e0c4f19921"
name = "474420502.top/eson/requests"
packages = ["."]
pruneopts = "UT"
revision = "965d1e6d16ae595fa6290744b0fc326a0ea83256"
version = "v1.0.1"
[[projects]]
digest = "1:7089ee8d60da4d122f6210df633eb886c8701aad0cab686e67ba099600c4c3be"
name = "474420502.top/eson/structure"
packages = [
"circular_linked",
"priority_list",
]
pruneopts = "UT"
revision = "fd4d47445f8c3f0a6e200be8996c984909f065ed"
version = "v1.0.0"
[[projects]]
digest = "1:ae83a71f9ccbcb9878c0396f0b42035f9f1b3edc466c783dcd17882d30290db1"
name = "474420502.top/test/logdb"
packages = ["."]
pruneopts = "UT"
revision = "06073e692c34dcd63b4867b369ddec0cc2388c96"
version = "v1.0.1"
[[projects]]
digest = "1:ddaeea4ba16546053455f7139c27b9efe16a6a739353043cc2af76a9437d73b2"
name = "github.com/Pallinder/go-randomdata"
packages = ["."]
pruneopts = "UT"
revision = "97a2356fcab20708fb8ae46cbbf69a5bc9feca63"
version = "v1.1.0"
[[projects]]
digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = "UT"
revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
version = "v1.1.1"
[[projects]]
digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65"
name = "github.com/go-sql-driver/mysql"
packages = ["."]
pruneopts = "UT"
revision = "72cd26f257d44c1114970e19afddcd812016007e"
version = "v1.4.1"
[[projects]]
branch = "master"
digest = "1:a3b8912deeef29007fab9a13a9f21b9e9b59c621a2ed61e2fe7b37320a71fbd5"
name = "github.com/satori/go.uuid"
packages = ["."]
pruneopts = "UT"
revision = "b2ce2384e17bbe0c6d34077efa39dbab3e09123b"
[[projects]]
digest = "1:2cab41f59638fdb77dda96ab4f9b5860ef30f48967557254eba127e43c756f2e"
name = "github.com/tidwall/gjson"
packages = ["."]
pruneopts = "UT"
revision = "1e3f6aeaa5bad08d777ea7807b279a07885dd8b2"
version = "v1.1.3"
[[projects]]
digest = "1:8453ddbed197809ee8ca28b06bd04e127bec9912deb4ba451fea7a1eca578328"
name = "github.com/tidwall/match"
packages = ["."]
pruneopts = "UT"
revision = "33827db735fff6510490d69a8622612558a557ed"
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:b6a3256ffb6ee9d7e7159c062f57e091c30c19e13ac3c676729baac7d83ecd21"
name = "golang.org/x/net"
packages = [
"idna",
"publicsuffix",
]
pruneopts = "UT"
revision = "927f97764cc334a6575f4b7a1584a147864d5723"
[[projects]]
digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
name = "golang.org/x/text"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable",
]
pruneopts = "UT"
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3"
name = "google.golang.org/appengine"
packages = ["cloudsql"]
pruneopts = "UT"
revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1"
version = "v1.4.0"
[[projects]]
digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = "UT"
revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
version = "v2.2.2"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"474420502.top/eson/crontabex",
"474420502.top/eson/gjson",
"474420502.top/eson/imitater",
"474420502.top/test/logdb",
"github.com/Pallinder/go-randomdata",
"github.com/tidwall/gjson",
]
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -1,59 +0,0 @@
# Gopkg.toml example
#
# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]
name = "474420502.top/eson/crontabex"
version = "2.0.2"
[[constraint]]
name = "474420502.top/eson/gjson"
branch = "eson"
# version = "e1.0.1"
[[constraint]]
name = "474420502.top/eson/imitater"
version = "1.0.0"
[[constraint]]
name = "474420502.top/test/logdb"
version = "1.0.1"
[[constraint]]
name = "github.com/Pallinder/go-randomdata"
version = "1.1.0"
[[constraint]]
name = "github.com/tidwall/gjson"
version = "1.1.3"
[[override]]
name = "github.com/satori/go.uuid"
branch = "master"
[prune]
go-tests = true
unused-packages = true

View File

@ -2,8 +2,9 @@ package main
import (
"log"
"regexp"
"474420502.top/eson/gjson"
"github.com/tidwall/gjson"
crontab "474420502.top/eson/crontabex"
"474420502.top/eson/imitater"
@ -38,28 +39,19 @@ func (te *TaskEx) Execute(glocal map[string]interface{}) imitater.ITask {
}
var adDataList []string
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`ordered_info.#.ad_data`)
if adData.Exists() {
for _, result := range adData.Array() {
rule := regexp.MustCompile(`\\"label\\"\:\\"广告\\"|\\"label_style\\"\:3`)
if rule.MatchString(resp.Content()) {
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
gADDate := P.Get(`data.ordered_info.#.ad_data`)
if gADDate.Exists() {
for _, result := range gADDate.Array() {
adDataList = append(adDataList, result.String())
}
}
relatedVideo := data.Get(`related_video_toutiao.#[show_tag = "广告"]#`)
if adData.Exists() {
for _, result := range relatedVideo.Array() {
adDataList = append(adDataList, result.String())
}
}
} else {
log.Println("be careful:", resp.Content())
}
} else {
log.Println("be careful:", resp.Content())
}
if imitater.ADDataSave(te, te.db, adDataList) {

View File

@ -2,45 +2,31 @@ package main
import (
"io/ioutil"
"log"
"regexp"
"testing"
"github.com/tidwall/gjson"
)
func TestTaskEx_Execute(t *testing.T) {
data, err := ioutil.ReadFile("../ssss.json")
rule := regexp.MustCompile(`\\"label\\"\:\\"广告\\"|\\"label_style\\"\:3`)
data, err := ioutil.ReadFile("../s.json")
if err != nil {
panic(err)
}
log.Println(rule.MatchString(string(data)))
log.Println(gjson.Valid(string(data)))
var adDataList []string
if gjson.Valid(string(data)) {
P := gjson.Parse(string(data))
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`ordered_info.#.ad_data`)
if adData.Exists() {
for _, result := range adData.Array() {
adDataList = append(adDataList, result.String())
}
} else {
t.Error(`ordered_info.#.ad_data not exists`)
}
P := gjson.Parse(string(data))
gADDate := P.Get(`data.ordered_info.#.ad_data`)
log.Println(gADDate.Exists(), gADDate.IsArray())
relatedVideo := data.Get(`related_video_toutiao.#[show_tag = "广告"]#`)
if adData.Exists() {
for _, result := range relatedVideo.Array() {
adDataList = append(adDataList, result.String())
}
} else {
t.Error(`related_video_toutiao.#[show_tag = "广告"]# not exists`)
}
}
if !(gADDate.Exists() && gADDate.IsArray()) {
t.Error("path is error")
}
if len(adDataList) != 2 {
t.Error("adDataList != 2")
for _, result := range gADDate.Array() {
log.Println(gjson.Valid(result.Raw))
}
}

View File

@ -1,6 +1,3 @@
curl --name "tt-article-v24" --task "article" 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com'
curl --name "tt-article-v25" --task "article" 'http://is.snssdk.com/2/article/information/v25/?latitude=23.060406&longitude=113.380702&group_id=6616903985464869383&item_id=6616903985464869383&aggr_type=1&context=1&flags=64&from_category=__all__&article_page=1&search_id=&query=&is_low_actived=0&iid=54363182959&device_id=55149394150&ac=wifi&channel=xiaomi&aid=13&app_name=news_article&version_code=703&version_name=7.0.3&device_platform=android&ab_version=664543%2C659641%2C640301%2C486952%2C654085%2C651369%2C662176%2C472113%2C292723%2C664015%2C571131%2C616214%2C665141%2C639002%2C239095%2C612191%2C641905%2C170988%2C643891%2C631016%2C659018%2C374117%2C652002%2C600231%2C664393%2C587817%2C655402%2C649422%2C633720%2C613176%2C550042%2C603542%2C659212%2C627550%2C663332%2C659530%2C649427%2C614100%2C522765%2C659286%2C416055%2C621360%2C651696%2C661405%2C558140%2C555254%2C640008%2C471406%2C603440%2C596392%2C660510%2C630576%2C598626%2C644857%2C646252%2C603379%2C603401%2C603404%2C603405%2C638928%2C662915%2C659552%2C646564%2C648850%2C656461%2C657631%2C661899%2C662644%2C629152%2C607361%2C567866%2C609337%2C662818%2C652366%2C662099%2C641414%2C664191%2C654358%2C651920%2C662280%2C655010%2C650887%2C622716%2C662145%2C622728%2C653957%2C659453%2C665452%2C640997%2C641080%2C661616%2C660462%2C664030%2C662397%2C631594%2C653241%2C658364%2C554836%2C549647%2C644131%2C472442%2C31210%2C572465%2C651597%2C644058%2C615291%2C606547%2C442255%2C660075%2C648180%2C630218%2C546701%2C281298%2C658129%2C622045%2C325619%2C665473%2C625066%2C652952%2C665098%2C659705%2C664311%2C431142%2C498375%2C657352%2C638335%2C467513%2C644240%2C631638%2C665565%2C655235%2C595556%2C664318%2C664673%2C661444%2C661450%2C654129%2C655710%2C658938%2C660830%2C648316%2C640210%2C656557%2C644675%2C662684%2C661781%2C293032%2C457481%2C649402%2C591908%2C655988&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167&ab_feature=94563%2C102749&abflag=3&ssmix=a&device_type=MI+8+SE&device_brand=Xiaomi&language=zh&os_api=27&os_version=8.1.0&uuid=99001141462873&openudid=58cccf5030e9cbd4&manifest_version_code=703&resolution=1080*2114&dpi=440&update_version_code=70315&_rticket=1545589377362&plugin=26958&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zrqWtqq2vsb_x_On06ej5-L-nr66zrautqa2rsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOprailq6mxv_zw_O3R_On06ej5-L-nr66zraiqr66r4A%3D%3D&fp=zlT_L2UeLlPrFlPrPrU1FYwIJlm1&tma_jssdk_version=1.7.1.4&rom_version=miui_v10_v10.0.1.0.oebcnfh&ts=1545589377&as=a2e55d11a1d89ca21f4355&mas=00175b1e44fd41af3b097c34ff95e7ea7a2c40a64206686ef6' -H 'Host: is.snssdk.com' -H 'Proxy-Connection: keep-alive' -H 'Cookie: odin_tt=5bc108effff4529886a0aa36812c115d38b0e739f9abff2139107953c017cc62343bab05794f7fb3075b25846f32c178; UM_distinctid=1655625f84a18e-0897e6c8fc225e-6b55284e-4e4b0-1655625f84c3b; __tea_sdk__ssid=a79e00a6-b717-423f-9e97-f8ac84635f7e; tt_webid=6591877343492081160; __tea_sdk__user_unique_id=6591877343492081160; CNZZDATA1264530760=1269028608-1534945655-%7C1534945655; _ga=GA1.2.1032699944.1534947531; sid_guard=5ccb73ded741fb5f56e64623ca03ac8b%7C1544665500%7C5184000%7CMon%2C+11-Feb-2019+01%3A45%3A00+GMT; uid_tt=1f76f0742e12b4f8a5f32c009d5f7c52; sid_tt=5ccb73ded741fb5f56e64623ca03ac8b; sessionid=5ccb73ded741fb5f56e64623ca03ac8b; alert_coverage=77; install_id=54363182959; ttreq=1$05885f6b61aaffcd5e901e181629f2e1d2851d21; qh[360]=1' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1545589377369' -H 'X-Tt-Token: 005ccb73ded741fb5f56e64623ca03ac8bdc236ae81584d4b02df898418e84e101852bddfc64ca324cdf12e2a5ea1d924039' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 SE MIUI/V10.0.1.0.OEBCNFH) NewsArticle/7.0.3 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'
curl -X 'GET' 'http://lf.snssdk.com/2/article/information/v25/?group_id=6631787616238829828&item_id=6631787616238829828&aggr_type=1&context=1&ad_id=1619088773454899&from_category=__all__&article_page=0&search_id=&query=&is_low_actived=0&iid=56493295608&device_id=53061747912&ac=wifi&channel=meizu&aid=13&app_name=news_article&version_code=704&version_name=7.0.4&device_platform=android&ab_version=679615%2C664543%2C680643%2C666150%2C486952%2C651365%2C662176%2C680305%2C675827%2C571131%2C665174%2C674057%2C594583%2C612193%2C641906%2C170988%2C680911%2C643892%2C659018%2C374116%2C655402%2C678350%2C550042%2C435216%2C603544%2C659218%2C680163%2C678792%2C649426%2C614096%2C677129%2C377573%2C522766%2C669470%2C416055%2C677236%2C558139%2C555254%2C679178%2C603441%2C596391%2C660508%2C598626%2C644855%2C680371%2C603386%2C603398%2C603404%2C603406%2C638927%2C681323%2C661907%2C662644%2C668775%2C673945%2C629152%2C607361%2C609337%2C666967%2C635530%2C652363%2C662099%2C641413%2C673358%2C664194%2C654355%2C669191%2C671245%2C681071%2C680507%2C667071%2C622716%2C672874%2C680352%2C640997%2C641077%2C671798%2C668774%2C679516%2C631595%2C469022%2C676441%2C679733%2C673696%2C554836%2C679903%2C549647%2C644131%2C572465%2C482355%2C644057%2C615291%2C606548%2C681182%2C678933%2C673168%2C678315%2C678279%2C671426%2C546703%2C641191%2C281297%2C664118%2C325620%2C678473%2C665472%2C625066%2C662507%2C663955%2C668676%2C664311%2C680283%2C638335%2C467515%2C679101%2C679985%2C678876%2C595556%2C679519%2C679764%2C669426%2C670151%2C661450%2C654124%2C680983%2C660830%2C679538%2C656557%2C633487%2C662683%2C661781%2C457481%2C649400%2C655989%2C677104&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167%2C94567%2C102754%2C181431&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=U20&device_brand=Meizu&language=zh&os_api=23&os_version=6.0&uuid=861578036046061&openudid=d67c17290ee19a25&manifest_version_code=704&resolution=1080*1920&dpi=480&update_version_code=70412&_rticket=1546587620850&fp=2ST_F255JlQZFlcuJ2U1FYmeFzGS&tma_jssdk_version=1.8.0.4&pos=5r_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupaukqKmxv_zw_O3R_On06ej5-L-nr66zrairqqWo4A%3D%3D&rom_version=23&plugin=26958&ts=1546587620&as=a2151082346e9cedff4355&mas=000db3e5e0c75dc76d2fe5c4ef01f5baf5440c084206686ee0' -H 'Host: lf.snssdk.com' -H 'Proxy-Connection: keep-alive' -H 'Cookie: odin_tt=b8dec487941fe35d2148bf9724de5e6659e65cf98541eee4f3573236e60d5f3c02a83ba067b8cd80a6685ac034db06ac; UM_distinctid=1653b99a5601-0c98c05be-2d4e2043-38400-1653b99a5641a; CNZZDATA1264530760=838951947-1534301713-%7C1534748760; __tea_sdk__ssid=290d7879-cb91-4e4c-aecd-b6239e2f04dc; tt_webid=6598313444424123907; __tea_sdk__user_unique_id=6598313444424123907; sid_guard=897e58fa1fa260bad70d802cbeee9225%7C1536824928%7C5184000%7CMon%2C+12-Nov-2018+07%3A48%3A48+GMT; CNZZDATA1263676333=575629806-1537840457-null%7C1537840457; qh[360]=1; install_id=56493295608; ttreq=1$b75d7a1dcbdafb18503a62139b7d79dc7aeeddb2' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1546587620857' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 6.0; U20 Build/MRA58K) NewsArticle/7.0.4 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'

View File

@ -1,6 +1,6 @@
mode : 0
# proxies : "socks5://10.10.0.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.0.1:8080", "socks5://10.10.0.1:8082", "socks5://10.10.0.1:8083", "socks5://10.10.0.1:8085", "socks5://10.10.0.1:8087", "socks5://10.10.0.1:8088", "socks5://10.10.0.1:8091"]
# proxies : "socks5://10.10.10.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8091"]
retry : 0
timeout: 12
priority : 10000

View File

@ -1 +0,0 @@
../vendor

52
glide.lock generated Normal file
View File

@ -0,0 +1,52 @@
hash: 0c58405d25d12251bdcdc8916b835c0e1150c624e86778a7993356b50e6a80cb
updated: 2018-12-23T18:54:39.299055148+08:00
imports:
- name: 474420502.top/eson/crontabex
version: b7f0209cc399e87215b0b082e3c4e83a0d7b3a9c
repo: http://474420502.top/eson/crontabex
- name: 474420502.top/eson/curl2info
version: 478ceec906c3c3a36503d317369d45b0527da3d5
repo: http://474420502.top/eson/curl2info
- name: 474420502.top/eson/gjson
version: 081192fa2e471ddff3ec98a3cc04fecfd031ed57
repo: http://474420502.top/eson/gjson
- name: 474420502.top/eson/imitater
version: 69ea336e44c26c301aae3fe9e25746b6d29df864
repo: http://474420502.top/eson/imitater
- name: 474420502.top/eson/requests
version: 965d1e6d16ae595fa6290744b0fc326a0ea83256
repo: http://474420502.top/eson/requests
- name: 474420502.top/eson/structure
version: fd4d47445f8c3f0a6e200be8996c984909f065ed
repo: http://474420502.top/eson/structure
subpackages:
- circular_linked
- priority_list
- name: 474420502.top/test/logdb
version: 2c996115f03d84733739c6743b057f1ded3ca390
repo: http://474420502.top/test/logdb
- name: github.com/davecgh/go-spew
version: d8f796af33cc11cb798c1aaeb27a4ebc5099927d
subpackages:
- spew
- name: github.com/go-sql-driver/mysql
version: c45f530f8e7fe40f4687eaa50d0c8c5f1b66f9e0
- name: github.com/Pallinder/go-randomdata
version: 97a2356fcab20708fb8ae46cbbf69a5bc9feca63
- name: github.com/satori/go.uuid
version: b2ce2384e17bbe0c6d34077efa39dbab3e09123b
- name: github.com/tidwall/gjson
version: 081192fa2e471ddff3ec98a3cc04fecfd031ed57
- name: github.com/tidwall/match
version: 33827db735fff6510490d69a8622612558a557ed
- name: golang.org/x/net
version: 927f97764cc334a6575f4b7a1584a147864d5723
subpackages:
- publicsuffix
- name: google.golang.org/appengine
version: e9657d882bb81064595ca3b56cbe2546bbabf7b1
subpackages:
- cloudsql
- name: gopkg.in/yaml.v2
version: 51d6538a90f86fe93ac480b35f37b2be17fef232
testImports: []

17
glide.yaml Normal file
View File

@ -0,0 +1,17 @@
package: .
import:
- package: src/474420502.top/eson/crontabex
repo: http://474420502.top/eson/crontabex
- package: src/474420502.top/eson/imitater
repo: http://474420502.top/eson/imitater
- package: src/474420502.top/test/logdb
repo: http://474420502.top/test/logdb
- package: src/474420502.top/eson/gjson
repo: http://474420502.top/eson/gjson
version: eson
- package: src/474420502.top/eson/structure
repo: http://474420502.top/eson/structure
- package: src/474420502.top/eson/curl2info
repo: http://474420502.top/eson/curl2info
- package: src/474420502.top/eson/requests
repo: http://474420502.top/eson/requests

View File

@ -1,72 +0,0 @@
package main
import (
"log"
"474420502.top/eson/gjson"
crontab "474420502.top/eson/crontabex"
"474420502.top/eson/imitater"
"474420502.top/test/logdb"
)
func main() {
imitater.Register("toutiao", &TaskEx{})
person := imitater.NewPerson()
person.Config("task.yaml")
person.Execute()
}
// TaskEx 任务相关类
type TaskEx struct {
imitater.Task
db *logdb.LogDB
}
// Init 初始化函数
func (te *TaskEx) Init() {
te.db = logdb.New("../logdb.yaml")
}
// Execute 执行过程的方法
func (te *TaskEx) Execute(glocal map[string]interface{}) imitater.ITask {
resp, err := te.Request()
if err != nil {
log.Println(err)
return te
}
var adDataList []string
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`ordered_info.#.ad_data`)
if adData.Exists() {
for _, result := range adData.Array() {
adDataList = append(adDataList, result.String())
}
}
relatedVideo := data.Get(`related_video_toutiao.#[show_tag = "广告"]#`)
if adData.Exists() {
for _, result := range relatedVideo.Array() {
adDataList = append(adDataList, result.String())
}
}
}
} else {
log.Println("be careful:", resp.Content())
}
if imitater.ADDataSave(te, te.db, adDataList) {
te.GetCrontab().SetStatus(crontab.SExecuteOK, true)
} else {
te.GetCrontab().SetStatus(crontab.SExecuteOK, false)
}
return te
}

View File

@ -1 +1 @@
curl --name 'tt-image' --task toutiao 'http://is.snssdk.com/2/article/information/v25/?latitude=23.060406&longitude=113.380702&group_id=6636256062225777160&item_id=6636256062225777160&aggr_type=1&context=1&from_category=%E7%BB%84%E5%9B%BE&article_page=2&search_id=&query=&is_low_actived=0&iid=54363182959&device_id=55149394150&ac=wifi&channel=xiaomi&aid=13&app_name=news_article&version_code=703&version_name=7.0.3&device_platform=android&ab_version=664543%2C659641%2C640301%2C486952%2C654085%2C651369%2C662176%2C472113%2C292723%2C664015%2C571131%2C616214%2C665141%2C639002%2C239095%2C612191%2C641905%2C170988%2C643891%2C631016%2C659018%2C374117%2C652002%2C600231%2C664393%2C587817%2C655402%2C649422%2C633720%2C613176%2C550042%2C603542%2C659212%2C627550%2C663332%2C659530%2C649427%2C614100%2C522765%2C659286%2C416055%2C621360%2C651696%2C661405%2C558140%2C555254%2C640008%2C471406%2C603440%2C596392%2C660510%2C630576%2C598626%2C644857%2C646252%2C603379%2C603401%2C603404%2C603405%2C638928%2C662915%2C659552%2C646564%2C648850%2C656461%2C657631%2C661899%2C662644%2C629152%2C607361%2C567866%2C609337%2C662818%2C652366%2C662099%2C641414%2C664191%2C654358%2C651920%2C662280%2C655010%2C650887%2C622716%2C662145%2C622728%2C653957%2C659453%2C665452%2C640997%2C641080%2C661616%2C660462%2C664030%2C662397%2C631594%2C653241%2C658364%2C554836%2C549647%2C644131%2C472442%2C31210%2C572465%2C651597%2C644058%2C615291%2C606547%2C442255%2C660075%2C648180%2C630218%2C546701%2C281298%2C658129%2C622045%2C325619%2C665473%2C625066%2C652952%2C665098%2C659705%2C664311%2C431142%2C498375%2C657352%2C638335%2C467513%2C644240%2C631638%2C665565%2C655235%2C595556%2C664318%2C664673%2C661444%2C661450%2C654129%2C655710%2C658938%2C660830%2C648316%2C640210%2C656557%2C644675%2C662684%2C661781%2C293032%2C457481%2C649402%2C591908%2C655988&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167&ab_feature=94563%2C102749&abflag=3&ssmix=a&device_type=MI+8+SE&device_brand=Xiaomi&language=zh&os_api=27&os_version=8.1.0&uuid=99001141462873&openudid=58cccf5030e9cbd4&manifest_version_code=703&resolution=1080*2114&dpi=440&update_version_code=70315&_rticket=1545592651899&plugin=26958&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zrqWtqq2vsb_x_On06ej5-L-nr66zrautqa2rsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOprailq6mxv_zw_O3R_On06ej5-L-nr66zraiqr66r4A%3D%3D&fp=zlT_L2UeLlPrFlPrPrU1FYwIJlm1&tma_jssdk_version=1.7.1.4&rom_version=miui_v10_v10.0.1.0.oebcnfh&ts=1545592651&as=a2950da1db843c1f3f4355&mas=00b3a3d12745a68d7285a73f411d8f9c89a6a2a24206686ea7' -H 'Host: is.snssdk.com' -H 'Proxy-Connection: keep-alive' -H 'Cookie: odin_tt=5bc108effff4529886a0aa36812c115d38b0e739f9abff2139107953c017cc62343bab05794f7fb3075b25846f32c178; UM_distinctid=1655625f84a18e-0897e6c8fc225e-6b55284e-4e4b0-1655625f84c3b; __tea_sdk__ssid=a79e00a6-b717-423f-9e97-f8ac84635f7e; tt_webid=6591877343492081160; __tea_sdk__user_unique_id=6591877343492081160; CNZZDATA1264530760=1269028608-1534945655-%7C1534945655; _ga=GA1.2.1032699944.1534947531; sid_guard=5ccb73ded741fb5f56e64623ca03ac8b%7C1544665500%7C5184000%7CMon%2C+11-Feb-2019+01%3A45%3A00+GMT; uid_tt=1f76f0742e12b4f8a5f32c009d5f7c52; sid_tt=5ccb73ded741fb5f56e64623ca03ac8b; sessionid=5ccb73ded741fb5f56e64623ca03ac8b; alert_coverage=77; install_id=54363182959; ttreq=1$05885f6b61aaffcd5e901e181629f2e1d2851d21; qh[360]=1' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1545592651908' -H 'X-Tt-Token: 005ccb73ded741fb5f56e64623ca03ac8bdc236ae81584d4b02df898418e84e101852bddfc64ca324cdf12e2a5ea1d924039' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 SE MIUI/V10.0.1.0.OEBCNFH) NewsArticle/7.0.3 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'
curl 'http://is.snssdk.com/2/article/information/v25/?latitude=23.060406&longitude=113.380702&group_id=6636256062225777160&item_id=6636256062225777160&aggr_type=1&context=1&from_category=%E7%BB%84%E5%9B%BE&article_page=2&search_id=&query=&is_low_actived=0&iid=54363182959&device_id=55149394150&ac=wifi&channel=xiaomi&aid=13&app_name=news_article&version_code=703&version_name=7.0.3&device_platform=android&ab_version=664543%2C659641%2C640301%2C486952%2C654085%2C651369%2C662176%2C472113%2C292723%2C664015%2C571131%2C616214%2C665141%2C639002%2C239095%2C612191%2C641905%2C170988%2C643891%2C631016%2C659018%2C374117%2C652002%2C600231%2C664393%2C587817%2C655402%2C649422%2C633720%2C613176%2C550042%2C603542%2C659212%2C627550%2C663332%2C659530%2C649427%2C614100%2C522765%2C659286%2C416055%2C621360%2C651696%2C661405%2C558140%2C555254%2C640008%2C471406%2C603440%2C596392%2C660510%2C630576%2C598626%2C644857%2C646252%2C603379%2C603401%2C603404%2C603405%2C638928%2C662915%2C659552%2C646564%2C648850%2C656461%2C657631%2C661899%2C662644%2C629152%2C607361%2C567866%2C609337%2C662818%2C652366%2C662099%2C641414%2C664191%2C654358%2C651920%2C662280%2C655010%2C650887%2C622716%2C662145%2C622728%2C653957%2C659453%2C665452%2C640997%2C641080%2C661616%2C660462%2C664030%2C662397%2C631594%2C653241%2C658364%2C554836%2C549647%2C644131%2C472442%2C31210%2C572465%2C651597%2C644058%2C615291%2C606547%2C442255%2C660075%2C648180%2C630218%2C546701%2C281298%2C658129%2C622045%2C325619%2C665473%2C625066%2C652952%2C665098%2C659705%2C664311%2C431142%2C498375%2C657352%2C638335%2C467513%2C644240%2C631638%2C665565%2C655235%2C595556%2C664318%2C664673%2C661444%2C661450%2C654129%2C655710%2C658938%2C660830%2C648316%2C640210%2C656557%2C644675%2C662684%2C661781%2C293032%2C457481%2C649402%2C591908%2C655988&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167&ab_feature=94563%2C102749&abflag=3&ssmix=a&device_type=MI+8+SE&device_brand=Xiaomi&language=zh&os_api=27&os_version=8.1.0&uuid=99001141462873&openudid=58cccf5030e9cbd4&manifest_version_code=703&resolution=1080*2114&dpi=440&update_version_code=70315&_rticket=1545592651899&plugin=26958&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zrqWtqq2vsb_x_On06ej5-L-nr66zrautqa2rsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOprailq6mxv_zw_O3R_On06ej5-L-nr66zraiqr66r4A%3D%3D&fp=zlT_L2UeLlPrFlPrPrU1FYwIJlm1&tma_jssdk_version=1.7.1.4&rom_version=miui_v10_v10.0.1.0.oebcnfh&ts=1545592651&as=a2950da1db843c1f3f4355&mas=00b3a3d12745a68d7285a73f411d8f9c89a6a2a24206686ea7' -H 'Host: is.snssdk.com' -H 'Proxy-Connection: keep-alive' -H 'Cookie: odin_tt=5bc108effff4529886a0aa36812c115d38b0e739f9abff2139107953c017cc62343bab05794f7fb3075b25846f32c178; UM_distinctid=1655625f84a18e-0897e6c8fc225e-6b55284e-4e4b0-1655625f84c3b; __tea_sdk__ssid=a79e00a6-b717-423f-9e97-f8ac84635f7e; tt_webid=6591877343492081160; __tea_sdk__user_unique_id=6591877343492081160; CNZZDATA1264530760=1269028608-1534945655-%7C1534945655; _ga=GA1.2.1032699944.1534947531; sid_guard=5ccb73ded741fb5f56e64623ca03ac8b%7C1544665500%7C5184000%7CMon%2C+11-Feb-2019+01%3A45%3A00+GMT; uid_tt=1f76f0742e12b4f8a5f32c009d5f7c52; sid_tt=5ccb73ded741fb5f56e64623ca03ac8b; sessionid=5ccb73ded741fb5f56e64623ca03ac8b; alert_coverage=77; install_id=54363182959; ttreq=1$05885f6b61aaffcd5e901e181629f2e1d2851d21; qh[360]=1' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1545592651908' -H 'X-Tt-Token: 005ccb73ded741fb5f56e64623ca03ac8bdc236ae81584d4b02df898418e84e101852bddfc64ca324cdf12e2a5ea1d924039' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 SE MIUI/V10.0.1.0.OEBCNFH) NewsArticle/7.0.3 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0' --compressed

View File

@ -1,19 +0,0 @@
mode : 0
# proxies : "socks5://10.10.0.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.0.1:8080", "socks5://10.10.0.1:8082", "socks5://10.10.0.1:8083", "socks5://10.10.0.1:8085", "socks5://10.10.0.1:8087", "socks5://10.10.0.1:8088", "socks5://10.10.0.1:8091"]
retry : 0
timeout: 12
priority : 10000
curls : "@task.curl" # 支持, 列表 与 单项字符串
task: "toutiao"
device : "eson-OnePlus"
platform : "Android"
# area_cc : 4401
channel : 105
media : 55
spider_id : 10001001
catch_account_id : 1001001
crontab: "f8>=240|f5>=120|f=60|s=5|s10>=10|s20>=5"

View File

@ -1 +0,0 @@
../vendor

View File

@ -2,6 +2,7 @@ package main
import (
"log"
"regexp"
crontab "474420502.top/eson/crontabex"
"474420502.top/eson/gjson"
@ -36,17 +37,19 @@ func (te *TaskEx) Execute(glocal map[string]interface{}) imitater.ITask {
}
var adDataList []string
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
gADDate := P.Get(`data.#[content ~ @"label":"广告"@ ]#.content`)
if gADDate.Exists() {
for _, result := range gADDate.Array() {
adDataList = append(adDataList, result.String())
rule := regexp.MustCompile(`\\"label\\"\:\\"广告\\"|\\"label_style\\"\:3`)
if rule.MatchString(resp.Content()) {
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
gADDate := P.Get(`data.#[content ~ @label_style\":3@ ]#.content`)
if gADDate.Exists() {
for _, result := range gADDate.Array() {
adDataList = append(adDataList, result.String())
}
}
} else {
log.Println("be careful:", resp.Content())
}
} else {
log.Println("be careful:", resp.Content())
}
if imitater.ADDataSave(te, te.db, adDataList) {

View File

@ -19,7 +19,7 @@ func TestTaskEx_Execute(t *testing.T) {
log.Println(gjson.Valid(string(data)))
P := gjson.Parse(string(data))
gADDate := P.Get(`data.#[content ~ @"label":"广告"@ ]#.content`)
gADDate := P.Get(`data.#[content ~ @label_style\":3@ ]#.content`)
log.Println(gADDate.Exists(), gADDate.IsArray())
if !(gADDate.Exists() && gADDate.IsArray()) {

View File

@ -1,4 +1 @@
curl --name "tt-info" --task "info" 'https://it.snssdk.com/api/news/feed/v88/?list_count=17&support_rn=4&concern_id=6286225228934679042&refer=1&refresh_reason=1&session_refresh_idx=2&count=20&min_behot_time=1545427212&last_refresh_sub_entrance_interval=1545427282&loc_mode=5&loc_time=1545425201&latitude=23.060406&longitude=113.380702&city=%E5%B9%BF%E5%B7%9E%E5%B8%82&tt_from=pull&plugin_enable=3&st_time=295&sati_extra_params=%7B%22last_click_item_list%22%3A%5B%5D%7D&iid=54363182959&device_id=55149394150&ac=wifi&channel=xiaomi&aid=13&app_name=news_article&version_code=703&version_name=7.0.3&device_platform=android&ab_version=611285%2C662505%2C659641%2C640301%2C661854%2C486952%2C654085%2C651369%2C662176%2C472113%2C292723%2C659345%2C571131%2C616214%2C638883%2C639002%2C239095%2C612191%2C641905%2C170988%2C655796%2C643891%2C631016%2C659018%2C374117%2C652002%2C600231%2C587817%2C649422%2C633720%2C613176%2C550042%2C603542%2C659212%2C627550%2C656673%2C659530%2C649427%2C614100%2C522765%2C659286%2C416055%2C621360%2C656462%2C651696%2C643100%2C661405%2C652331%2C558140%2C555254%2C640008%2C659763%2C471406%2C603440%2C596392%2C660510%2C630576%2C598626%2C644857%2C646252%2C603379%2C603401%2C603404%2C603405%2C638928%2C662915%2C659552%2C646564%2C648850%2C656461%2C657631%2C661899%2C662644%2C629152%2C607361%2C567866%2C609337%2C662818%2C652366%2C662099%2C641414%2C654358%2C651920%2C662280%2C655010%2C650887%2C622716%2C662145%2C622728%2C653957%2C659453%2C663071%2C640997%2C641080%2C661616%2C660462%2C662397%2C631594%2C653241%2C658364%2C639497%2C554836%2C549647%2C644131%2C472442%2C31210%2C572465%2C651597%2C644058%2C615291%2C606547%2C442255%2C662167%2C660075%2C648180%2C630218%2C546701%2C281298%2C658129%2C622045%2C325619%2C649526%2C659121%2C634871%2C625066%2C652952%2C659705%2C653947%2C431142%2C498375%2C657352%2C638335%2C467513%2C655847%2C644240%2C631638%2C661473%2C648895%2C661697%2C655235%2C595556%2C662226%2C661444%2C661450%2C654129%2C655710%2C658938%2C660830%2C656557%2C644675%2C662684%2C661781%2C293032%2C457481%2C649402%2C591908%2C655988%2C648316%2C640210&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167&ab_feature=94563%2C102749&abflag=3&ssmix=a&device_type=MI+8+SE&device_brand=Xiaomi&language=zh&os_api=27&os_version=8.1.0&uuid=99001141462873&openudid=58cccf5030e9cbd4&manifest_version_code=703&resolution=1080*2114&dpi=440&update_version_code=70315&_rticket=1545427282300&plugin=26958&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zrqWtqq2vsb_x_On06ej5-L-nr66zrautqa2rsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOprailq6qxv_zw_O3R_On06ej5-L-nr66zraiqr66k4A%3D%3D&fp=zlT_L2UeLlPrFlPrPrU1FYwIJlm1&tma_jssdk_version=1.7.1.4&rom_version=miui_v10_v10.0.1.0.oebcnfh&ts=1545427282&as=a2e5950142f59cb9ad4355&mas=003b846ae93dd45c6424279ced4ec37f45f047464606686ecd&cp=50c91ad559952q1' -H 'Host: it.snssdk.com' -H 'Connection: keep-alive' -H 'Cookie: odin_tt=5bc108effff4529886a0aa36812c115d38b0e739f9abff2139107953c017cc62343bab05794f7fb3075b25846f32c178; UM_distinctid=1655625f84a18e-0897e6c8fc225e-6b55284e-4e4b0-1655625f84c3b; __tea_sdk__ssid=a79e00a6-b717-423f-9e97-f8ac84635f7e; tt_webid=6591877343492081160; __tea_sdk__user_unique_id=6591877343492081160; _ga=GA1.2.1032699944.1534947531; sid_guard=5ccb73ded741fb5f56e64623ca03ac8b%7C1544665500%7C5184000%7CMon%2C+11-Feb-2019+01%3A45%3A00+GMT; uid_tt=1f76f0742e12b4f8a5f32c009d5f7c52; sid_tt=5ccb73ded741fb5f56e64623ca03ac8b; sessionid=5ccb73ded741fb5f56e64623ca03ac8b; install_id=54363182959; ttreq=1$05885f6b61aaffcd5e901e181629f2e1d2851d21; qh[360]=1; alert_coverage=77' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1545427282310' -H 'X-Tt-Token: 005ccb73ded741fb5f56e64623ca03ac8bdc236ae81584d4b02df898418e84e101852bddfc64ca324cdf12e2a5ea1d924039' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 SE MIUI/V10.0.1.0.OEBCNFH) NewsArticle/7.0.3 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'
curl 'http://is.snssdk.com/api/news/feed/v88/?list_count=17&support_rn=4&category=news_entertainment&concern_id=6215497896830175745&refer=1&refresh_reason=1&session_refresh_idx=2&count=20&min_behot_time=1546566352&last_refresh_sub_entrance_interval=1546566639&loc_mode=5&loc_time=1546564033&latitude=23.060406&longitude=113.380702&city=%E5%B9%BF%E5%B7%9E%E5%B8%82&tt_from=pull&plugin_enable=3&sati_extra_params=%7B%22last_click_item_list%22%3A%5B%5D%7D&iid=55784134153&device_id=55149394150&ac=wifi&channel=xiaomi&aid=13&app_name=news_article&version_code=704&version_name=7.0.4&device_platform=android&ab_version=679615%2C664543%2C680643%2C666150%2C677773%2C486952%2C673652%2C651369%2C662176%2C472113%2C292723%2C680425%2C571131%2C674048%2C639002%2C612191%2C641905%2C170988%2C651014%2C643891%2C631016%2C659018%2C374117%2C652002%2C671751%2C645865%2C587817%2C655402%2C678353%2C613176%2C550042%2C679703%2C603542%2C659212%2C627550%2C680164%2C678790%2C649427%2C614100%2C677128%2C522765%2C669469%2C666019%2C416055%2C558140%2C555254%2C679177%2C471406%2C603440%2C596392%2C660510%2C598626%2C644857%2C680371%2C603379%2C603401%2C603404%2C603405%2C638928%2C662915%2C677669%2C661899%2C662644%2C668775%2C673945%2C629152%2C607361%2C609337%2C666967%2C635531%2C652366%2C662099%2C641414%2C664191%2C654358%2C651920%2C671245%2C655010%2C680507%2C622716%2C672873%2C678532%2C640997%2C641080%2C668774%2C679518%2C678338%2C631594%2C671445%2C676442%2C679732%2C554836%2C679903%2C549647%2C644131%2C472442%2C31210%2C572465%2C675212%2C651597%2C644058%2C615291%2C606547%2C678933%2C673168%2C678317%2C678279%2C546701%2C281298%2C658129%2C325619%2C678477%2C665473%2C625066%2C652952%2C677794%2C664311%2C431142%2C679143%2C678265%2C638335%2C467513%2C679101%2C679985%2C678878%2C595556%2C679519%2C678327%2C669426%2C670151%2C661450%2C654129%2C660830%2C656557%2C662684%2C661781%2C293032%2C457481%2C649402%2C591908%2C655988%2C648316%2C640210&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167&ab_feature=94563%2C102749&abflag=3&ssmix=a&device_type=MI+8+SE&device_brand=Xiaomi&language=zh&os_api=27&os_version=8.1.0&uuid=99001141462873&openudid=58cccf5030e9cbd4&manifest_version_code=704&resolution=1080*2114&dpi=440&update_version_code=70412&_rticket=1546566639364&plugin=26958&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zrqWtqq2vsb_x_On06ej5-L-nr66zrautqa2r4A%3D%3D&fp=zlT_L2UeLlPrFlPrPrU1FYwIJlm1&tma_jssdk_version=1.8.0.4&rom_version=miui_v10_v10.2.1.0.oebcnfk&ts=1546566640&as=a2a5ebb270cf3c5bee4355&mas=00fdecf117bbae1b81758aae487cb50d0e84cac44e06686ee3&cp=5ec52ae7b7befq1' -H 'Host: is.snssdk.com' -H 'Proxy-Connection: keep-alive' -H 'Cookie: odin_tt=5bc108effff4529886a0aa36812c115d38b0e739f9abff2139107953c017cc62343bab05794f7fb3075b25846f32c178; UM_distinctid=1655625f84a18e-0897e6c8fc225e-6b55284e-4e4b0-1655625f84c3b; __tea_sdk__ssid=a79e00a6-b717-423f-9e97-f8ac84635f7e; tt_webid=6591877343492081160; __tea_sdk__user_unique_id=6591877343492081160; CNZZDATA1264530760=1269028608-1534945655-%7C1534945655; _ga=GA1.2.1032699944.1534947531; sid_guard=5ccb73ded741fb5f56e64623ca03ac8b%7C1544665500%7C5184000%7CMon%2C+11-Feb-2019+01%3A45%3A00+GMT; uid_tt=1f76f0742e12b4f8a5f32c009d5f7c52; sid_tt=5ccb73ded741fb5f56e64623ca03ac8b; sessionid=5ccb73ded741fb5f56e64623ca03ac8b; install_id=55784134153; ttreq=1$2b24406b6e1991a32caf4e4246870c3aad035627; qh[360]=1' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1546566639369' -H 'X-Tt-Token: 005ccb73ded741fb5f56e64623ca03ac8b4fdb0fecc878b0c12132ca816432015792c6fb48ef4c9d781db882a81ad5f96715' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 SE MIUI/V10.2.1.0.OEBCNFK) NewsArticle/7.0.4 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'
curl -X 'GET' 'http://lf.snssdk.com/api/news/feed/v88/?list_count=142&support_rn=4&concern_id=6286225228934679042&refer=1&refresh_reason=1&session_refresh_idx=13&count=20&min_behot_time=1546582457&last_refresh_sub_entrance_interval=1546582480&loc_mode=4&tt_from=pull&lac=9502&cid=112375554&plugin_enable=3&st_time=1798&sati_extra_params=%7B%22last_click_item_list%22%3A%5B%5D%7D&iid=56493295608&device_id=53061747912&ac=wifi&channel=meizu&aid=13&app_name=news_article&version_code=704&version_name=7.0.4&device_platform=android&ab_version=629152%2C607361%2C609337%2C666967%2C635530%2C652363%2C662099%2C641413%2C673358%2C664194%2C654355%2C669191%2C671245%2C681071%2C680507%2C667071%2C622716%2C672874%2C680352%2C640997%2C641077%2C671798%2C668774%2C679516%2C631595%2C469022%2C676441%2C679733%2C673696%2C554836%2C679903%2C549647%2C644131%2C572465%2C482355%2C644057%2C615291%2C606548%2C678933%2C673168%2C678315%2C678279%2C671426%2C546703%2C641191%2C281297%2C664118%2C325620%2C678473%2C665472%2C625066%2C662507%2C663955%2C668676%2C664311%2C680283%2C638335%2C467515%2C679101%2C679985%2C678876%2C595556%2C679519%2C679764%2C669426%2C670151%2C661450%2C654124%2C680983%2C660830%2C679615%2C664543%2C680643%2C666150%2C486952%2C651365%2C662176%2C680305%2C675827%2C571131%2C665174%2C674057%2C594583%2C612193%2C641906%2C170988%2C680911%2C643892%2C659018%2C374116%2C655402%2C678350%2C550042%2C435216%2C603544%2C659218%2C680163%2C678792%2C649426%2C614096%2C677129%2C377573%2C522766%2C669470%2C416055%2C677236%2C558139%2C555254%2C679178%2C603441%2C596391%2C660508%2C598626%2C644855%2C680371%2C603386%2C603398%2C603404%2C603406%2C638927%2C661907%2C662644%2C668775%2C673945%2C679538%2C656557%2C633487%2C662683%2C661781%2C457481%2C649400%2C655989%2C677104&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167%2C94567%2C102754%2C181431&ab_feature=102754%2C94567&abflag=3&ssmix=a&device_type=U20&device_brand=Meizu&language=zh&os_api=23&os_version=6.0&uuid=861578036046061&openudid=d67c17290ee19a25&manifest_version_code=704&resolution=1080*1920&dpi=480&update_version_code=70412&_rticket=1546582480148&fp=2ST_F255JlQZFlcuJ2U1FYmeFzGS&tma_jssdk_version=1.1.0.13&pos=5r_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupaukqKmxv_zw_O3R_On06ej5-L-nr66zrairqqWo4A%3D%3D&rom_version=23&plugin=26958&ts=1546582480&as=a2b53ff2e0ad3c590e4355&mas=00f49a898835c1da8d1028395f0076f7c25ecd6e4e06686eb8&cp=55ca22eaf39d0q1' -H 'Host: lf.snssdk.com' -H 'Proxy-Connection: keep-alive' -H 'Cookie: odin_tt=b8dec487941fe35d2148bf9724de5e6659e65cf98541eee4f3573236e60d5f3c02a83ba067b8cd80a6685ac034db06ac; UM_distinctid=1653b99a5601-0c98c05be-2d4e2043-38400-1653b99a5641a; CNZZDATA1264530760=838951947-1534301713-%7C1534748760; __tea_sdk__ssid=290d7879-cb91-4e4c-aecd-b6239e2f04dc; tt_webid=6598313444424123907; __tea_sdk__user_unique_id=6598313444424123907; sid_guard=897e58fa1fa260bad70d802cbeee9225%7C1536824928%7C5184000%7CMon%2C+12-Nov-2018+07%3A48%3A48+GMT; CNZZDATA1263676333=575629806-1537840457-null%7C1537840457; qh[360]=1; install_id=56493295608; ttreq=1$b75d7a1dcbdafb18503a62139b7d79dc7aeeddb2' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1546582480161' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 6.0; U20 Build/MRA58K) NewsArticle/7.0.4 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'
curl --name "tt-info" --task "info" 'https://it.snssdk.com/api/news/feed/v88/?list_count=17&support_rn=4&concern_id=6286225228934679042&refer=1&refresh_reason=1&session_refresh_idx=2&count=20&min_behot_time=1545427212&last_refresh_sub_entrance_interval=1545427282&loc_mode=5&loc_time=1545425201&latitude=23.060406&longitude=113.380702&city=%E5%B9%BF%E5%B7%9E%E5%B8%82&tt_from=pull&plugin_enable=3&st_time=295&sati_extra_params=%7B%22last_click_item_list%22%3A%5B%5D%7D&iid=54363182959&device_id=55149394150&ac=wifi&channel=xiaomi&aid=13&app_name=news_article&version_code=703&version_name=7.0.3&device_platform=android&ab_version=611285%2C662505%2C659641%2C640301%2C661854%2C486952%2C654085%2C651369%2C662176%2C472113%2C292723%2C659345%2C571131%2C616214%2C638883%2C639002%2C239095%2C612191%2C641905%2C170988%2C655796%2C643891%2C631016%2C659018%2C374117%2C652002%2C600231%2C587817%2C649422%2C633720%2C613176%2C550042%2C603542%2C659212%2C627550%2C656673%2C659530%2C649427%2C614100%2C522765%2C659286%2C416055%2C621360%2C656462%2C651696%2C643100%2C661405%2C652331%2C558140%2C555254%2C640008%2C659763%2C471406%2C603440%2C596392%2C660510%2C630576%2C598626%2C644857%2C646252%2C603379%2C603401%2C603404%2C603405%2C638928%2C662915%2C659552%2C646564%2C648850%2C656461%2C657631%2C661899%2C662644%2C629152%2C607361%2C567866%2C609337%2C662818%2C652366%2C662099%2C641414%2C654358%2C651920%2C662280%2C655010%2C650887%2C622716%2C662145%2C622728%2C653957%2C659453%2C663071%2C640997%2C641080%2C661616%2C660462%2C662397%2C631594%2C653241%2C658364%2C639497%2C554836%2C549647%2C644131%2C472442%2C31210%2C572465%2C651597%2C644058%2C615291%2C606547%2C442255%2C662167%2C660075%2C648180%2C630218%2C546701%2C281298%2C658129%2C622045%2C325619%2C649526%2C659121%2C634871%2C625066%2C652952%2C659705%2C653947%2C431142%2C498375%2C657352%2C638335%2C467513%2C655847%2C644240%2C631638%2C661473%2C648895%2C661697%2C655235%2C595556%2C662226%2C661444%2C661450%2C654129%2C655710%2C658938%2C660830%2C656557%2C644675%2C662684%2C661781%2C293032%2C457481%2C649402%2C591908%2C655988%2C648316%2C640210&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167&ab_feature=94563%2C102749&abflag=3&ssmix=a&device_type=MI+8+SE&device_brand=Xiaomi&language=zh&os_api=27&os_version=8.1.0&uuid=99001141462873&openudid=58cccf5030e9cbd4&manifest_version_code=703&resolution=1080*2114&dpi=440&update_version_code=70315&_rticket=1545427282300&plugin=26958&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zrqWtqq2vsb_x_On06ej5-L-nr66zrautqa2rsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOprailq6qxv_zw_O3R_On06ej5-L-nr66zraiqr66k4A%3D%3D&fp=zlT_L2UeLlPrFlPrPrU1FYwIJlm1&tma_jssdk_version=1.7.1.4&rom_version=miui_v10_v10.0.1.0.oebcnfh&ts=1545427282&as=a2e5950142f59cb9ad4355&mas=003b846ae93dd45c6424279ced4ec37f45f047464606686ecd&cp=50c91ad559952q1' -H 'Host: it.snssdk.com' -H 'Connection: keep-alive' -H 'Cookie: odin_tt=5bc108effff4529886a0aa36812c115d38b0e739f9abff2139107953c017cc62343bab05794f7fb3075b25846f32c178; UM_distinctid=1655625f84a18e-0897e6c8fc225e-6b55284e-4e4b0-1655625f84c3b; __tea_sdk__ssid=a79e00a6-b717-423f-9e97-f8ac84635f7e; tt_webid=6591877343492081160; __tea_sdk__user_unique_id=6591877343492081160; _ga=GA1.2.1032699944.1534947531; sid_guard=5ccb73ded741fb5f56e64623ca03ac8b%7C1544665500%7C5184000%7CMon%2C+11-Feb-2019+01%3A45%3A00+GMT; uid_tt=1f76f0742e12b4f8a5f32c009d5f7c52; sid_tt=5ccb73ded741fb5f56e64623ca03ac8b; sessionid=5ccb73ded741fb5f56e64623ca03ac8b; install_id=54363182959; ttreq=1$05885f6b61aaffcd5e901e181629f2e1d2851d21; qh[360]=1; alert_coverage=77' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1545427282310' -H 'X-Tt-Token: 005ccb73ded741fb5f56e64623ca03ac8bdc236ae81584d4b02df898418e84e101852bddfc64ca324cdf12e2a5ea1d924039' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 SE MIUI/V10.0.1.0.OEBCNFH) NewsArticle/7.0.3 cronet/TTNetVersion:a729d5c3 2018-11-25' -H 'X-SS-TC: 0'

View File

@ -1,6 +1,6 @@
mode : 0
# proxies : "socks5://10.10.0.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.0.1:8080", "socks5://10.10.0.1:8082", "socks5://10.10.0.1:8083", "socks5://10.10.0.1:8085", "socks5://10.10.0.1:8087", "socks5://10.10.0.1:8088", "socks5://10.10.0.1:8091"]
# proxies : "socks5://10.10.10.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8091"]
retry : 0
timeout: 12
priority : 10000

View File

@ -1 +0,0 @@
../vendor

File diff suppressed because it is too large Load Diff

View File

@ -1,71 +0,0 @@
package main
import (
"log"
"github.com/Pallinder/go-randomdata"
"474420502.top/eson/gjson"
crontab "474420502.top/eson/crontabex"
"474420502.top/eson/imitater"
"474420502.top/test/logdb"
)
func main() {
imitater.Register("toutiao", &TaskEx{})
person := imitater.NewPerson()
person.Config("task.yaml")
person.Execute()
}
// TaskEx 任务相关类
type TaskEx struct {
imitater.Task
db *logdb.LogDB
}
// Init 初始化函数
func (te *TaskEx) Init() {
te.db = logdb.New("../logdb.yaml")
}
// Execute 执行过程的方法
func (te *TaskEx) Execute(glocal map[string]interface{}) imitater.ITask {
wf := te.Workflow(true)
kw := randomdata.StringSample("手游", "游戏推荐", "赚钱游戏", "武侠游戏", "战略游戏", "网游", "游戏", "单机游戏")
wf.GetQuery().Set("keyword", kw)
log.Println(kw)
resp, err := wf.Execute()
if err != nil {
log.Println(err)
return te
}
var adDataList []string
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`#[label = "广告"]#`)
if adData.Exists() {
for _, result := range adData.Array() {
adDataList = append(adDataList, result.String())
}
}
}
} else {
log.Println("be careful:", resp.Content())
}
if imitater.ADDataSave(te, te.db, adDataList) {
te.GetCrontab().SetStatus(crontab.SExecuteOK, true)
} else {
te.GetCrontab().SetStatus(crontab.SExecuteOK, false)
}
return te
}

View File

@ -1,38 +0,0 @@
package main
import (
"io/ioutil"
"testing"
"474420502.top/eson/gjson"
)
func TestTaskEx_Execute(t *testing.T) {
data, err := ioutil.ReadFile("../search.json")
if err != nil {
panic(err)
}
var adDataList []string
if gjson.Valid(string(data)) {
P := gjson.Parse(string(data))
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`#[label = "广告"]#`)
if adData.Exists() {
for _, result := range adData.Array() {
adDataList = append(adDataList, result.String())
}
} else {
t.Error(`expr error`)
}
}
}
if len(adDataList) == 1 {
t.Error("adDataList != 1")
t.Error(adDataList)
}
}

View File

@ -1 +0,0 @@
curl -X 'GET' 'https://lf.snssdk.com/api/search/content/?from=search_tab&keyword=%E7%BD%91%E6%B8%B8&cur_tab_title=search_tab&plugin_enable=3&iid=56493295608&device_id=53061747912&ac=wifi&channel=meizu&aid=13&app_name=news_article&version_code=704&version_name=7.0.4&device_platform=android&ab_group=100167%252C94567%252C102754%252C181431&abflag=3&device_type=U20&device_brand=Meizu&language=zh&os_api=23&os_version=6.0&uuid=861578036046061&openudid=d67c17290ee19a25&manifest_version_code=704&resolution=1080*1920&dpi=480&update_version_code=70412&_rticket=1546583815966&fp=2ST_F255JlQZFlcuJ2U1FYmeFzGS&tma_jssdk_version=1.8.0.4&pos=5r_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupaukqKmxv_zw_O3R_On06ej5-L-nr66zrairqqWo4A%253D%253D&rom_version=23&plugin=26958&search_sug=1&forum=1&count=10&format=json&source=input&pd=synthesis&keyword_type=&action_type=input_keyword_search&search_position=search_tab&from_search_subtab=&offset=0&search_id=&has_count=0&qc_query=' -H 'Host: lf.snssdk.com' -H 'Connection: keep-alive' -H 'Accept: text/javascript, text/html, application/xml, text/xml, */*' -H 'X-Requested-With: XMLHttpRequest' -H 'User-Agent: Mozilla/5.0 (Linux; Android 6.0; U20 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.147 Mobile Safari/537.36 JsSdk/2 NewsArticle/7.0.4 NetType/wifi' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Referer: https://lf.snssdk.com/feoffline/search/template/tt_search/search/search.html?from=search_tab&keyword=&cur_tab_title=search_tab&plugin_enable=3&followbtn_template=%7B%22color_style%22%3A%22red%22%7D&iid=56493295608&device_id=53061747912&ac=wifi&channel=meizu&aid=13&app_name=news_article&version_code=704&version_name=7.0.4&device_platform=android&ab_version=680305%2C675827%2C571131%2C665174%2C674057%2C594583%2C612193%2C641906%2C170988%2C680911%2C643892%2C659018%2C374116%2C655402%2C678350%2C550042%2C435216%2C603544%2C659218%2C680163%2C678792%2C649426%2C614096%2C677129%2C377573%2C522766%2C669470%2C416055%2C677236%2C558139%2C555254%2C679178%2C603441%2C596391%2C660508%2C598626%2C644855%2C680371%2C603386%2C603398%2C603404%2C603406%2C638927%2C681323%2C661907%2C662644%2C668775%2C673945%2C629152%2C607361%2C609337%2C666967%2C635530%2C652363%2C662099%2C641413%2C673358%2C664194%2C654355%2C669191%2C671245%2C681071%2C680507%2C667071%2C622716%2C672874%2C680352%2C640997%2C641077%2C671798%2C668774%2C679516%2C631595%2C469022%2C676441%2C679733%2C673696%2C554836%2C679903%2C549647%2C644131%2C572465%2C482355%2C644057%2C615291%2C606548%2C681182%2C678933%2C673168%2C678315%2C678279%2C671426%2C546703%2C641191%2C281297%2C664118%2C325620%2C678473%2C665472%2C625066%2C662507%2C663955%2C668676%2C664311%2C680283%2C638335%2C467515%2C679101%2C679985%2C678876%2C595556%2C679519%2C679764%2C669426%2C670151%2C661450%2C654124%2C680983%2C660830%2C679615%2C664543%2C680643%2C666150%2C486952%2C651365%2C662176%2C679538%2C656557%2C633487%2C662683%2C661781%2C457481%2C649400%2C655989%2C677104&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167%2C94567%2C102754%2C181431&ab_feature=94567%2C102754&abflag=3&device_type=U20&device_brand=Meizu&language=zh&os_api=23&os_version=6.0&uuid=861578036046061&openudid=d67c17290ee19a25&manifest_version_code=704&resolution=1080*1920&dpi=480&update_version_code=70412&_rticket=1546583815966&fp=2ST_F255JlQZFlcuJ2U1FYmeFzGS&tma_jssdk_version=1.8.0.4&pos=5r_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupaukqKmxv_zw_O3R_On06ej5-L-nr66zrairqqWo4A%3D%3D&rom_version=23&plugin=26958&search_sug=1&forum=1' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: zh-CN,en-US;q=0.8' -H 'Cookie: odin_tt=b8dec487941fe35d2148bf9724de5e6659e65cf98541eee4f3573236e60d5f3c02a83ba067b8cd80a6685ac034db06ac; UM_distinctid=1653b99a5601-0c98c05be-2d4e2043-38400-1653b99a5641a; CNZZDATA1264530760=838951947-1534301713-%7C1534748760; __tea_sdk__ssid=290d7879-cb91-4e4c-aecd-b6239e2f04dc; tt_webid=6598313444424123907; __tea_sdk__user_unique_id=6598313444424123907; sid_guard=897e58fa1fa260bad70d802cbeee9225%7C1536824928%7C5184000%7CMon%2C+12-Nov-2018+07%3A48%3A48+GMT; CNZZDATA1263676333=575629806-1537840457-null%7C1537840457; qh[360]=1; install_id=56493295608; ttreq=1$b75d7a1dcbdafb18503a62139b7d79dc7aeeddb2' --compressed

View File

@ -1,19 +0,0 @@
mode : 0
# proxies : "socks5://10.10.0.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.0.1:8080", "socks5://10.10.0.1:8082", "socks5://10.10.0.1:8083", "socks5://10.10.0.1:8085", "socks5://10.10.0.1:8087", "socks5://10.10.0.1:8088", "socks5://10.10.0.1:8091"]
retry : 0
timeout: 12
priority : 10000
curls : "@task.curl" # 支持, 列表 与 单项字符串
task: "toutiao"
device : "eson-OnePlus"
platform : "Android"
# area_cc : 4401
channel : 105
media : 55
spider_id : 10001003
catch_account_id : 1001003
crontab: "f8>=360|f5>=240|f=120|s=20|s10>=60"

View File

@ -1 +0,0 @@
../vendor

View File

@ -1,279 +0,0 @@
{
"message": "success",
"data": {
"penalty_period": [
4102416000,
4102416000
],
"sdk_splash": 0,
"splash_interval": 5400,
"leave_interval": 600,
"show_limit": 0,
"is_need_ack": true,
"splash": [
{
"log_extra": "{\"splash_send\": \"1621698206869630 1621698215318535 1621431002013758 1621634980708413 1621634988588039\", \"ad_id\": 1621697700496392, \"vid\": \"680305,675827,571131,665174,674057,594583,612193,641906,170988,680911,643892,659018,374116,677696,655402,678350,550042,435216,603544,659218,680163,678792,649426,614096,677129,377573,522766,669470,416055,677236,558139,555254,679178,603441,596391,660508,598626,644855,680371,603386,603398,603404,603406,638927,681323,661907,662644,668775,673945,631595,469022,676441,679733,673696,554836,679903,549647,644131,572465,482355,644057,615291,606548,681182,678933,673168,678315,678279,671426,546703,641191,281297,664118,325620,678473,665472,625066,662507,663955,668676,664311,680283,638335,467515,679101,679985,678876,595556,679519,679764,669426,670151,661450,654124,680983,660830\", \"city_id\": 0, \"rit\": 2, \"an\": \"news_article\", \"req_id\": \"201901041601500100140361088919AE\", \"province_id\": 0}",
"open_url": "sslocal://profile?uid=96979344506",
"track_url": "",
"web_url": "https://www.toutiao.com/c/user/96979344506/#mid=1597977670828036",
"image_mode": 0,
"splash_id": 1621698206869630,
"max_display_time_ms": 7000,
"skip_btn": 1,
"id": 1621698206869630,
"display_density": "1242x2208",
"expire_seconds": 86400.0,
"display_after": 28689,
"title": "\u5973\u8db3\u8d75\u4e3d\u5a1c\u901b\u8fea\u58eb\u5c3c\u7684\u4e00\u5929",
"click_track_url_list": [],
"expire_timestamp": 1546704000.0,
"splash_type": 2,
"track_url_list": [],
"image_info": {
"width": 1242,
"url_list": [
{
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5"
},
{
"url": "https://sf3-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5"
},
{
"url": "https://sf1-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5"
}
],
"uri": "web.business.image/201901045d0d1117bf25635e473c90d5",
"height": 2208
},
"type": "web",
"display_time": 3,
"repeat": 0,
"splash_load_type": 3,
"predownload": 1,
"web_title": "\u5973\u8db3\u8d75\u4e3d\u5a1c\u901b\u8fea\u58eb\u5c3c\u7684\u4e00\u5929",
"click_btn": 0,
"banner_mode": 0,
"url": "aHR0cDovL3YzLXR0Lml4aWd1YS5jb20vZmY2NDI5ZTc3NmEzMmY5N2Y3ZWE0ZmQ2MzA5OWVkMWQvNWMyZjIxMDQvdmlkZW8vbS8yMjAyMzg1MjY4MTA2ZmI0OGQ0YmFkZDA4Njg2ZTgzODU3OTExNjEzMzhkYjAwMDA2MDM1NTY1NWVkNzgvP3JjPWFtODFkMmMwWjJWbWFqTXpaRG96TTBBcFFIUkFiMGc4UERzME5qY3pORE0wT1RjMFBETkFLWFVwUUdjemR5bEFabXhrYm1ReFpHWTJRRzVrTFhJd2NESnFhMTh0TFRJdEwzTnpMVzhqYnlNJTJCTGpVdE15MHVMUzB5TFM0dExTNHZhVHBpTFc4ak9tQXRieU50SzJCdFlUb2pMbDQlM0Q=",
"display_time_ms": 5000,
"video_info": {
"play_track_url_list": [],
"video_url_list": [
"aHR0cDovL3YxLXR0Lml4aWd1YXZpZGVvLmNvbS9lODMyZDU4NWE2NGI2ZGUyNjlmZmMwMWZlNDA4NTAwMy81YzJmMjEwNC92aWRlby9tLzIyMDIzODUyNjgxMDZmYjQ4ZDRiYWRkMDg2ODZlODM4NTc5MTE2MTMzOGRiMDAwMDYwMzU1NjU1ZWQ3OC8/cmM9YW04MWQyYzBaMlZtYWpNelpEb3pNMEFwUUhSQWIwZzhQRHMwTmpjek5ETTBPVGMwUEROQUtYVXBRR2N6ZHlsQVpteGtibVF4WkdZMlFHNWtMWEl3Y0RKcWExOHRMVEl0TDNOekxXOGpieU0lMkJMalV0TXkwdUxTMHlMUzR0TFM0dmFUcGlMVzhqT21BdGJ5TnRLMkJ0WVRvakxsNCUzRA==",
"aHR0cDovL3YzLXR0Lml4aWd1YS5jb20vYTFjN2E1YmJlNGJlMjdkYzJhZDE5NTJjOTFkN2YwY2IvNWMyZjIxMDQvdmlkZW8vbS8yMjAzMTdjNDFkYmViYmQ0YWM4ODJkODhhYzg1YTE5NWMyODExNjEzMzIwODAwMDA0ZjczZDIxNDg2Y2MvP3JjPWFtODFkMmMwWjJWbWFqTXpaRG96TTBBcFFIUkFiMGc4UERzME5qY3pORE0wT1RjMFBETkFLWFVwUUdjemR5bEFabXhrYm1ReFpHWTJRRzVrTFhJd2NESnFhMTh0TFRJdEwzTnpMVzhqYnlNJTJCTGpVdE15MHVMUzB5TFM0dExTNHZhVHBpTFc4ak9tQXRieU50SzJCdFlUb2pMbDQlM0Q=",
"aHR0cDovL3YxLXR0Lml4aWd1YXZpZGVvLmNvbS8zZmY4ZDM1YTU1ODZhYjQ4M2ZmNWQ4OGUzMjkzNjc1OS81YzJmMjEwNC92aWRlby9tLzIyMDMxN2M0MWRiZWJiZDRhYzg4MmQ4OGFjODVhMTk1YzI4MTE2MTMzMjA4MDAwMDRmNzNkMjE0ODZjYy8/cmM9YW04MWQyYzBaMlZtYWpNelpEb3pNMEFwUUhSQWIwZzhQRHMwTmpjek5ETTBPVGMwUEROQUtYVXBRR2N6ZHlsQVpteGtibVF4WkdZMlFHNWtMWEl3Y0RKcWExOHRMVEl0TDNOekxXOGpieU0lMkJMalV0TXkwdUxTMHlMUzR0TFM0dmFUcGlMVzhqT21BdGJ5TnRLMkJ0WVRvakxsNCUzRA==",
"aHR0cDovL3YzLXR0Lml4aWd1YS5jb20vYjc1MjI3ZmJjZTg3MTIyNWQ3Nzc1ZmRmZmYzZTZiMTcvNWMyZjIxMDQvdmlkZW8vbS8yMjBiOTE0ZjcxNWMzOTg0MTU4ODRkMGM5NGY1MGViMjcwNTExNjEzNDM1MDAwMDA2MzFkNzFiZWMzMzAvP3JjPWFtODFkMmMwWjJWbWFqTXpaRG96TTBBcFFIUkFiMGc4UERzME5qY3pORE0wT1RjMFBETkFLWFVwUUdjemR5bEFabXhrYm1ReFpHWTJRRzVrTFhJd2NESnFhMTh0TFRJdEwzTnpMVzhqYnlNJTJCTGpVdE15MHVMUzB5TFM0dExTNHZhVHBpTFc4ak9tQXRieU50SzJCdFlUb2pMbDQlM0Q=",
"aHR0cDovL3YxLXR0Lml4aWd1YXZpZGVvLmNvbS80ZGRiZmNhODNlMWYwNzFjNmJiNjcxMzk4MTM3YWM3Ny81YzJmMjEwNC92aWRlby9tLzIyMGI5MTRmNzE1YzM5ODQxNTg4NGQwYzk0ZjUwZWIyNzA1MTE2MTM0MzUwMDAwMDYzMWQ3MWJlYzMzMC8/cmM9YW04MWQyYzBaMlZtYWpNelpEb3pNMEFwUUhSQWIwZzhQRHMwTmpjek5ETTBPVGMwUEROQUtYVXBRR2N6ZHlsQVpteGtibVF4WkdZMlFHNWtMWEl3Y0RKcWExOHRMVEl0TDNOekxXOGpieU0lMkJMalV0TXkwdUxTMHlMUzR0TFM0dmFUcGlMVzhqT21BdGJ5TnRLMkJ0WVRvakxsNCUzRA=="
],
"video_density": "640x1136",
"video_id": "v020075a0000bgncmb5ds13dut02glqg",
"voice_switch": false,
"video_group_id": 6642475889151967502,
"action_track_url_list": []
},
"action": ""
},
{
"log_extra": "{\"splash_send\": \"1621698206869630 1621698215318535 1621431002013758 1621634980708413 1621634988588039\", \"ad_id\": 1621697700496392, \"vid\": \"680305,675827,571131,665174,674057,594583,612193,641906,170988,680911,643892,659018,374116,677696,655402,678350,550042,435216,603544,659218,680163,678792,649426,614096,677129,377573,522766,669470,416055,677236,558139,555254,679178,603441,596391,660508,598626,644855,680371,603386,603398,603404,603406,638927,681323,661907,662644,668775,673945,631595,469022,676441,679733,673696,554836,679903,549647,644131,572465,482355,644057,615291,606548,681182,678933,673168,678315,678279,671426,546703,641191,281297,664118,325620,678473,665472,625066,662507,663955,668676,664311,680283,638335,467515,679101,679985,678876,595556,679519,679764,669426,670151,661450,654124,680983,660830\", \"city_id\": 0, \"rit\": 2, \"an\": \"news_article\", \"req_id\": \"201901041601500100140361088919AE\", \"province_id\": 0}",
"open_url": "sslocal://profile?uid=96979344506",
"track_url": "",
"web_url": "https://www.toutiao.com/c/user/96979344506/#mid=1597977670828036",
"image_mode": 0,
"splash_id": 1621698215318535,
"max_display_time_ms": 4000,
"skip_btn": 1,
"id": 1621698215318535,
"display_density": "1242x2208",
"expire_seconds": 5,
"display_after": 8640000,
"title": "\u5973\u8db3\u8d75\u4e3d\u5a1c\u901b\u8fea\u58eb\u5c3c\u7684\u4e00\u5929",
"click_track_url_list": [],
"expire_timestamp": 1546704000.0,
"track_url_list": [],
"image_info": {
"width": 1242,
"url_list": [
{
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5"
},
{
"url": "https://sf3-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5"
},
{
"url": "https://sf1-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5"
}
],
"uri": "web.business.image/201901045d0d1117bf25635e473c90d5",
"height": 2208
},
"type": "web",
"display_time": 3,
"repeat": 0,
"splash_load_type": 3,
"predownload": 31,
"web_title": "\u5973\u8db3\u8d75\u4e3d\u5a1c\u901b\u8fea\u58eb\u5c3c\u7684\u4e00\u5929",
"click_btn": 0,
"banner_mode": 0,
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901045d0d1117bf25635e473c90d5",
"display_time_ms": 3000,
"action": ""
},
{
"log_extra": "{\"splash_send\": \"1621698206869630 1621698215318535 1621431002013758 1621634980708413 1621634988588039\", \"ad_id\": 1621430937808925, \"vid\": \"680305,675827,571131,665174,674057,594583,612193,641906,170988,680911,643892,659018,374116,677696,655402,678350,550042,435216,603544,659218,680163,678792,649426,614096,677129,377573,522766,669470,416055,677236,558139,555254,679178,603441,596391,660508,598626,644855,680371,603386,603398,603404,603406,638927,681323,661907,662644,668775,673945,631595,469022,676441,679733,673696,554836,679903,549647,644131,572465,482355,644057,615291,606548,681182,678933,673168,678315,678279,671426,546703,641191,281297,664118,325620,678473,665472,625066,662507,663955,668676,664311,680283,638335,467515,679101,679985,678876,595556,679519,679764,669426,670151,661450,654124,680983,660830\", \"city_id\": 0, \"rit\": 2, \"an\": \"news_article\", \"req_id\": \"201901041601500100140361088919AE\", \"province_id\": 0}",
"open_url": "snssdk143://detail?groupid=6641384331082203399&ad_id=1621431002013758",
"track_url": "http://v.admaster.com.cn/i/a121666,b3155991,c2343,i13805,m202,8a2,8b3,j53061747912,0a0,nd41d8cd98f00b204e9800998ecf8427e,z__IDFA__,0d9623a21460e5edeb17d087ca394ec4b5,0c83defe6828e3ec0dd2a5202376b14dd3,f183.63.164.154,t1546588911000,l__LBS__,h",
"web_url": "http://event.toutiaocloud.com/trumpchi/201901/gm6/",
"image_mode": 0,
"splash_id": 1621431002013758,
"max_display_time_ms": 4000,
"skip_btn": 1,
"id": 1621431002013758,
"display_density": "1242x1786",
"expire_seconds": 5,
"display_after": 8640000,
"title": "\u4f20\u797aGM6 \u5168\u80fd\u5bbd\u4eabMPV",
"click_track_url_list": [
"http://clickc.admaster.com.cn/c/a121666,b3155991,c2343,i13805,m101,8a2,8b3,j53061747912,0a0,nd41d8cd98f00b204e9800998ecf8427e,z__IDFA__,0d9623a21460e5edeb17d087ca394ec4b5,0c83defe6828e3ec0dd2a5202376b14dd3,f183.63.164.154,t1546588911000,l__LBS__,h"
],
"expire_timestamp": 1546704000.0,
"track_url_list": [
"http://v.admaster.com.cn/i/a121666,b3155991,c2343,i13805,m202,8a2,8b3,j53061747912,0a0,nd41d8cd98f00b204e9800998ecf8427e,z__IDFA__,0d9623a21460e5edeb17d087ca394ec4b5,0c83defe6828e3ec0dd2a5202376b14dd3,f183.63.164.154,t1546588911000,l__LBS__,h"
],
"image_info": {
"width": 1242,
"url_list": [
{
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d86acb5908ce24b0c9920"
},
{
"url": "https://sf1-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d86acb5908ce24b0c9920"
},
{
"url": "https://sf3-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d86acb5908ce24b0c9920"
}
],
"uri": "web.business.image/201901035d0d86acb5908ce24b0c9920",
"height": 1786
},
"type": "web",
"display_time": 3,
"repeat": 0,
"splash_load_type": 3,
"predownload": 31,
"web_title": "\u4f20\u797aGM6 \u5168\u80fd\u5bbd\u4eabMPV",
"click_btn": 0,
"banner_mode": 1,
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d86acb5908ce24b0c9920",
"display_time_ms": 3000,
"action": ""
},
{
"log_extra": "{\"splash_send\": \"1621698206869630 1621698215318535 1621431002013758 1621634980708413 1621634988588039\", \"ad_id\": 1621633648371735, \"vid\": \"680305,675827,571131,665174,674057,594583,612193,641906,170988,680911,643892,659018,374116,677696,655402,678350,550042,435216,603544,659218,680163,678792,649426,614096,677129,377573,522766,669470,416055,677236,558139,555254,679178,603441,596391,660508,598626,644855,680371,603386,603398,603404,603406,638927,681323,661907,662644,668775,673945,631595,469022,676441,679733,673696,554836,679903,549647,644131,572465,482355,644057,615291,606548,681182,678933,673168,678315,678279,671426,546703,641191,281297,664118,325620,678473,665472,625066,662507,663955,668676,664311,680283,638335,467515,679101,679985,678876,595556,679519,679764,669426,670151,661450,654124,680983,660830\", \"city_id\": 0, \"rit\": 2, \"an\": \"news_article\", \"req_id\": \"201901041601500100140361088919AE\", \"province_id\": 0}",
"open_url": "snssdk143://detail?groupid=6642216913403379982&ad_id=1621634980708413",
"track_url": "",
"web_url": "http://toutiao-sports.vvsvc.com/?app_from=ttkp",
"image_mode": 0,
"splash_id": 1621634980708413,
"max_display_time_ms": 7000,
"skip_btn": 1,
"id": 1621634980708413,
"display_density": "1242x2208",
"expire_seconds": 5,
"display_after": 8640000,
"title": "\u4f53\u575b\u7535\u5f71\u8282\uff0c\u8c01\u662f\u771f\u620f\u7cbe",
"click_track_url_list": [],
"expire_timestamp": 1546704000.0,
"splash_type": 2,
"track_url_list": [],
"image_info": {
"width": 1242,
"url_list": [
{
"url": "https://sf1-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20"
},
{
"url": "https://sf3-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20"
},
{
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20"
}
],
"uri": "web.business.image/201901035d0d679e106268264da89c20",
"height": 2208
},
"type": "web",
"display_time": 3,
"repeat": 0,
"splash_load_type": 3,
"predownload": 1,
"web_title": "\u4f53\u575b\u7535\u5f71\u8282\uff0c\u8c01\u662f\u771f\u620f\u7cbe",
"click_btn": 0,
"banner_mode": 0,
"url": "aHR0cDovL3YzLXR0Lml4aWd1YS5jb20vZDFmZGI0NGM5NTA4ZTg5YTg1MDQ0MzE5Y2ViMjBkMDQvNWMyZjIxMDQvdmlkZW8vbS8yMjBiNGZjYzA2NDQ1M2E0ODMzYTRhN2JjN2Y5NzA4MDMxYzExNjEzMmU1ZTAwMDA4Mzg3NzZkMmM3N2MvP3JjPWFuUjFOMjQ3UERkNGFqTXpNem96TTBBcFFIUkFiMGc4UERzME5qY3pORE0wT1RjMFBETkFLWFVwUUdjemR5bEFabXhrYm1ReFpHWTJRQzFrYTJvMU16SXRhbDh0TFdFdEwzTnpMVzhqYnlNJTJCTGpVdE15MHVMUzB5TFM0dExTNHZhVHBpTFc4ak9tQXRieU50SzJCdFlUb2pMbDQlM0Q=",
"display_time_ms": 5000,
"video_info": {
"play_track_url_list": [],
"video_url_list": [
"aHR0cDovL3YxLXR0Lml4aWd1YXZpZGVvLmNvbS81M2NhYmY3M2EzYzNlNjkwZGZjZDU0OWNmMjRhNGI1MC81YzJmMjEwNC92aWRlby9tLzIyMGI0ZmNjMDY0NDUzYTQ4MzNhNGE3YmM3Zjk3MDgwMzFjMTE2MTMyZTVlMDAwMDgzODc3NmQyYzc3Yy8/cmM9YW5SMU4yNDdQRGQ0YWpNek16b3pNMEFwUUhSQWIwZzhQRHMwTmpjek5ETTBPVGMwUEROQUtYVXBRR2N6ZHlsQVpteGtibVF4WkdZMlFDMWthMm8xTXpJdGFsOHRMV0V0TDNOekxXOGpieU0lMkJMalV0TXkwdUxTMHlMUzR0TFM0dmFUcGlMVzhqT21BdGJ5TnRLMkJ0WVRvakxsNCUzRA==",
"aHR0cDovL3YzLXR0Lml4aWd1YS5jb20vNWY0MGEwNWQxNjlhYzlmMDA0NjMxZTNhYTQzM2IwMjcvNWMyZjIxMDQvdmlkZW8vbS8yMjA5Y2Q5NWZhZmVhODg0NGRkYjMxNTViZjQ0YjlkM2EzNzExNjEzMmEyZjAwMDA0M2JmM2IwM2QxNTIvP3JjPWFuUjFOMjQ3UERkNGFqTXpNem96TTBBcFFIUkFiMGc4UERzME5qY3pORE0wT1RjMFBETkFLWFVwUUdjemR5bEFabXhrYm1ReFpHWTJRQzFrYTJvMU16SXRhbDh0TFdFdEwzTnpMVzhqYnlNJTJCTGpVdE15MHVMUzB5TFM0dExTNHZhVHBpTFc4ak9tQXRieU50SzJCdFlUb2pMbDQlM0Q=",
"aHR0cDovL3YxLXR0Lml4aWd1YXZpZGVvLmNvbS9hNDUwYjEwYTU0ZDM0YmIxZGFkYzQwYWZlM2UxNGZlZS81YzJmMjEwNC92aWRlby9tLzIyMDljZDk1ZmFmZWE4ODQ0ZGRiMzE1NWJmNDRiOWQzYTM3MTE2MTMyYTJmMDAwMDQzYmYzYjAzZDE1Mi8/cmM9YW5SMU4yNDdQRGQ0YWpNek16b3pNMEFwUUhSQWIwZzhQRHMwTmpjek5ETTBPVGMwUEROQUtYVXBRR2N6ZHlsQVpteGtibVF4WkdZMlFDMWthMm8xTXpJdGFsOHRMV0V0TDNOekxXOGpieU0lMkJMalV0TXkwdUxTMHlMUzR0TFM0dmFUcGlMVzhqT21BdGJ5TnRLMkJ0WVRvakxsNCUzRA==",
"aHR0cDovL3YzLXR0Lml4aWd1YS5jb20vNGRhYjMwN2U2YzExNDg2MmRhYTEzMDAyMTQwZTVjZDUvNWMyZjIxMDQvdmlkZW8vbS8yMjBiYjhlZDEyNTNmYjQ0MTdlOGM0MTIwM2FiNzI4ZjY5MDExNjEzNDAwZDAwMDAyY2E0ZTk2ZWFlNDMvP3JjPWFuUjFOMjQ3UERkNGFqTXpNem96TTBBcFFIUkFiMGc4UERzME5qY3pORE0wT1RjMFBETkFLWFVwUUdjemR5bEFabXhrYm1ReFpHWTJRQzFrYTJvMU16SXRhbDh0TFdFdEwzTnpMVzhqYnlNJTJCTGpVdE15MHVMUzB5TFM0dExTNHZhVHBpTFc4ak9tQXRieU50SzJCdFlUb2pMbDQlM0Q=",
"aHR0cDovL3YxLXR0Lml4aWd1YXZpZGVvLmNvbS8yMzQ0YTc5ZjA2ZjM5NzZhNmZjZDZiY2E2ZGY4YjE5Yi81YzJmMjEwNC92aWRlby9tLzIyMGJiOGVkMTI1M2ZiNDQxN2U4YzQxMjAzYWI3MjhmNjkwMTE2MTM0MDBkMDAwMDJjYTRlOTZlYWU0My8/cmM9YW5SMU4yNDdQRGQ0YWpNek16b3pNMEFwUUhSQWIwZzhQRHMwTmpjek5ETTBPVGMwUEROQUtYVXBRR2N6ZHlsQVpteGtibVF4WkdZMlFDMWthMm8xTXpJdGFsOHRMV0V0TDNOekxXOGpieU0lMkJMalV0TXkwdUxTMHlMUzR0TFM0dmFUcGlMVzhqT21BdGJ5TnRLMkJ0WVRvakxsNCUzRA=="
],
"video_density": "640x1136",
"video_id": "v02007d00000bgmu0459688km4nrgq0g",
"voice_switch": false,
"video_group_id": 6642216912493216014,
"action_track_url_list": []
},
"action": ""
},
{
"log_extra": "{\"splash_send\": \"1621698206869630 1621698215318535 1621431002013758 1621634980708413 1621634988588039\", \"ad_id\": 1621633648371735, \"vid\": \"680305,675827,571131,665174,674057,594583,612193,641906,170988,680911,643892,659018,374116,677696,655402,678350,550042,435216,603544,659218,680163,678792,649426,614096,677129,377573,522766,669470,416055,677236,558139,555254,679178,603441,596391,660508,598626,644855,680371,603386,603398,603404,603406,638927,681323,661907,662644,668775,673945,631595,469022,676441,679733,673696,554836,679903,549647,644131,572465,482355,644057,615291,606548,681182,678933,673168,678315,678279,671426,546703,641191,281297,664118,325620,678473,665472,625066,662507,663955,668676,664311,680283,638335,467515,679101,679985,678876,595556,679519,679764,669426,670151,661450,654124,680983,660830\", \"city_id\": 0, \"rit\": 2, \"an\": \"news_article\", \"req_id\": \"201901041601500100140361088919AE\", \"province_id\": 0}",
"open_url": "snssdk143://detail?groupid=6642216913403379982&ad_id=1621634988588039",
"track_url": "",
"web_url": "http://toutiao-sports.vvsvc.com/?app_from=ttkp",
"image_mode": 0,
"splash_id": 1621634988588039,
"max_display_time_ms": 4000,
"skip_btn": 1,
"id": 1621634988588039,
"display_density": "1242x2208",
"expire_seconds": 5,
"display_after": 8640000,
"title": "\u4f53\u575b\u7535\u5f71\u8282\uff0c\u8c01\u662f\u771f\u620f\u7cbe",
"click_track_url_list": [],
"expire_timestamp": 1546704000.0,
"track_url_list": [],
"image_info": {
"width": 1242,
"url_list": [
{
"url": "https://sf1-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20"
},
{
"url": "https://sf3-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20"
},
{
"url": "https://sf6-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20"
}
],
"uri": "web.business.image/201901035d0d679e106268264da89c20",
"height": 2208
},
"type": "web",
"display_time": 3,
"repeat": 0,
"splash_load_type": 3,
"predownload": 31,
"web_title": "\u4f53\u575b\u7535\u5f71\u8282\uff0c\u8c01\u662f\u771f\u620f\u7cbe",
"click_btn": 0,
"banner_mode": 0,
"url": "https://sf1-ttcdn-tos.pstatp.com/obj/web.business.image/201901035d0d679e106268264da89c20",
"display_time_ms": 3000,
"action": ""
}
],
"display_area": [],
"splash_load_interval": 20
}
}

View File

@ -1,62 +0,0 @@
package main
import (
"log"
crontab "474420502.top/eson/crontabex"
"474420502.top/eson/gjson"
"474420502.top/eson/imitater"
"474420502.top/test/logdb"
)
func main() {
imitater.Register("info", &TaskEx{})
person := imitater.NewPerson()
person.Config("task.yaml")
person.Execute()
}
// TaskEx 任务相关类
type TaskEx struct {
imitater.Task
db *logdb.LogDB
}
// Init 初始化函数
func (te *TaskEx) Init() {
te.db = logdb.New("../logdb.yaml")
}
// Execute 执行过程的方法
func (te *TaskEx) Execute(glocal map[string]interface{}) imitater.ITask {
resp, err := te.Request()
if err != nil {
log.Println(err)
return te
}
var adDataList []string
if gjson.Valid(resp.Content()) {
P := gjson.Parse(resp.Content())
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`splash`)
if adData.Exists() {
for _, result := range adData.Array() {
adDataList = append(adDataList, result.String())
}
}
}
} else {
log.Println("be careful:", resp.Content())
}
if imitater.ADDataSave(te, te.db, adDataList) {
te.GetCrontab().SetStatus(crontab.SExecuteOK, true)
} else {
te.GetCrontab().SetStatus(crontab.SExecuteOK, false)
}
return te
}

View File

@ -1,38 +0,0 @@
package main
import (
"io/ioutil"
"testing"
"474420502.top/eson/gjson"
)
func TestTaskEx_Execute(t *testing.T) {
data, err := ioutil.ReadFile("../splash.json")
if err != nil {
panic(err)
}
var adDataList []string
if gjson.Valid(string(data)) {
P := gjson.Parse(string(data))
data := P.Get(`data`)
if data.Exists() {
adData := data.Get(`splash`)
if adData.Exists() {
for _, result := range adData.Array() {
adDataList = append(adDataList, result.String())
}
} else {
t.Error(`expr error`)
}
}
}
if len(adDataList) != 1 {
t.Error("adDataList != 1")
t.Error(adDataList)
}
}

View File

@ -1 +0,0 @@
curl -X 'GET' 'https://lf.snssdk.com/api/ad/splash/news_article/v14/?_unused=0&carrier=%E4%B8%AD%E5%9B%BD%E8%81%94%E9%80%9A&mcc_mnc=46001&ad_area=1080x1854&sdk_version=1.5.5&os_api=23&device_platform=android&os_version=6.0&display_density=1080x1920&dpi=480&device_brand=Meizu&device_type=U20&bh=369&display_dpi=480&density=3.0&ac=wifi&channel=meizu&aid=13&app_name=news_article&language=zh&iid=56493295608&device_id=53061747912&version_code=704&version_name=7.0.4&ab_version=679615%2C664543%2C680643%2C666150%2C486952%2C651365%2C662176%2C680305%2C675827%2C571131%2C665174%2C674057%2C594583%2C612193%2C641906%2C170988%2C680911%2C643892%2C659018%2C374116%2C655402%2C678350%2C550042%2C435216%2C603544%2C659218%2C680163%2C678792%2C649426%2C614096%2C677129%2C377573%2C522766%2C669470%2C416055%2C677236%2C558139%2C555254%2C679178%2C603441%2C596391%2C660508%2C598626%2C644855%2C680371%2C603386%2C603398%2C603404%2C603406%2C638927%2C681323%2C661907%2C662644%2C668775%2C673945%2C629152%2C607361%2C609337%2C666967%2C635530%2C652363%2C662099%2C641413%2C673358%2C664194%2C654355%2C669191%2C671245%2C681071%2C680507%2C667071%2C622716%2C672874%2C680352%2C640997%2C641077%2C671798%2C668774%2C679516%2C631595%2C469022%2C676441%2C679733%2C673696%2C554836%2C679903%2C549647%2C644131%2C572465%2C482355%2C644057%2C615291%2C606548%2C681182%2C678933%2C673168%2C678315%2C678279%2C671426%2C546703%2C641191%2C281297%2C664118%2C325620%2C678473%2C665472%2C625066%2C662507%2C663955%2C668676%2C664311%2C680283%2C638335%2C467515%2C679101%2C679985%2C678876%2C595556%2C679519%2C679764%2C669426%2C670151%2C661450%2C654124%2C680983%2C660830%2C679538%2C656557%2C633487%2C662683%2C661781%2C457481%2C649400%2C655989%2C677104&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=100167%2C94567%2C102754%2C181431&ab_feature=94567%2C102754&abflag=3&ssmix=a&uuid=861578036046061&openudid=d67c17290ee19a25&manifest_version_code=704&resolution=1080*1920&update_version_code=70412&_rticket=1546587650823&fp=2ST_F255JlQZFlcuJ2U1FYmeFzGS&tma_jssdk_version=1.8.0.4&pos=5r_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupaukqKmxv_zw_O3R_On06ej5-L-nr66zrairqqWo4A%3D%3D&rom_version=23&plugin=26958&ts=1546587650&as=a28520c22210acee6f4355&mas=00cd3f14e3393d1162dfbf7b40be8d6139c40ea84206686ee1' -H 'Host: lf.snssdk.com' -H 'Connection: keep-alive' -H 'Cookie: odin_tt=b8dec487941fe35d2148bf9724de5e6659e65cf98541eee4f3573236e60d5f3c02a83ba067b8cd80a6685ac034db06ac; UM_distinctid=1653b99a5601-0c98c05be-2d4e2043-38400-1653b99a5641a; CNZZDATA1264530760=838951947-1534301713-%7C1534748760; __tea_sdk__ssid=290d7879-cb91-4e4c-aecd-b6239e2f04dc; tt_webid=6598313444424123907; __tea_sdk__user_unique_id=6598313444424123907; sid_guard=897e58fa1fa260bad70d802cbeee9225%7C1536824928%7C5184000%7CMon%2C+12-Nov-2018+07%3A48%3A48+GMT; CNZZDATA1263676333=575629806-1537840457-null%7C1537840457; qh[360]=1; install_id=56493295608; ttreq=1$b75d7a1dcbdafb18503a62139b7d79dc7aeeddb2' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1546587650838' -H 'sdk-version: 1' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 6.0; U20 Build/MRA58K) NewsArticle/7.0.4 cronet/TTNetVersion:a729d5c3 2018-11-25' --compressed

View File

@ -1,19 +0,0 @@
mode : 0
# proxies : "socks5://10.10.0.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.0.1:8080", "socks5://10.10.0.1:8082", "socks5://10.10.0.1:8083", "socks5://10.10.0.1:8085", "socks5://10.10.0.1:8087", "socks5://10.10.0.1:8088", "socks5://10.10.0.1:8091"]
retry : 0
timeout: 12
priority : 10000
curls : "@task.curl" # 支持, 列表 与 单项字符串
task: "info"
device : "eson-OnePlus"
platform : "Android"
# area_cc : 4401
channel : 105
media : 55
spider_id : 10001004
catch_account_id : 1001004
crontab: "f=180|s=30"

View File

@ -1 +0,0 @@
../vendor

View File

@ -1,10 +0,0 @@
#! /usr/bin/python3
import glob
import os
if __name__ == "__main__":
for p in glob.glob("./*"):
if os.path.isdir(p):
bn = os.path.basename(p)
os.system("cd {0} && go build ./ && screen -L -dmS {0} ./{0}".format(bn))

View File

@ -1,9 +0,0 @@
#! /usr/bin/python3
import glob
import os
if __name__ == "__main__":
for p in glob.glob("./*"):
if os.path.isdir(p):
bn = os.path.basename(p)
os.system("cd {0} && screen -X -S {0} quit ".format(bn))

View File

@ -1 +0,0 @@
package crontab

View File

@ -1,351 +0,0 @@
package crontab
import (
"errors"
"fmt"
"log"
"reflect"
"regexp"
"strings"
"time"
"github.com/satori/go.uuid"
"474420502.top/eson/structure/circular_linked"
"github.com/Pallinder/go-randomdata"
"github.com/davecgh/go-spew/spew"
)
// StatusType 设置状态的类型
type StatusType int
const (
_ StatusType = iota
// SExecuteOK 设置这个次成功或者失败的状态
SExecuteOK
// SExecuteSleep 设置下次Sleep时间, 只影响一次
SExecuteSleep
// SExecuteCrontab 设置改写Crontab, 覆盖以前的Crontab
SExecuteCrontab
)
// Force 强制下次执行
type Force struct {
sleep time.Duration
}
// NextTime 下次执行时间
func (force *Force) NextTime() time.Time {
return time.Now().Add(force.sleep)
}
// Crontab 的string解析
type Crontab struct {
crontab string
uid uuid.UUID
force *Force
min []timePointer
hour []timePointer
day []timePointer
month []timePointer
week []timePointer
WillPlans []time.Time
SkipPlans []time.Time
YearPlan *trieYear
interval *clinked.CircularLinked
lastStatus bool
isCalculated bool
trueCount int
failCount int
nextTime time.Time
}
// NewCrontab create 一个crontab
func NewCrontab(crontab string) *Crontab {
cron := &Crontab{}
cron.crontab = strings.TrimSpace(crontab)
cron.FromString(cron.crontab)
return cron
}
// UnmarshalYAML 添加序列化接口 Marshal没实现, 需要的时候可以自己添加
func (cron *Crontab) UnmarshalYAML(unmarshal func(interface{}) error) error {
var buf string
err := unmarshal(&buf)
if err != nil {
return nil
}
if err := cron.FromString(buf); err != nil {
return err
}
return nil
}
// SetStatus 设置上次状态 true false
func (cron *Crontab) SetStatus(statusType StatusType, statusValue ...interface{}) {
switch statusType {
case SExecuteOK:
if cron.interval != nil {
cron.lastStatus = statusValue[0].(bool)
}
case SExecuteSleep:
force := new(Force)
ivalue := statusValue[0]
switch value := ivalue.(type) {
case int:
force.sleep = time.Duration(value) * time.Second
case int64:
force.sleep = time.Duration(value) * time.Second
case time.Duration:
force.sleep = value
default:
panic(errors.New("statusValue type is error, the type is" + reflect.TypeOf(ivalue).String()))
}
cron.force = force
case SExecuteCrontab:
crontab := statusValue[0].(string)
cron.crontab = strings.TrimSpace(crontab)
cron.FromString(cron.crontab)
default:
panic(errors.New("StatusType is unknown, the type " + reflect.TypeOf(statusType).String()))
}
}
// // GetStatus 获取上次状态 true false
// func (cron *Crontab) GetStatus() (status bool) {
// return cron.lastStatus
// }
// TimeUp 是否时间快到, 时间间隔调用完TimeUp后必须调用NextTime, 为了精准控制时间
func (cron *Crontab) TimeUp() bool {
if cron.interval != nil {
return cron.intervalTimeUp()
}
return cron.linuxTimeUp()
}
// NextTime 返回下次任务的时间
func (cron *Crontab) NextTime() time.Time {
if cron.force != nil {
nt := cron.force.NextTime()
cron.force = nil
return nt
}
if cron.interval != nil {
if !cron.isCalculated {
now := time.Now()
cron.intervalCalculateNextTime(now)
}
return cron.nextTime
}
if len(cron.WillPlans) > 0 {
return cron.WillPlans[0]
}
return time.Now().Add(time.Second * 2)
}
func (cron *Crontab) String() string {
return fmt.Sprintf("min:%s\nhour:%s\nday:%s\nmonth:%s\nweek:%s\n", spew.Sdump(cron.min), spew.Sdump(cron.hour), spew.Sdump(cron.day), spew.Sdump(cron.month), spew.Sdump(cron.week))
}
// FromString 解析crontab 的 表达式
func (cron *Crontab) FromString(crontab string) error {
crontab = cron.crontab
uid, err := uuid.NewV4()
if err != nil {
panic(err)
}
cron.uid = uid
cron.interval = nil
cron.min = nil
cron.hour = nil
cron.day = nil
cron.month = nil
cron.week = nil
cron.WillPlans = nil
cron.SkipPlans = nil
cron.YearPlan = nil
matches := regexp.MustCompile("[^ ]+").FindAllString(crontab, -1)
mlen := len(matches)
switch mlen {
case 1:
// "f1-2|5-10x5,f1|10m,10-15,f1"
cron.lastStatus = true
cron.isCalculated = true
cron.nextTime = time.Now()
cron.interval = clinked.NewCircularLinked()
var intervalList []interface{}
intervalList = parseIntervalString(matches[0])
cron.interval.Append(intervalList...)
case 5:
cron.min = createTimePointer(matches[0], 0, 59, true)
cron.hour = createTimePointer(matches[1], 0, 23, true)
cron.day = createTimePointer(matches[2], 1, 31, false)
cron.month = createTimePointer(matches[3], 1, 12, true)
cron.week = createTimePointer(matches[4], 0, 6, true)
cron.createYearPlan()
cron.TimeUp()
default:
return errors.New("mathches len != want, check crontab string")
}
return nil
}
// createYearPlan 创建年度计划
func (cron *Crontab) createYearPlan() {
cron.YearPlan = newTrieYear()
cron.YearPlan.FromCrontab(cron)
}
func (cron *Crontab) linuxTimeUp() bool {
now := time.Now()
maxlen := 1000
createlen := 500
plen := len(cron.WillPlans)
if plen <= createlen { // 如果当前生成的计划表少于 限制的500. 就生成新的计划表
var lastplan time.Time
if plen == 0 {
lastplan = now
} else {
lastplan = cron.WillPlans[plen-1].Add(time.Minute)
}
if !cron.YearPlan.CheckYear() {
cron.createYearPlan()
}
timeplans := cron.YearPlan.GetPlanTime(cron, lastplan, uint(maxlen-plen))
cron.WillPlans = append(cron.WillPlans, timeplans...)
}
if len(cron.WillPlans) > 0 {
istimeup := false
for i := 0; i < maxlen; i++ {
// 统计过了多少计划任务时间表 i - 1
if now.Unix() >= cron.WillPlans[i].Unix() {
istimeup = true
} else {
// 清除过时的计划任务时间表
if istimeup {
if i-1 > 0 {
cron.SkipPlans = append(cron.SkipPlans, cron.WillPlans[0:i-1]...)
if len(cron.SkipPlans) >= maxlen+200 {
cron.SkipPlans = cron.SkipPlans[200:]
}
}
cron.WillPlans = cron.WillPlans[i:]
return istimeup
}
return istimeup
}
}
// 如果所有计划表都不符合, 全部放到忽略的计划表上, 这个表方便打印查看, 因为程序执行超时过了多少计划任务
cron.SkipPlans = append(cron.SkipPlans, cron.WillPlans...)
cron.WillPlans = nil
return istimeup
}
log.Panicln("error willplans range")
return false
}
// IntervalCalculateNextTime 计算时间间隔的下次时间
func (cron *Crontab) intervalCalculateNextTime(now time.Time) {
iv := cron.interval.Cursor().GetValue().(*hInterval)
isecond := 0
if iv.PlanFailCount.Size() == 0 && len(iv.PlanFail) == 0 {
cron.lastStatus = true
}
if cron.lastStatus {
cron.trueCount++
cron.failCount = 0
isecond = intervalPriorityListISecond(&iv.PlanTrueCount, cron.trueCount)
if isecond == -1 {
if len(iv.PlanTrue) > 0 {
idx := randomdata.Number(len(iv.PlanTrue))
lr := iv.PlanTrue[idx]
isecond = randomdata.Number(lr.left, lr.right+1)
} else {
isecond = 0
}
}
fmt.Println(time.Now().Format("01-02 15:04:05"), cron.uid.String(), "success:", cron.trueCount, " wait:", isecond)
} else {
cron.failCount++
cron.trueCount = 0
isecond = intervalPriorityListISecond(&iv.PlanFailCount, cron.failCount)
if isecond == -1 {
if len(iv.PlanFail) > 0 {
idx := randomdata.Number(len(iv.PlanFail))
lr := iv.PlanFail[idx]
isecond = randomdata.Number(lr.left, lr.right+1)
} else {
isecond = 0
}
}
fmt.Println(time.Now().Format("01-02 15:04:05"), cron.uid.String(), "fail:", cron.failCount, " wait:", isecond)
}
iv.Count--
if iv.Count <= 0 {
iv.reset()
cron.interval.MoveNext()
}
cron.isCalculated = true
cron.nextTime = now.Add(time.Duration(isecond) * time.Second)
}
func (cron *Crontab) intervalTimeUp() bool {
if cron.isCalculated { // 需要调用nexttime()才能正常正常计算下次的时间
now := time.Now()
if now.Unix() >= cron.nextTime.Unix() {
cron.isCalculated = false
return true
}
}
return false
}

View File

@ -1,205 +0,0 @@
package crontab
import (
"errors"
"strconv"
"strings"
"474420502.top/eson/structure/priority_list"
randomdata "github.com/Pallinder/go-randomdata"
)
type randLR struct {
left, right int
}
// NodeCount 用于priority_list
type NodeCount struct {
plist.Node
randLR
}
type hInterval struct {
PlanFailCount plist.PriorityList
PlanTrueCount plist.PriorityList
PlanFail []randLR
PlanTrue []randLR
Count int
ConstCount int
}
func (interval *hInterval) reset() {
interval.Count = interval.ConstCount
}
// Compare NodeCount比较函数
func (rlr *NodeCount) Compare(v plist.INode) bool {
return rlr.GetValue().(int) > v.GetValue().(int)
}
func parseIntervalFail(interval *hInterval, FN string) {
scharIndex := strings.Index(FN, ">")
if scharIndex != -1 {
if FN[scharIndex+1] != '=' {
panic(errors.New("= is not exist"))
}
fc := FN[0:scharIndex]
flr := FN[scharIndex+2:]
node := new(NodeCount)
node.SetValue(getInt(fc[1:]))
node.randLR = parseRandLR(flr)
interval.PlanFailCount.Insert(node)
} else {
if FN[1] != '=' {
panic(errors.New("= is not exist"))
}
fvalue := FN[2:]
interval.PlanFail = append(interval.PlanFail, parseRandLR(fvalue))
}
}
func parseIntervalSuccess(interval *hInterval, FN string) {
scharIndex := strings.Index(FN, ">")
if scharIndex != -1 {
if FN[scharIndex+1] != '=' {
panic(errors.New("= is not exist"))
}
tc := FN[0:scharIndex]
tlr := FN[scharIndex+2:]
node := new(NodeCount)
node.SetValue(getInt(tc[1:]))
node.randLR = parseRandLR(tlr)
interval.PlanTrueCount.Insert(node)
} else {
if FN[1] != '=' {
panic(errors.New("= is not exist"))
}
tvalue := FN[2:]
interval.PlanTrue = append(interval.PlanTrue, parseRandLR(tvalue))
}
}
func parseIntervalString(crontab string) []interface{} {
var result []interface{}
values := strings.Split(crontab, ",")
for _, value := range values {
interval := &hInterval{}
// 次数
valuesCounts := strings.Split(value, "x")
switch len(valuesCounts) {
case 1:
interval.ConstCount = 1
case 2:
count, err := strconv.Atoi(valuesCounts[1])
if err != nil {
panic(err)
}
interval.ConstCount = count
default:
panic("valuesCounts error, the len is not in range")
}
// 统计失败与普通间隔值的数组
failAndNormal := valuesCounts[0]
valuesFN := strings.Split(failAndNormal, "|")
for _, FN := range valuesFN {
if FN == "" {
continue
}
switch FN[0] {
case 'f', 'F':
parseIntervalFail(interval, FN)
case 's', 'S':
parseIntervalSuccess(interval, FN)
default:
FN = "s=" + FN
parseIntervalSuccess(interval, FN)
}
}
interval.reset()
result = append(result, interval)
}
return result
}
func parseRandLR(lrvalue string) randLR {
vlen := len(lrvalue)
lastchar := lrvalue[vlen-1]
lr := strings.Split(lrvalue, "-")
switch len(lr) {
case 1:
lr := randLR{parseTimeValue(lr[0], lastchar), parseTimeValue(lr[0], lastchar)}
return lr
case 2:
lr := randLR{parseTimeValue(lr[0], lastchar), parseTimeValue(lr[1], lastchar)}
return lr
default:
panic("lr is error")
}
}
// intervalPriorityListISecond 获取优先链表比较的值
func intervalPriorityListISecond(planlist *plist.PriorityList, count int) int {
if planlist.Size() > 0 {
node := new(NodeCount)
node.SetValue(count)
iwantNode := planlist.GetCompare(node)
if iwantNode != nil {
wantNode := iwantNode.(*NodeCount)
lr := wantNode.randLR
return randomdata.Number(lr.left, lr.right+1)
}
}
return -1
}
func getInt(v string) int {
vint, err := strconv.Atoi(v)
if err != nil {
panic(err)
}
return vint
}
func parseTimeValue(v string, lastchar byte) int {
vlen := len(v)
switch lastchar {
case 's':
return getInt(v[:vlen-1])
case 'm':
return getInt(v[:vlen-1]) * 60
case 'h':
return getInt(v[:vlen-1]) * 3600
case 'd':
return getInt(v[:vlen-1]) * 3600 * 24
default:
return getInt(v)
}
}

View File

@ -1,269 +0,0 @@
package crontab
import (
"time"
)
type minuteNode struct {
}
func newMinuteNode() *minuteNode {
return &minuteNode{}
}
type hourNode struct {
Minute [60]*minuteNode
}
func (hour *hourNode) CreateMinute(nminute int) {
min := &minuteNode{}
hour.Minute[nminute] = min
}
func newHourNode() *hourNode {
return &hourNode{}
}
type dayNode struct {
IsClear bool
Week time.Weekday
Hour [24]*hourNode
}
func (day *dayNode) CreateHour(nhour int) {
hour := &hourNode{}
day.Hour[nhour] = hour
}
func newDayNode(curday *time.Time) *dayNode {
day := &dayNode{}
week := curday.Weekday()
day.Week = week
day.IsClear = true
return day
}
type monthNode struct {
MaxDay int
First *time.Time
Day [32]*dayNode
}
func (month *monthNode) CreateDay(nday int) {
day := month.First.AddDate(0, 0, nday-1)
month.Day[nday] = newDayNode(&day)
}
func newMonthNode(year, month int) *monthNode {
Month := &monthNode{}
First := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local)
Month.First = &First
Month.MaxDay = Month.First.AddDate(0, 1, -1).Day()
return Month
}
type trieYear struct {
Year int
Month [13]*monthNode
}
// CheckYear 跨年判断
func (ty *trieYear) CheckYear() bool {
year := time.Now().Year()
return ty.Year == year
}
func (ty *trieYear) clearHour() {
for _, month := range ty.Month {
if month != nil {
for _, day := range month.Day {
if day != nil {
for i := 0; i <= 23; i++ {
day.Hour[i] = nil
day.IsClear = true
}
}
}
}
}
}
func newTrieYear() *trieYear {
ty := trieYear{}
ty.Year = time.Now().Year()
return &ty
}
// GetPlanTime 获取计划表
func (ty *trieYear) GetPlanTime(cron *Crontab, aftertime time.Time, count uint) []time.Time {
now := aftertime
nmonth := int(now.Month())
nday := now.Day()
nhour := now.Hour()
nminute := now.Minute()
var result []time.Time
for i := 1; i <= 12; i++ {
if i < nmonth {
continue
}
month := ty.Month[i]
if month != nil {
for j := 1; j <= 31; j++ {
if nmonth == i {
if j < nday { // 获取当天的计划表
continue
}
}
day := month.Day[j]
if day != nil {
if day.IsClear {
insertHour(cron, day) // 生成小时的计划表
}
for k := 0; k <= 23; k++ {
if nmonth == i && nday == j {
if k < nhour {
continue
}
}
hour := day.Hour[k]
// 遍历24小时 生成需要的计划表
if hour != nil {
for n := 0; n <= 59; n++ {
if nmonth == i && nday == j && nhour == k {
if n < nminute {
continue
}
}
min := hour.Minute[n]
if min != nil {
curTime := time.Date(ty.Year, time.Month(i), j, k, n, 0, 0, time.Local)
result = append(result, curTime)
count-- // 统计完当前需要的计划任务后就结束
if count <= 0 {
ty.clearHour()
return result
}
}
}
}
}
}
}
}
}
ty.clearHour()
return result
}
// FromCrontab 从Crontab生成树
func (ty *trieYear) FromCrontab(cron *Crontab) {
// 月的填充
for _, month := range cron.month {
left := month.left
right := month.right
for i := left; i <= right; i += month.per {
curMonth := newMonthNode(ty.Year, i)
ty.Month[i] = curMonth
// 天的填充
insertDay(cron, curMonth)
}
}
}
func filterDay(cron *Crontab, curday *dayNode) bool {
for _, w := range cron.week {
if w.isAll {
return false
}
for n := w.left; n <= w.right; n += w.per {
if n == int(curday.Week) {
return false
}
}
}
return true
}
func insertDay(cron *Crontab, curMonth *monthNode) {
for _, day := range cron.day {
left := day.left
if left < 0 {
left += curMonth.MaxDay + 1
}
right := day.right
if right < 0 {
right += curMonth.MaxDay + 1
}
for j := left; j <= right; j += day.per {
curMonth.CreateDay(j)
curDay := curMonth.Day[j]
if filterDay(cron, curDay) {
curMonth.Day[j] = nil
} else {
// insertHour(cron, curDay)
}
}
}
}
func insertHour(cron *Crontab, curDay *dayNode) {
curDay.IsClear = false
// 时的填充
for _, hour := range cron.hour {
left := hour.left
right := hour.right
for k := left; k <= right; k += hour.per {
curDay.CreateHour(k)
curHour := curDay.Hour[k]
insertMinute(cron, curHour)
}
}
}
func insertMinute(cron *Crontab, curHour *hourNode) {
for _, min := range cron.min {
left := min.left
right := min.right
for l := left; l <= right; l += min.per {
curHour.CreateMinute(l)
}
}
}

View File

@ -1,107 +0,0 @@
package crontab
import (
"errors"
"fmt"
"strconv"
"strings"
)
type timePointer struct {
left, right int
leftlimit, rightlimit int
per int
isAll bool
}
func (tp *timePointer) String() string {
return fmt.Sprintf("left: %d, right: %d, leftlimit: %d, rightlimit: %d, per: %d", tp.left, tp.right, tp.leftlimit, tp.rightlimit, tp.per)
}
func createTimePointer(min string, llimit, rlimit int, fixedLeftRight bool) []timePointer {
var result []timePointer
exelist := strings.Split(min, ",")
for _, exe := range exelist {
tp := timePointer{}
takeper := strings.Split(exe, "/") // per
var rangevalue, per string
if len(takeper) == 1 {
rangevalue = exe
per = "1"
} else {
rangevalue = takeper[0]
per = takeper[1]
}
// takeRange
be := strings.Split(rangevalue, "-")
var left, rigth string
switch len(be) {
case 1:
left = be[0]
rigth = be[0]
case 2:
left = be[0]
rigth = be[1]
default:
panic(errors.New("range value is > 2"))
}
if left == "*" {
tp.left = llimit
} else {
ileft, err := strconv.Atoi(strings.Replace(left, "^", "-", -1))
if err != nil {
panic(err)
}
tp.left = ileft
}
if rigth == "*" {
tp.right = rlimit
} else {
iright, err := strconv.Atoi(strings.Replace(rigth, "^", "-", -1))
if err != nil {
panic(err)
}
tp.right = iright
}
iper, err := strconv.Atoi(per)
if err != nil {
panic(err)
}
tp.per = iper
tp.leftlimit = llimit
tp.rightlimit = rlimit
// 修正左值
leftfixed := tp.left
if leftfixed < 0 {
leftfixed += tp.rightlimit + 1
if fixedLeftRight {
tp.left = leftfixed
}
}
rightfixed := tp.right
if rightfixed < 0 {
rightfixed += tp.rightlimit + 1
if fixedLeftRight {
tp.right = rightfixed
}
}
// 全部符合 当左等于左 且 右等于右最大 并且 per == 1
if leftfixed == tp.leftlimit && rightfixed == tp.rightlimit && tp.per == 1 {
tp.isAll = true
}
result = append(result, tp)
}
return result
}

View File

@ -1,2 +0,0 @@
*.pyc
*.vscode

View File

@ -1,230 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package curl2info
import (
"net"
"net/http"
"strings"
)
var isTokenTable = [127]bool{
'!': true,
'#': true,
'$': true,
'%': true,
'&': true,
'\'': true,
'*': true,
'+': true,
'-': true,
'.': true,
'0': true,
'1': true,
'2': true,
'3': true,
'4': true,
'5': true,
'6': true,
'7': true,
'8': true,
'9': true,
'A': true,
'B': true,
'C': true,
'D': true,
'E': true,
'F': true,
'G': true,
'H': true,
'I': true,
'J': true,
'K': true,
'L': true,
'M': true,
'N': true,
'O': true,
'P': true,
'Q': true,
'R': true,
'S': true,
'T': true,
'U': true,
'W': true,
'V': true,
'X': true,
'Y': true,
'Z': true,
'^': true,
'_': true,
'`': true,
'a': true,
'b': true,
'c': true,
'd': true,
'e': true,
'f': true,
'g': true,
'h': true,
'i': true,
'j': true,
'k': true,
'l': true,
'm': true,
'n': true,
'o': true,
'p': true,
'q': true,
'r': true,
's': true,
't': true,
'u': true,
'v': true,
'w': true,
'x': true,
'y': true,
'z': true,
'|': true,
'~': true,
}
func isTokenRune(r rune) bool {
i := int(r)
return i < len(isTokenTable) && isTokenTable[i]
}
// ReadRawCookies parses all "Cookie" values from the rawcookie and
// returns the successfully parsed Cookies.
//
// if filter isn't empty, only cookies of that name are returned
func ReadRawCookies(soptions string, filter string) []*http.Cookie {
line := soptions
cookies := []*http.Cookie{}
parts := strings.Split(strings.TrimSpace(line), ";")
if len(parts) == 1 && parts[0] == "" {
return cookies
}
// Per-line attributes
for i := 0; i < len(parts); i++ {
parts[i] = strings.TrimSpace(parts[i])
if len(parts[i]) == 0 {
continue
}
name, val := parts[i], ""
if j := strings.Index(name, "="); j >= 0 {
name, val = name[:j], name[j+1:]
}
if !isCookieNameValid(name) {
continue
}
if filter != "" && filter != name {
continue
}
val, ok := parseCookieValue(val, true)
if !ok {
continue
}
cookies = append(cookies, &http.Cookie{Name: name, Value: val})
}
return cookies
}
// validCookieDomain returns whether v is a valid cookie domain-value.
func validCookieDomain(v string) bool {
if isCookieDomainName(v) {
return true
}
if net.ParseIP(v) != nil && !strings.Contains(v, ":") {
return true
}
return false
}
// isCookieDomainName returns whether s is a valid domain name or a valid
// domain name with a leading dot '.'. It is almost a direct copy of
// package net's isDomainName.
func isCookieDomainName(s string) bool {
if len(s) == 0 {
return false
}
if len(s) > 255 {
return false
}
if s[0] == '.' {
// A cookie a domain attribute may start with a leading dot.
s = s[1:]
}
last := byte('.')
ok := false // Ok once we've seen a letter.
partlen := 0
for i := 0; i < len(s); i++ {
c := s[i]
switch {
default:
return false
case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
// No '_' allowed here (in contrast to package net).
ok = true
partlen++
case '0' <= c && c <= '9':
// fine
partlen++
case c == '-':
// Byte before dash cannot be dot.
if last == '.' {
return false
}
partlen++
case c == '.':
// Byte before dot cannot be dot, dash.
if last == '.' || last == '-' {
return false
}
if partlen > 63 || partlen == 0 {
return false
}
partlen = 0
}
last = c
}
if last == '-' || partlen > 63 {
return false
}
return ok
}
func validCookieValueByte(b byte) bool {
return 0x20 <= b && b < 0x7f && b != '"' && b != ';' && b != '\\'
}
func parseCookieValue(raw string, allowDoubleQuote bool) (string, bool) {
// Strip the quotes, if present.
if allowDoubleQuote && len(raw) > 1 && raw[0] == '"' && raw[len(raw)-1] == '"' {
raw = raw[1 : len(raw)-1]
}
for i := 0; i < len(raw); i++ {
if !validCookieValueByte(raw[i]) {
return "", false
}
}
return raw, true
}
func isCookieNameValid(raw string) bool {
if raw == "" {
return false
}
return strings.IndexFunc(raw, isNotToken) < 0
}
func isNotToken(r rune) bool {
return !isTokenRune(r)
}

View File

@ -1,224 +0,0 @@
package curl2info
import (
"io/ioutil"
"log"
"os"
"regexp"
"strconv"
"strings"
"474420502.top/eson/requests"
)
func init() {
optionTrie = newTrie()
oelist := []*optionExecute{
{"-H", 10, parseHeader, nil},
{"-X", 10, parseOptX, nil},
{"-A", 15, parseUserAgent, &extract{re: "^-A +(.+)", execute: extractData}},
{"-I", 15, parseOptI, nil},
{"-d", 10, parseBodyASCII, &extract{re: "^-d +(.+)", execute: extractData}},
{"-u", 15, parseUser, &extract{re: "^-u +(.+)", execute: extractData}},
{"-k", 15, parseInsecure, nil},
// Body
{"--data", 10, parseBodyASCII, &extract{re: "--data +(.+)", execute: extractData}},
{"--data-urlencode", 10, parseBodyURLEncode, &extract{re: "--data-urlencode +(.+)", execute: extractData}},
{"--data-binary", 10, parseBodyBinary, &extract{re: "--data-binary +(.+)", execute: extractData}},
{"--data-ascii", 10, parseBodyASCII, &extract{re: "--data-ascii +(.+)", execute: extractData}},
{"--data-raw", 10, parseBodyRaw, &extract{re: "--data-raw +(.+)", execute: extractData}},
//"--"
{"--header", 10, parseHeader, nil},
{"--insecure", 15, parseInsecure, nil},
{"--user-agent", 15, parseUserAgent, &extract{re: "--user-agent +(.+)", execute: extractData}},
{"--user", 15, parseUser, &extract{re: "--user +(.+)", execute: extractData}},
{"--connect-timeout", 15, parseTimeout, &extract{re: "--connect-timeout +(.+)", execute: extractData}},
// 自定义
{"--task", 10, parseITask, &extract{re: "--task +(.+)", execute: extractData}},
{"--crontab", 10, parseCrontab, &extract{re: "--crontab +(.+)", execute: extractData}},
{"--name", 10, parseName, &extract{re: "--name +(.+)", execute: extractData}},
}
for _, oe := range oelist {
optionTrie.Insert(oe)
}
// log.Println("support options:", optionTrie.AllWords())
}
// extract 用于提取设置的数据
type extract struct {
re string
execute func(re, soption string) string
}
func (et *extract) Execute(soption string) string {
return et.execute(et.re, soption)
}
// OptionTrie 设置的前缀树
var optionTrie *hTrie
type optionExecute struct {
Prefix string
Priority int
Parse func(*CURL, string) // 执行函数
Extract *extract // 提取的方法结构与参数
}
func (oe *optionExecute) GetWord() string {
return oe.Prefix + " "
}
func (oe *optionExecute) BuildFunction(curl *CURL, soption string) *parseFunction {
data := soption
if oe.Extract != nil {
data = oe.Extract.Execute(data)
}
return &parseFunction{ParamCURL: curl, ParamData: data, ExecuteFunction: oe.Parse, Priority: oe.Priority}
}
func judgeOptions(u *CURL, soption string) *parseFunction {
word := trieStrWord(soption)
if ioe := optionTrie.SearchDepth(&word); ioe != nil {
oe := ioe.(*optionExecute)
return oe.BuildFunction(u, soption)
}
log.Println(soption, " not this option")
return nil
}
func extractData(re, soption string) string {
datas := regexp.MustCompile(re).FindStringSubmatch(soption)
return strings.Trim(datas[1], "'\"")
}
func parseName(u *CURL, value string) {
u.Name = value
}
func parseCrontab(u *CURL, value string) {
u.Crontab = value
}
func parseITask(u *CURL, value string) {
u.ITask = value
}
func parseTimeout(u *CURL, value string) {
timeout, err := strconv.Atoi(value)
if err != nil {
panic(err)
}
u.Timeout = timeout
}
func parseInsecure(u *CURL, soption string) {
u.Insecure = true
}
func parseUser(u *CURL, soption string) {
auth := strings.Split(soption, ":")
u.Auth = &requests.BasicAuth{User: auth[0], Password: auth[1]}
}
func parseUserAgent(u *CURL, value string) {
u.Header.Add("User-Agent", value)
}
func parseOptI(u *CURL, soption string) {
u.Method = "HEAD"
}
func parseOptX(u *CURL, soption string) {
matches := regexp.MustCompile("-X +(.+)").FindStringSubmatch(soption)
method := strings.Trim(matches[1], "'")
u.Method = method
}
func parseBodyURLEncode(u *CURL, data string) {
if u.Method != "" {
u.Method = "POST"
}
u.Body.SetPrefix(requests.TypeURLENCODED)
u.Body.SetIOBody(data)
}
func parseBodyRaw(u *CURL, data string) {
if u.Method != "" {
u.Method = "POST"
}
u.Body.SetPrefix(requests.TypeURLENCODED)
u.Body.SetIOBody(data)
}
func parseBodyASCII(u *CURL, data string) {
if u.Method != "" {
u.Method = "POST"
}
u.Body.SetPrefix(requests.TypeURLENCODED)
if data[0] != '@' {
u.Body.SetIOBody(data)
} else {
f, err := os.Open(data[1:])
if err != nil {
panic(err)
}
defer f.Close()
bdata, err := ioutil.ReadAll(f)
if err != nil {
panic(err)
}
u.Body.SetIOBody(bdata)
}
}
// 处理@ 并且替/r/n符号
func parseBodyBinary(u *CURL, data string) {
if u.Method != "" {
u.Method = "POST"
}
u.Body.SetPrefix(requests.TypeURLENCODED)
if data[0] != '@' {
u.Body.SetIOBody(data)
} else {
f, err := os.Open(data[1:])
if err != nil {
panic(err)
}
defer f.Close()
bdata, err := ioutil.ReadAll(f)
if err != nil {
panic(err)
}
bdata = regexp.MustCompile("\n|\r").ReplaceAll(bdata, []byte(""))
u.Body.SetIOBody(bdata)
}
}
func parseHeader(u *CURL, soption string) {
matches := regexp.MustCompile(`'([^:]+): ([^']+)'`).FindAllStringSubmatch(soption, 1)[0]
key := matches[1]
value := matches[2]
switch key {
case "Cookie":
u.Cookies = ReadRawCookies(value, "")
u.CookieJar.SetCookies(u.ParsedURL, u.Cookies)
case "Content-Type":
u.Body.SetPrefix(value)
default:
u.Header.Add(key, value)
}
}

View File

@ -1,153 +0,0 @@
package curl2info
import (
"fmt"
"net/http"
"net/http/cookiejar"
"net/url"
"regexp"
"strings"
"474420502.top/eson/requests"
)
// CURL 信息结构
type CURL struct {
ParsedURL *url.URL
Method string
Header http.Header
CookieJar http.CookieJar
Cookies []*http.Cookie
Body *requests.Body
Auth *requests.BasicAuth
Timeout int // second
Insecure bool
ITask string
Crontab string
Name string
}
// New new 一个 curl 出来
func New() *CURL {
u := &CURL{}
u.Insecure = false
u.Header = make(http.Header)
u.CookieJar, _ = cookiejar.New(nil)
u.Body = requests.NewBody()
u.Timeout = 30
return u
}
func (curl *CURL) String() string {
if curl != nil {
return fmt.Sprintf("Method: %s\nParsedURL: %s\nHeader: %s\nCookie: %s",
curl.Method, curl.ParsedURL.String(), curl.Header, curl.Cookies)
}
return ""
}
// CreateSession 创建Session
func (curl *CURL) CreateSession() *requests.Session {
ses := requests.NewSession()
ses.SetHeader(curl.Header)
ses.SetCookies(curl.ParsedURL, curl.Cookies)
ses.SetConfig(requests.CRequestTimeout, curl.Timeout)
if curl.Auth != nil {
ses.SetConfig(requests.CBasicAuth, curl.Auth)
}
if curl.Insecure {
ses.SetConfig(requests.CInsecure, curl.Insecure)
}
return ses
}
// CreateWorkflow 根据Session 创建Workflow
func (curl *CURL) CreateWorkflow(ses *requests.Session) *requests.Workflow {
var wf *requests.Workflow
if ses == nil {
ses = curl.CreateSession()
}
switch curl.Method {
case "HEAD":
wf = ses.Head(curl.ParsedURL.String())
case "GET":
wf = ses.Get(curl.ParsedURL.String())
case "POST":
wf = ses.Post(curl.ParsedURL.String())
case "PUT":
wf = ses.Put(curl.ParsedURL.String())
case "PATCH":
wf = ses.Patch(curl.ParsedURL.String())
case "OPTIONS":
wf = ses.Options(curl.ParsedURL.String())
case "DELETE":
wf = ses.Delete(curl.ParsedURL.String())
}
wf.SetBody(curl.Body)
return wf
}
// ParseRawCURL curl_bash 可以用trie改进 没空改
func ParseRawCURL(scurl string) (cURL *CURL) {
executor := newPQueueExecute()
curl := New()
if len(scurl) <= 4 {
panic("scurl error:" + scurl)
}
if scurl[0] == '"' && scurl[len(scurl)-1] == '"' {
scurl = strings.Trim(scurl, `"`)
} else if scurl[0] == '\'' && scurl[len(scurl)-1] == '\'' {
scurl = strings.Trim(scurl, `'`)
}
scurl = strings.TrimSpace(scurl)
scurl = strings.TrimLeft(scurl, "curl")
mathches := regexp.MustCompile(
`--[^ ]+ +'[^']+'|`+
`--[^ ]+ +[^ ]+|`+
`-[A-Za-z] +'[^']+'|`+
`-[A-Za-z] +[^ ]+|`+
` '[^']+'|`+
`--[a-z]+ {0,}`,
).FindAllString(scurl, -1)
for _, m := range mathches {
m = strings.TrimSpace(m)
switch v := m[0]; v {
case '\'':
purl, err := url.Parse(strings.Trim(m, "'"))
if err != nil {
panic(err)
}
curl.ParsedURL = purl
case '-':
exec := judgeOptions(curl, m)
if exec != nil {
executor.Push(exec)
}
}
}
for executor.Len() > 0 {
exec := executor.Pop()
exec.Execute()
}
if curl.Method == "" {
curl.Method = "GET"
}
return curl
}

View File

@ -1,483 +0,0 @@
package curl2info
import (
"container/heap"
"log"
"strings"
"github.com/davecgh/go-spew/spew"
)
// trieWord Trie 需要的Word接口
type trieWord interface {
GetWord() string
}
// TrieStrWord 最简单的TrieWord 结构
type trieStrWord string
// GetWord 获取单词
func (tsw *trieStrWord) GetWord() string {
return (string)(*tsw)
}
// Trie 前缀树
type hTrie struct {
isWord bool
value interface{}
char byte
prev *hTrie
next map[byte]*hTrie
}
// newTrie Initialize your data structure here.
func newTrie() *hTrie {
return &hTrie{next: make(map[byte]*hTrie)}
}
// Insert a word into the trie.
func (trie *hTrie) Insert(iword trieWord) {
cur := trie
word := iword.GetWord()
l := len(word)
for i := 0; i < l; i++ {
c := word[i]
if next, ok := cur.next[c]; ok {
cur = next
} else {
create := newTrie()
cur.next[c] = create
create.char = c
create.prev = cur
cur = create
}
}
cur.isWord = true
cur.value = iword
}
// AllWords 所有单词
func (trie *hTrie) AllWords() []string {
var result []string
for _, v := range trie.next {
look(v, "", &result)
}
return result
}
func look(cur *hTrie, content string, result *[]string) {
content += string(cur.char)
if cur.isWord {
*result = append(*result, content)
}
for _, v := range cur.next {
look(v, content, result)
}
}
// Remove 移除单词
func (trie *hTrie) Remove(word string) {
cur := trie
l := len(word)
for i := 0; i < l; i++ {
c := word[i]
if next, ok := cur.next[c]; ok {
cur = next
} else {
return
}
}
if cur != nil {
cur.isWord = false
cur.value = nil
lastchar := cur.char
if len(cur.next) == 0 {
for cur.isWord != true && cur.prev != nil {
lastchar = cur.char
cur = cur.prev
if len(cur.next) > 1 {
return
}
}
delete(cur.next, lastchar)
}
}
}
// SearchMostPrefix Returns if the word is in the trie.
func (trie *hTrie) SearchDepth(iword trieWord) interface{} {
cur := trie
word := iword.GetWord()
l := len(word)
var result interface{}
for i := 0; i < l; i++ {
c := word[i]
if next, ok := cur.next[c]; ok {
cur = next
if cur.isWord {
result = cur.value
} else {
result = nil
}
} else {
return result
}
}
return result
}
// Match Returns if the word is in the trie.
func (trie *hTrie) Match(iword trieWord) interface{} {
cur := trie
word := iword.GetWord()
l := len(word)
for i := 0; i < l; i++ {
c := word[i]
if next, ok := cur.next[c]; ok {
cur = next
} else {
return nil
}
}
return cur.value
}
// StartsWith Returns if there is any word in the trie that starts with the given prefix. */
func (trie *hTrie) StartsWith(prefix string) bool {
cur := trie
l := len(prefix)
for i := 0; i < l; i++ {
c := prefix[i]
if next, ok := cur.next[c]; ok {
cur = next
} else {
return false
}
}
return true
}
// 优先队列 所在的域
// parseQueue for Heap, Container List
type parseQueue []*parseFunction
// parseFunction 优先执行参数
type parseFunction struct {
ExecuteFunction func(u *CURL, soption string)
ParamCURL *CURL
ParamData string
Priority int
}
// Execute 执行 函数
func (pf *parseFunction) Execute() {
pf.ExecuteFunction(pf.ParamCURL, pf.ParamData)
}
// Swap 实现sort.Iterface
func (nodes *parseQueue) Swap(i, j int) {
ns := *nodes
ns[i], ns[j] = ns[j], ns[i]
}
// Less Priority Want Less
func (nodes *parseQueue) Less(i, j int) bool {
ns := *nodes
return ns[i].Priority < ns[j].Priority
}
// Push 实现heap.Interface接口定义的额外方法
func (nodes *parseQueue) Push(exec interface{}) {
*nodes = append(*nodes, exec.(*parseFunction))
}
// Pop 堆顶
func (nodes *parseQueue) Pop() (exec interface{}) {
nlen := nodes.Len()
exec = (*nodes)[nlen-1] // 返回删除的元素
*nodes = (*nodes)[:nlen-1] // [n:m]不包括下标为m的元素
return exec
}
// Len len(nodes)
func (nodes *parseQueue) Len() int {
return len(*nodes)
}
// pQueueExecute 优先函数队列
type pQueueExecute struct {
nodes parseQueue
}
// newPQueueExecute Create A pQueueExecute
func newPQueueExecute() *pQueueExecute {
pe := &pQueueExecute{}
pe.nodes = make(parseQueue, 0)
heap.Init(&pe.nodes)
return pe
}
// Push Create A pQueueExecute
func (pqe *pQueueExecute) Push(exec *parseFunction) {
heap.Push(&pqe.nodes, exec)
}
// Pop Create A pQueueExecute
func (pqe *pQueueExecute) Pop() *parseFunction {
return heap.Pop(&pqe.nodes).(*parseFunction)
}
// Len Create A pQueueExecute
func (pqe *pQueueExecute) Len() int {
return pqe.nodes.Len()
}
// func (pqe *pQueueExecute) String() string {
// content := ""
// for _, node := range pqe.nodes {
// content += strconv.Itoa(node.Prioty)
// content += " "
// }
// return content
// }
// CNode 循环链表 三色标记 不确定是否会清除循环引用, 网上说会
type CNode struct {
value interface{}
prev *CNode
next *CNode
}
// GetValue 获取到Node的值
func (node *CNode) GetValue() interface{} {
return node.value
}
// SetValue 获取到Node的值
func (node *CNode) SetValue(value interface{}) {
node.value = value
}
// CircularLinked 循环链表
type CircularLinked struct {
cursor *CNode
head *CNode
tail *CNode
size uint64
}
// NewCircularLinked create a CircularLinked
func NewCircularLinked(values ...interface{}) *CircularLinked {
list := &CircularLinked{}
if len(values) > 0 {
list.Append(values...)
}
return list
}
// Cursor get current Cursor
func (list *CircularLinked) Cursor() *CNode {
if list.cursor == nil {
list.cursor = list.head
}
return list.cursor
}
// MoveNext get next Cursor
func (list *CircularLinked) MoveNext() *CNode {
if list.cursor == nil {
list.cursor = list.head
}
list.cursor = list.cursor.next
return list.cursor
}
// MovePrev get prev Cursor
func (list *CircularLinked) MovePrev() *CNode {
if list.cursor == nil {
list.cursor = list.head
}
list.cursor = list.cursor.prev
return list.cursor
}
// CursorToHead cursor move to head
func (list *CircularLinked) CursorToHead() *CNode {
list.cursor = list.head
return list.cursor
}
// CursorToTail cursor move to tail
func (list *CircularLinked) CursorToTail() *CNode {
list.cursor = list.tail
return list.cursor
}
// GetLoopValues 获取从头到尾的值
func (list *CircularLinked) GetLoopValues() []*CNode {
var result []*CNode
if list.head != nil {
result = append(result, list.head)
for cur := list.head.next; cur != list.head; cur = cur.next {
result = append(result, cur)
}
}
return result
}
// Append a value (one or more) at the end of the list (same as Append())
func (list *CircularLinked) Append(values ...interface{}) {
for _, value := range values {
node := &CNode{value: value}
if list.size == 0 {
list.head = node
list.tail = node
node.next = node
node.prev = node
} else {
list.tail.next = node
node.next = list.head
node.prev = list.tail
list.tail = node
}
list.size++
}
}
// Remove 移除一些节点
func (list *CircularLinked) Remove(node *CNode) {
switch list.size {
case 0:
list.errorNotInList(node)
case 1:
if list.head == node {
list.head = nil
list.tail = nil
node.next = nil
node.prev = nil
list.cursor = nil
list.size--
} else {
list.errorNotInList(node)
}
case 2:
node.prev = nil
node.next = nil
switch node {
case list.head:
list.head = list.tail
list.tail.prev = list.head
list.head.next = list.tail
list.cursor = list.head
list.size--
case list.tail:
list.tail = list.head
list.tail.prev = list.head
list.head.next = list.tail
list.cursor = list.head
list.size--
default:
list.errorNotInList(node)
}
default:
switch node {
case list.head:
_, next := list.cutAndSplice(node)
list.size--
list.head = next
if list.cursor == node {
list.cursor = next
}
case list.tail:
prev, _ := list.cutAndSplice(node)
list.size--
list.tail = prev
if list.cursor == node {
list.cursor = prev
}
default:
_, next := list.cutAndSplice(node)
list.size--
if list.cursor == node {
list.cursor = next
}
}
}
}
// LookCursor for list show
func (list *CircularLinked) LookCursor() string {
cursor := list.Cursor()
content := "->["
cur := list.head
if list.size != 0 {
for size := uint64(0); size < list.size; size++ {
if cursor == cur {
content += "(" + spew.Sprint(cur.value) + ")" + ", "
} else {
content += spew.Sprint(cur.value) + ", "
}
cur = cur.next
}
}
content = strings.TrimRight(content, ", ")
showlen := len(content)
if showlen >= 64 {
showlen = 64
}
content += "]" + content[0:showlen] + " ..."
return content
}
// Clear for list show
func (list *CircularLinked) Clear() {
if list.size != 0 {
list.head.prev = nil
list.tail.next = nil
list.head = nil
list.tail = nil
list.cursor = nil
list.size = 0
}
}
// Size for list show
func (list *CircularLinked) Size() uint64 {
return list.size
}
func (list *CircularLinked) errorNotInList(node *CNode) {
log.Println("the node value ", spew.Sprint(node), " is not in list")
}
// cutAndSplice 不考虑边界情况 上层使用的是否判断
func (list *CircularLinked) cutAndSplice(node *CNode) (prev, next *CNode) {
prev = node.prev
next = node.next
prev.next = next
next.prev = prev
node.prev = nil
node.next = nil
return prev, next
}

View File

@ -1 +0,0 @@
language: go

View File

@ -1,20 +0,0 @@
The MIT License (MIT)
Copyright (c) 2016 Josh Baker
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -1,402 +0,0 @@
<p align="center">
<img
src="logo.png"
width="240" height="78" border="0" alt="GJSON">
<br>
<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
<a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
</p>
<p align="center">get json values quickly</a></p>
GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [parsing json lines](#json-lines).
Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.
Getting Started
===============
## Installing
To start using GJSON, install Go and run `go get`:
```sh
$ go get -u github.com/tidwall/gjson
```
This will retrieve the library.
## Get a value
Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". When the value is found it's returned immediately.
```go
package main
import "github.com/tidwall/gjson"
const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}`
func main() {
value := gjson.Get(json, "name.last")
println(value.String())
}
```
This will print:
```
Prichard
```
*There's also the [GetMany](#get-multiple-values-at-once) function to get multiple values at once, and [GetBytes](#working-with-bytes) for working with JSON byte slices.*
## Path Syntax
A path is a series of keys separated by a dot.
A key may contain special wildcard characters '\*' and '?'.
To access an array value use the index as the key.
To get the number of elements in an array or to access a child path, use the '#' character.
The dot and wildcard characters can be escaped with '\\'.
```json
{
"name": {"first": "Tom", "last": "Anderson"},
"age":37,
"children": ["Sara","Alex","Jack"],
"fav.movie": "Deer Hunter",
"friends": [
{"first": "Dale", "last": "Murphy", "age": 44},
{"first": "Roger", "last": "Craig", "age": 68},
{"first": "Jane", "last": "Murphy", "age": 47}
]
}
```
```
"name.last" >> "Anderson"
"age" >> 37
"children" >> ["Sara","Alex","Jack"]
"children.#" >> 3
"children.1" >> "Alex"
"child*.2" >> "Jack"
"c?ildren.0" >> "Sara"
"fav\.movie" >> "Deer Hunter"
"friends.#.first" >> ["Dale","Roger","Jane"]
"friends.1.last" >> "Craig"
```
You can also query an array for the first match by using `#[...]`, or find all matches with `#[...]#`.
Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators and the simple pattern matching `%` (like) and `!%` (not like) operators.
```
friends.#[last=="Murphy"].first >> "Dale"
friends.#[last=="Murphy"]#.first >> ["Dale","Jane"]
friends.#[age>45]#.last >> ["Craig","Murphy"]
friends.#[first%"D*"].last >> "Murphy"
friends.#[first!%"D*"].last >> "Craig"
```
## JSON Lines
There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array.
For example:
```
{"name": "Gilbert", "age": 61}
{"name": "Alexa", "age": 34}
{"name": "May", "age": 57}
{"name": "Deloise", "age": 44}
```
```
..# >> 4
..1 >> {"name": "Alexa", "age": 34}
..3 >> {"name": "Deloise", "age": 44}
..#.name >> ["Gilbert","Alexa","May","Deloise"]
..#[name="May"].age >> 57
```
The `ForEachLines` function will iterate through JSON lines.
```go
gjson.ForEachLine(json, func(line gjson.Result) bool{
println(line.String())
return true
})
```
## Result Type
GJSON supports the json types `string`, `number`, `bool`, and `null`.
Arrays and Objects are returned as their raw json types.
The `Result` type holds one of these:
```
bool, for JSON booleans
float64, for JSON numbers
string, for JSON string literals
nil, for JSON null
```
To directly access the value:
```go
result.Type // can be String, Number, True, False, Null, or JSON
result.Str // holds the string
result.Num // holds the float64 number
result.Raw // holds the raw json
result.Index // index of raw value in original json, zero means index unknown
```
There are a variety of handy functions that work on a result:
```go
result.Exists() bool
result.Value() interface{}
result.Int() int64
result.Uint() uint64
result.Float() float64
result.String() string
result.Bool() bool
result.Time() time.Time
result.Array() []gjson.Result
result.Map() map[string]gjson.Result
result.Get(path string) Result
result.ForEach(iterator func(key, value Result) bool)
result.Less(token Result, caseSensitive bool) bool
```
The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types:
The `result.Array()` function returns back an array of values.
If the result represents a non-existent value, then an empty array will be returned.
If the result is not a JSON array, the return value will be an array containing one result.
```go
boolean >> bool
number >> float64
string >> string
null >> nil
array >> []interface{}
object >> map[string]interface{}
```
### 64-bit integers
The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers.
```go
result.Int() int64 // -9223372036854775808 to 9223372036854775807
result.Uint() int64 // 0 to 18446744073709551615
```
## Get nested array values
Suppose you want all the last names from the following json:
```json
{
"programmers": [
{
"firstName": "Janet",
"lastName": "McLaughlin",
}, {
"firstName": "Elliotte",
"lastName": "Hunter",
}, {
"firstName": "Jason",
"lastName": "Harold",
}
]
}
```
You would use the path "programmers.#.lastName" like such:
```go
result := gjson.Get(json, "programmers.#.lastName")
for _, name := range result.Array() {
println(name.String())
}
```
You can also query an object inside an array:
```go
name := gjson.Get(json, `programmers.#[lastName="Hunter"].firstName`)
println(name.String()) // prints "Elliotte"
```
## Iterate through an object or array
The `ForEach` function allows for quickly iterating through an object or array.
The key and value are passed to the iterator function for objects.
Only the value is passed for arrays.
Returning `false` from an iterator will stop iteration.
```go
result := gjson.Get(json, "programmers")
result.ForEach(func(key, value gjson.Result) bool {
println(value.String())
return true // keep iterating
})
```
## Simple Parse and Get
There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result.
For example, all of these will return the same result:
```go
gjson.Parse(json).Get("name").Get("last")
gjson.Get(json, "name").Get("last")
gjson.Get(json, "name.last")
```
## Check for the existence of a value
Sometimes you just want to know if a value exists.
```go
value := gjson.Get(json, "name.last")
if !value.Exists() {
println("no last name")
} else {
println(value.String())
}
// Or as one step
if gjson.Get(json, "name.last").Exists() {
println("has a last name")
}
```
## Validate JSON
The `Get*` and `Parse*` functions expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results.
If you are consuming JSON from an unpredictable source then you may want to validate prior to using GJSON.
```go
if !gjson.Valid(json) {
return errors.New("invalid json")
}
value := gjson.Get(json, "name.last")
```
## Unmarshal to a map
To unmarshal to a `map[string]interface{}`:
```go
m, ok := gjson.Parse(json).Value().(map[string]interface{})
if !ok {
// not a map
}
```
## Working with Bytes
If your JSON is contained in a `[]byte` slice, there's the [GetBytes](https://godoc.org/github.com/tidwall/gjson#GetBytes) function. This is preferred over `Get(string(data), path)`.
```go
var json []byte = ...
result := gjson.GetBytes(json, path)
```
If you are using the `gjson.GetBytes(json, path)` function and you want to avoid converting `result.Raw` to a `[]byte`, then you can use this pattern:
```go
var json []byte = ...
result := gjson.GetBytes(json, path)
var raw []byte
if result.Index > 0 {
raw = json[result.Index:result.Index+len(result.Raw)]
} else {
raw = []byte(result.Raw)
}
```
This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`.
## Get multiple values at once
The `GetMany` function can be used to get multiple values at the same time.
```go
results := gjson.GetMany(json, "name.first", "name.last", "age")
```
The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths.
## Performance
Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/),
[ffjson](https://github.com/pquerna/ffjson),
[EasyJSON](https://github.com/mailru/easyjson),
[jsonparser](https://github.com/buger/jsonparser),
and [json-iterator](https://github.com/json-iterator/go)
```
BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op
BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op
BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op
BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op
BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op
BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op
BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op 0 allocs/op
BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op
```
JSON document used:
```json
{
"widget": {
"debug": "on",
"window": {
"title": "Sample Konfabulator Widget",
"name": "main_window",
"width": 500,
"height": 500
},
"image": {
"src": "Images/Sun.png",
"hOffset": 250,
"vOffset": 250,
"alignment": "center"
},
"text": {
"data": "Click Here",
"size": 36,
"style": "bold",
"vOffset": 100,
"alignment": "center",
"onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
}
}
}
```
Each operation was rotated though one of the following search paths:
```
widget.window.name
widget.image.hOffset
widget.text.onMouseUp
```
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
GJSON source code is available under the MIT [License](/LICENSE).

File diff suppressed because it is too large Load Diff

View File

@ -1,10 +0,0 @@
//+build appengine
package gjson
func getBytes(json []byte, path string) Result {
return Get(string(json), path)
}
func fillIndex(json string, c *parseContext) {
// noop. Use zero for the Index value.
}

View File

@ -1,73 +0,0 @@
//+build !appengine
package gjson
import (
"reflect"
"unsafe"
)
// getBytes casts the input json bytes to a string and safely returns the
// results as uniquely allocated data. This operation is intended to minimize
// copies and allocations for the large json string->[]byte.
func getBytes(json []byte, path string) Result {
var result Result
if json != nil {
// unsafe cast to string
result = Get(*(*string)(unsafe.Pointer(&json)), path)
result = fromBytesGet(result)
}
return result
}
func fromBytesGet(result Result) Result {
// safely get the string headers
rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
// create byte slice headers
rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
if strh.Data == 0 {
// str is nil
if rawh.Data == 0 {
// raw is nil
result.Raw = ""
} else {
// raw has data, safely copy the slice header to a string
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
}
result.Str = ""
} else if rawh.Data == 0 {
// raw is nil
result.Raw = ""
// str has data, safely copy the slice header to a string
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
} else if strh.Data >= rawh.Data &&
int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
// Str is a substring of Raw.
start := int(strh.Data - rawh.Data)
// safely copy the raw slice header
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
// substring the raw
result.Str = result.Raw[start : start+strh.Len]
} else {
// safely copy both the raw and str slice headers to strings
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
}
return result
}
// fillIndex finds the position of Raw data and assigns it to the Index field
// of the resulting value. If the position cannot be found then Index zero is
// used instead.
func fillIndex(json string, c *parseContext) {
if len(c.value.Raw) > 0 && !c.calcd {
jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
c.value.Index = int(rhdr.Data - jhdr.Data)
if c.value.Index < 0 || c.value.Index >= len(json) {
c.value.Index = 0
}
}
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 16 KiB

View File

@ -1,4 +0,0 @@
*.pyc
*.vscode
*.test
imitater

View File

@ -1 +0,0 @@
package imitater

View File

@ -1,214 +0,0 @@
package imitater
import (
"errors"
"io/ioutil"
"net/http"
"os"
"reflect"
"strings"
"474420502.top/eson/structure/circular_linked"
yaml "gopkg.in/yaml.v2"
)
// YamlCurls 为了自定义序列化函数
type YamlCurls []string
// UnmarshalYAML YamlCurls反序列化函数
func (curls *YamlCurls) UnmarshalYAML(unmarshal func(interface{}) error) error {
var buf interface{}
err := unmarshal(&buf)
if err != nil {
return nil
}
switch tbuf := buf.(type) {
case string:
if tbuf != "" {
for _, curlinfo := range parseCurl(tbuf) {
*curls = append(*curls, curlinfo)
}
}
case []interface{}:
for _, ifa := range tbuf {
curlstr := ifa.(string)
if curlstr != "" {
for _, curlinfo := range parseCurl(curlstr) {
*curls = append(*curls, curlinfo)
}
}
}
default:
return errors.New("read curls is error, " + reflect.TypeOf(buf).String())
}
return nil
}
// MarshalYAML YamlCurls序列化函数
func (curls *YamlCurls) MarshalYAML() (interface{}, error) {
content := "["
for _, curl := range []string(*curls) {
content += "\"" + curl + "\"" + ", "
}
content = strings.TrimRight(content, ", ")
content += "]"
return content, nil
}
// YamlProxies 为了自定义序列化函数
type YamlProxies clinked.CircularLinked
// UnmarshalYAML YamlProxies反序列化函数
func (proxies *YamlProxies) UnmarshalYAML(unmarshal func(interface{}) error) error {
var buf interface{}
err := unmarshal(&buf)
if err != nil {
return nil
}
switch tbuf := buf.(type) {
case string:
p := (*clinked.CircularLinked)(proxies)
p.Append(tbuf)
case []interface{}:
p := (*clinked.CircularLinked)(proxies)
for _, ifa := range tbuf {
p.Append(ifa.(string))
}
default:
return errors.New("read curls is error, " + reflect.TypeOf(buf).String())
}
return nil
}
// MarshalYAML YamlProxies 序列化函数
func (proxies *YamlProxies) MarshalYAML() (interface{}, error) {
content := "["
p := (*clinked.CircularLinked)(proxies)
for _, cnode := range p.GetLoopValues() {
content += "\"" + cnode.GetValue().(string) + "\"" + ", "
}
content = strings.TrimRight(content, ", ")
content += "]"
return content, nil
}
// ADInfo ad 的一些属性,基础信息等
type ADInfo struct {
Priority int `yaml:"priority"`
Device string `yaml:"device"`
Platform string `yaml:"platform"`
AreaCC string `yaml:"area_cc"`
Channel int `yaml:"channel"`
Media int `yaml:"media"`
SpiderID int `yaml:"spider_id"`
CatchAccountID int `yaml:"catch_account_id"`
}
// Config 任务加载的默认配置
type Config struct {
// Session int `yaml:"session"`
Mode int `yaml:"mode"`
Proxies *YamlProxies `yaml:"proxies"`
Retry int `yaml:"retry"`
Curls YamlCurls `yaml:"curls"`
Crontab string `yaml:"crontab"`
ITask string `yaml:"task"`
ADInfo `yaml:",inline"`
}
// newDefaultConfig create a default config
func newDefaultConfig() *Config {
conf := &Config{
// Session: 1,
Mode: 0,
Retry: 0,
Crontab: "",
ADInfo: ADInfo{
Device: "",
Platform: "",
AreaCC: "",
Channel: -1,
Media: -1,
SpiderID: -1,
CatchAccountID: -1,
},
}
return conf
}
// NewConfig 加载并返回Config
func NewConfig(p string) *Config {
f, err := os.Open(p)
defer f.Close()
if err != nil {
panic(err)
}
conf := newDefaultConfig()
err = yaml.NewDecoder(f).Decode(conf)
if err != nil {
panic(err)
}
return conf
}
func parseCurl(curl string) []string {
var result []string
switch curl[0] {
case '@':
curlfile, err := os.Open(curl[1:])
defer curlfile.Close()
if err != nil {
panic(err)
}
curldata, err := ioutil.ReadAll(curlfile)
for _, curlinfo := range strings.Split(string(curldata), "\n") {
curlstr := strings.Trim(curlinfo, "\r\n ")
if len(curlstr) >= 4 {
result = append(result, curlstr)
}
}
case '#':
resp, err := http.Get(curl[1:])
if err != nil {
panic(err)
}
curldata, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
for _, curlinfo := range strings.Split(string(curldata), "\n") {
curlstr := strings.Trim(curlinfo, "\r\n ")
if len(curlstr) >= 4 {
result = append(result, curlstr)
}
}
default:
curlstr := strings.Trim(curl, "\r\n ")
if len(curlstr) >= 4 {
result = append(result, curlstr)
}
}
return result
}

View File

@ -1,193 +0,0 @@
package imitater
import (
"fmt"
"log"
"reflect"
"time"
"474420502.top/eson/crontabex"
"474420502.top/eson/structure/circular_linked"
"474420502.top/eson/curl2info"
)
// ITask 继承这个接口的类
type ITask interface {
Execute(data map[string]interface{}) ITask
Init()
SetCrontab(cron string)
GetCrontab() *crontab.Crontab
SetName(name string)
GetName() string
SetCurl(Curl *curl2info.CURL)
GetCurl() *curl2info.CURL
GetProxies() *clinked.CircularLinked
AddProxies(proxy string)
SetADInfo(adinfo ADInfo)
GetADInfo() *ADInfo
TimeUp() bool
NextTime() time.Time
}
var register = make(map[string]reflect.Type)
func init() {
log.SetFlags(log.Llongfile)
}
// Register 注册 类型 ITask为样本的类型
func Register(name string, itask ITask) {
register[name] = reflect.TypeOf(itask).Elem()
}
func makeRegisterType(name string) ITask {
result := reflect.New(register[name]).Interface().(ITask)
result.SetName(name)
return result
}
// Person 以人为单位
type Person struct {
Tasks []ITask
Conf *Config
}
// NewPerson 创建一个人实例
func NewPerson() *Person {
person := &Person{}
// person.Conf = NewConfig(conf)
// person.Tasks = SplitTasks(person.Conf)
return person
}
// Config 加载配置
func (person *Person) Config(conf string) {
person.Conf = NewConfig(conf)
person.Tasks = splitTasks(person.Conf)
}
// NewPersonWithConfig 创建一个person
func NewPersonWithConfig(conf string) *Person {
person := NewPerson()
person.Config(conf)
return person
}
// SplitTasks 拆开出需求的任务
func splitTasks(conf *Config) []ITask {
var tasks []ITask
proxies := (*clinked.CircularLinked)(conf.Proxies)
for _, scurl := range conf.Curls {
curl := curl2info.ParseRawCURL(scurl)
if curl.ITask == "" {
curl.ITask = conf.ITask
}
task := makeRegisterType(curl.ITask)
switch conf.Mode {
case 0:
initTask(conf, task, curl)
// 初始化代理
if proxies != nil {
for _, cnode := range proxies.GetLoopValues() {
proxy := cnode.GetValue().(string)
task.AddProxies(proxy)
}
}
tasks = append(tasks, task)
case 1:
if proxies != nil {
for _, cnode := range proxies.GetLoopValues() {
proxy := cnode.GetValue().(string)
ncurl := curl2info.ParseRawCURL(scurl)
if curl.ITask == "" {
curl.ITask = conf.ITask
}
ptask := makeRegisterType(ncurl.ITask).(ITask)
initTask(conf, ptask, ncurl)
ptask.AddProxies(proxy)
tasks = append(tasks, ptask)
}
}
}
}
return tasks
}
// Execute 人的执行所有任务
func (person *Person) Execute() {
taskLen := len(person.Tasks)
result := make(chan string, 1)
for _, task := range person.Tasks {
go ExecuteOnPlan(task, result)
}
for t := range result {
log.Println(t)
taskLen--
if taskLen <= 0 {
close(result)
}
}
// engine := gin.Default()
// engine.Run()
}
// initTask 生成一个新任务
func initTask(conf *Config, task ITask, curl *curl2info.CURL) {
if curl.Crontab != "" {
task.SetCrontab(curl.Crontab)
} else {
task.SetCrontab(conf.Crontab)
}
task.SetADInfo(conf.ADInfo)
task.SetCurl(curl)
task.Init()
}
// ExecuteOnPlan 按照计划执行任务并返回结果
func ExecuteOnPlan(task ITask, result chan string) {
data := make(map[string]interface{})
var rtask ITask
taskname := task.GetName()
urlname := task.GetCurl().Name
task.NextTime() // 必须调用NextTime 才能触发TimeUp
for {
interval := time.Second * 2
if task.TimeUp() {
rtask = task.Execute(data) // 事件 在这里变化
ntime := task.NextTime()
interval = ntime.Sub(time.Now())
if rtask == nil {
// log.Println("rtask is nil")
result <- fmt.Sprintf("rtask is nil, the first task = %s, name = %s", taskname, urlname)
return
}
task = rtask
}
time.Sleep(interval)
}
}

View File

@ -1,157 +0,0 @@
package imitater
import (
"encoding/json"
"log"
"time"
"474420502.top/test/logdb"
"474420502.top/eson/structure/circular_linked"
uuid "github.com/satori/go.uuid"
"474420502.top/eson/crontabex"
"474420502.top/eson/curl2info"
"474420502.top/eson/requests"
)
// Task 任务
type Task struct {
ITask
name string
crontab *crontab.Crontab
curl *curl2info.CURL
workflow *requests.Workflow
session *requests.Session
proxies clinked.CircularLinked
adinfo ADInfo
}
// Init 初始化, 会被Persion默认调用, 可以用来覆盖
func (task *Task) Init() {
}
// SetName 任务的名字
func (task *Task) SetName(name string) {
task.name = name
}
// GetName 获取任务的名字
func (task *Task) GetName() string {
return task.name
}
// SetCurl 设置任务的curl信息类
func (task *Task) SetCurl(curl *curl2info.CURL) {
task.curl = curl
}
// GetCurl 获取任务的curl信息类
func (task *Task) GetCurl() *curl2info.CURL {
return task.curl
}
// AddProxies 添加代理集合
func (task *Task) AddProxies(proxy string) {
task.proxies.Append(proxy)
}
// GetProxies 获取代理的字符串
func (task *Task) GetProxies() *clinked.CircularLinked {
return &task.proxies
}
// SetADInfo 设置广告的信息结构
func (task *Task) SetADInfo(adinfo ADInfo) {
task.adinfo = adinfo
}
// GetADInfo 获取广告的信息结构
func (task *Task) GetADInfo() *ADInfo {
return &task.adinfo
}
// SetCrontab 设置crontab的控制规则字符串
func (task *Task) SetCrontab(cron string) {
task.crontab = crontab.NewCrontab(cron)
}
// GetCrontab 设置上次执行的状态 成功true 失败false
func (task *Task) GetCrontab() *crontab.Crontab {
return task.crontab
}
// TimeUp 判断是否到了下次执行的时间点
func (task *Task) TimeUp() bool {
return task.crontab.TimeUp()
}
// NextTime 下次执行的时间点
func (task *Task) NextTime() time.Time {
return task.crontab.NextTime()
}
// Workflow 根据persistent 是否返回现有session(or 新建 session),创建Workflow的信息, 便于设置或者更改参数, Session持久化
func (task *Task) Workflow(persistent bool) *requests.Workflow {
if persistent {
task.workflow = task.curl.CreateWorkflow(task.Session())
return task.workflow
}
proxies := task.GetProxies()
ses := task.curl.CreateSession()
if proxies.Size() > 0 {
proxy := proxies.Cursor().GetValue().(string)
ses.SetConfig(requests.CProxy, proxy)
proxies.MoveNext()
}
return task.curl.CreateWorkflow(ses)
}
// Request 根据curl信息执行,没持久化
func (task *Task) Request() (*requests.Response, error) {
return task.Workflow(false).Execute()
}
// Session 获取Session的信息 只保留session的数据和url参数.
func (task *Task) Session() *requests.Session {
if task.session == nil {
task.session = task.curl.CreateSession()
}
if task.proxies.Size() > 0 {
proxy := task.proxies.Cursor().GetValue().(string)
task.session.SetConfig(requests.CProxy, proxy)
task.proxies.MoveNext()
}
return task.session
}
// ADDataSave 公众存储数据的接口
func ADDataSave(task ITask, db *logdb.LogDB, adDataList []string) bool {
if len(adDataList) != 0 {
adDataStore, err := json.Marshal(adDataList)
if err != nil {
panic(err)
}
// log.Println(string(adDataStore))
uid, err := uuid.NewV4()
if err != nil {
panic(err)
}
log.Println(uid.String())
adinfo := task.GetADInfo()
db.ADInsert(uid.String(), adinfo.Device, adinfo.Platform, adinfo.AreaCC, "",
string(adDataStore), adinfo.SpiderID, adinfo.Channel, adinfo.Media, adinfo.CatchAccountID, 0, adinfo.Priority, time.Now())
return true
}
return false
}

View File

@ -1,2 +0,0 @@
curl --name "cip.cc" 'http://cip.cc'
curl --name "ag-cn" 'https://appgrowing.cn/' -H 'authority: appgrowing.cn' -H 'cache-control: max-age=0' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: zh' -H 'cookie: _ga=GA1.2.1371058419.1533104518; _gid=GA1.2.1726457685.1544092110; _gat_gtag_UA_4002880_19=1' -H 'if-none-match: W/"5c088f5a-ca6"' -H 'if-modified-since: Thu, 06 Dec 2018 02:54:18 GMT' --compressed

View File

@ -1,21 +0,0 @@
mode : 0
# proxies : "socks5://10.10.10.1:8080" // 支持, 列表 与 单项字符串
proxies : []
retry : 0
timeout: 12
priority : 10000
# curls: "@test.curl" // 支持, 列表 与 单项字符串
curls : "@test2.curl"
task: "toutiao"
# next_do : "doothers"
device : "eson-OnePlus"
platform : "Android"
area_cc : 4401
channel : 105
media : 55
spider_id : 73
catch_account_id : 123
crontab: "2s"

View File

@ -1 +0,0 @@
curl --name "tt-information" 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' --task toutiao -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'

View File

@ -1,2 +0,0 @@
*.pyc
*.vscode

View File

@ -1,53 +0,0 @@
package requests
import (
"bytes"
"errors"
"net/http"
"reflect"
)
func buildBodyRequest(wf *Workflow) *http.Request {
var req *http.Request
var err error
contentType := ""
if wf.Body.GetIOBody() == nil {
req, err = http.NewRequest(wf.Method, wf.GetRawURL(), nil)
} else {
var bodybuf *bytes.Buffer
switch wf.Body.GetIOBody().(type) {
case []byte:
bodybuf = bytes.NewBuffer(wf.Body.GetIOBody().([]byte))
case string:
bodybuf = bytes.NewBuffer([]byte(wf.Body.GetIOBody().(string)))
case *bytes.Buffer:
bodybuf = bytes.NewBuffer(wf.Body.GetIOBody().(*bytes.Buffer).Bytes())
default:
panic(errors.New("the type is not exist, type is " + reflect.TypeOf(wf.Body.GetIOBody()).String()))
}
req, err = http.NewRequest(wf.Method, wf.GetRawURL(), bodybuf)
}
if err != nil {
panic(err)
}
if wf.Body.ContentType() != "" {
contentType = wf.Body.ContentType()
} else {
contentType = ""
if contentType == "" {
if wf.Method == "POST" || wf.Method == "PUT" || wf.Method == "PATCH" {
contentType = TypeURLENCODED
}
}
}
if contentType != "" {
req.Header.Set(HeaderKeyContentType, contentType)
}
return req
}

View File

@ -1,98 +0,0 @@
package requests
import (
"bytes"
"io"
"log"
"mime/multipart"
"net/url"
"strconv"
)
func writeFormUploadFile(mwriter *multipart.Writer, ufile *UploadFile) {
part, err := mwriter.CreateFormFile(ufile.FieldName, ufile.FileName)
if err != nil {
log.Panic(err)
}
io.Copy(part, ufile.FileReaderCloser)
err = ufile.FileReaderCloser.Close()
if err != nil {
panic(err)
}
}
func createMultipart(postParams IBody, params []interface{}) {
plen := len(params)
body := &bytes.Buffer{}
mwriter := multipart.NewWriter(body)
for _, iparam := range params[0 : plen-1] {
switch param := iparam.(type) {
case *UploadFile:
if param.FieldName == "" {
param.FieldName = "file0"
}
writeFormUploadFile(mwriter, param)
case []*UploadFile:
for i, p := range param {
if p.FieldName == "" {
p.FieldName = "file" + strconv.Itoa(i)
}
writeFormUploadFile(mwriter, p)
}
case string:
uploadFiles, err := UploadFileFromGlob(param)
if err != nil {
log.Println(err)
} else {
for i, p := range uploadFiles {
if p.FieldName == "" {
p.FieldName = "file" + strconv.Itoa(i)
}
writeFormUploadFile(mwriter, p)
}
}
case []string:
for i, glob := range param {
uploadFiles, err := UploadFileFromGlob(glob)
if err != nil {
log.Println(err)
} else {
for ii, p := range uploadFiles {
if p.FieldName == "" {
p.FieldName = "file" + strconv.Itoa(ii) + "_" + strconv.Itoa(i)
}
writeFormUploadFile(mwriter, p)
}
}
}
case map[string]string:
for k, v := range param {
mwriter.WriteField(k, v)
}
case map[string][]string:
for k, vs := range param {
for _, v := range vs {
mwriter.WriteField(k, v)
}
}
case url.Values:
for k, vs := range param {
for _, v := range vs {
mwriter.WriteField(k, v)
}
}
}
}
postParams.AddContentType(mwriter.FormDataContentType())
postParams.SetIOBody(body)
err := mwriter.Close()
if err != nil {
panic(err)
}
}

View File

@ -1,54 +0,0 @@
package requests
import (
"bytes"
"compress/gzip"
"compress/zlib"
"io/ioutil"
"net/http"
)
// Response 响应内容包含http.Response
type Response struct {
DContent string
GResponse *http.Response
}
// FromHTTPResponse 生成Response 从标准http.Response
func FromHTTPResponse(resp *http.Response) (*Response, error) {
// 复制response 返回内容 并且测试是否有解压的需求
srcbuf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
resp.Body.Close()
content := ""
srcReader := bytes.NewReader(srcbuf)
if r, err := gzip.NewReader(srcReader); err == nil {
defer r.Close()
buf, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
content = string(buf)
} else if r, err := zlib.NewReader(srcReader); err == nil {
defer r.Close()
buf, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
content = string(buf)
} else {
content = string(srcbuf)
}
return &Response{DContent: content, GResponse: resp}, nil
}
// Content 返回解压后的内容
func (gresp *Response) Content() string {
return gresp.DContent
}

View File

@ -1,379 +0,0 @@
package requests
import (
"crypto/tls"
"errors"
"net/http"
"net/http/cookiejar"
"net/url"
"reflect"
"runtime"
"strings"
"time"
"golang.org/x/net/publicsuffix"
)
// Body 相关参数结构
type Body struct {
// Query map[string][]string
ioBody interface{}
// prefix ContentType 前缀
prefix string
// Files []UploadFile
contentTypes map[string]int
}
// NewBody new body pointer
func NewBody() *Body {
b := &Body{}
b.contentTypes = make(map[string]int)
return b
}
// SetIOBody 设置IOBody的值
func (body *Body) SetIOBody(iobody interface{}) {
body.ioBody = iobody
}
// GetIOBody 获取ioBody值
func (body *Body) GetIOBody() interface{} {
return body.ioBody
}
// ContentType 获取ContentType
func (body *Body) ContentType() string {
content := body.prefix
for kvalue := range body.contentTypes {
content += kvalue + ";"
}
return strings.TrimRight(content, ";")
}
// SetPrefix SetPrefix 和 AddContentType的顺序会影响到ContentType()的返回结果
func (body *Body) SetPrefix(ct string) {
body.prefix = strings.TrimRight(ct, ";") + ";"
}
// AddContentType 添加 Add Type类型
func (body *Body) AddContentType(ct string) {
for _, v := range strings.Split(ct, ";") {
v = strings.Trim(v, " ")
if v != "" {
if body.prefix != v {
body.contentTypes[v] = 1
}
}
}
}
// IBody 相关参数结构
type IBody interface {
// GetIOBody 获取iobody data
GetIOBody() interface{}
// SetIOBody 设置iobody data
SetIOBody(iobody interface{})
// ContentType 返回包括 Prefix 所有的ContentType
ContentType() string
// AppendContent
AddContentType(ct string)
// SetPrefix 设置 Prefix; 唯一前缀
SetPrefix(ct string)
}
// BasicAuth 帐号认真结构
type BasicAuth struct {
// User 帐号
User string
// Password 密码
Password string
}
// Session 的基本方法
type Session struct {
auth *BasicAuth
body IBody
client *http.Client
cookiejar http.CookieJar
transport *http.Transport
Header http.Header
Query url.Values
}
const (
// TypeJSON 类型
TypeJSON = "application/json"
// TypeXML 类型
TypeXML = "text/xml"
// TypePlain 类型
TypePlain = "text/plain"
// TypeHTML 类型
TypeHTML = "text/html"
// TypeURLENCODED 类型
TypeURLENCODED = "application/x-www-form-urlencoded"
// TypeForm PostForm类型
TypeForm = TypeURLENCODED
// TypeStream application/octet-stream 只能提交一个二进制流, 很少用
TypeStream = "application/octet-stream"
// TypeFormData 类型
TypeFormData = "multipart/form-data"
// TypeMixed Mixed类型
TypeMixed = "multipart/mixed"
// HeaderKeyHost Host
HeaderKeyHost = "Host"
// HeaderKeyUA User-Agent
HeaderKeyUA = "User-Agent"
// HeaderKeyContentType Content-Type
HeaderKeyContentType = "Content-Type"
)
// TypeConfig 配置类型
type TypeConfig int
const (
_ TypeConfig = iota
// CRequestTimeout request 包括 dial request redirect 总时间超时
CRequestTimeout // 支持time.Duration 和 int(秒为单位)
// CDialTimeout 一个Connect过程的Timeout
CDialTimeout // 支持time.Duration 和 int(秒为单位)
// CKeepAlives 默认不KeepAlives, 容易被一直KeepAlives 没关闭链接
CKeepAlives
// CProxy 代理链接
CProxy // http, https, socks5
// CInsecure InsecureSkipVerify
CInsecure // true, false
// CBasicAuth 帐号认证
CBasicAuth // user pwd
// CTLS 帐号认证
CTLS // user pwd
// CCookiejar 持久化 CookieJar
CCookiejar // true or false ; default = true
)
// NewSession 创建Session
func NewSession() *Session {
client := &http.Client{}
transport := &http.Transport{DisableCompression: true, DisableKeepAlives: true}
EnsureTransporterFinalized(transport)
client.Transport = transport
cjar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
if err != nil {
panic(err)
}
client.Jar = cjar
return &Session{client: client, body: NewBody(), transport: transport, auth: nil, cookiejar: client.Jar, Header: make(http.Header)}
}
// SetConfig 设置配置
func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) {
switch typeConfig {
case CRequestTimeout:
switch v := values.(type) {
case time.Duration:
ses.client.Timeout = v
case int:
ses.client.Timeout = time.Duration(v * int(time.Second))
case int64:
ses.client.Timeout = time.Duration(v * int64(time.Second))
case float32:
ses.client.Timeout = time.Duration(v * float32(time.Second))
case float64:
ses.client.Timeout = time.Duration(v * float64(time.Second))
default:
panic(errors.New("error type " + reflect.TypeOf(v).String()))
}
case CDialTimeout:
// 没时间实现这些小细节
case CKeepAlives:
ses.transport.DisableKeepAlives = !values.(bool)
case CCookiejar:
v := values.(bool)
if v {
if ses.client.Jar == nil {
ses.client.Jar = ses.cookiejar
}
} else {
ses.client.Jar = nil
}
case CProxy:
switch v := values.(type) {
case string:
purl, err := (url.Parse(v))
if err != nil {
panic(err)
}
ses.transport.Proxy = http.ProxyURL(purl)
case *url.URL:
ses.transport.Proxy = http.ProxyURL(v)
case nil:
ses.transport.Proxy = nil
}
case CInsecure:
ses.transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: values.(bool)}
case CTLS:
ses.transport.TLSClientConfig = values.(*tls.Config)
case CBasicAuth:
if ses.auth == nil {
ses.auth = &BasicAuth{}
}
switch v := values.(type) {
case *BasicAuth:
ses.auth.User = v.User
ses.auth.User = v.Password
case BasicAuth:
ses.auth.User = v.User
ses.auth.User = v.Password
case []string:
ses.auth.User = v[0]
ses.auth.User = v[1]
case nil:
ses.auth = nil
}
default:
panic(errors.New("unknown typeConfig " + reflect.TypeOf(typeConfig).String()))
}
return
}
// SetQuery 设置url query的持久参数的值
func (ses *Session) SetQuery(values url.Values) {
ses.Query = values
}
// GetQuery 获取get query的值
func (ses *Session) GetQuery() url.Values {
return ses.Query
}
// SetHeader 设置set Header的值
func (ses *Session) SetHeader(header http.Header) {
ses.Header = header
}
// GetHeader 获取get Header的值
func (ses *Session) GetHeader() http.Header {
return ses.Header
}
// SetCookies 设置Cookies 或者添加Cookies Del
func (ses *Session) SetCookies(u *url.URL, cookies []*http.Cookie) {
ses.cookiejar.SetCookies(u, cookies)
}
// Cookies 返回 Cookies
func (ses *Session) Cookies(u *url.URL) []*http.Cookie {
return ses.cookiejar.Cookies(u)
}
// DelCookies 删除 Cookies
func (ses *Session) DelCookies(u *url.URL, name string) {
cookies := ses.cookiejar.Cookies(u)
for _, c := range cookies {
if c.Name == name {
c.MaxAge = -1
}
}
ses.SetCookies(u, cookies)
}
// ClearCookies 清楚所有cookiejar上的cookies
func (ses *Session) ClearCookies() {
cjar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
if err != nil {
panic(err)
}
ses.cookiejar = cjar
ses.client.Jar = ses.cookiejar
}
// Head 请求
func (ses *Session) Head(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "HEAD"
return wf
}
// Get 请求
func (ses *Session) Get(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "GET"
return wf
}
// Post 请求
func (ses *Session) Post(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "POST"
return wf
}
// Put 请求
func (ses *Session) Put(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "PUT"
return wf
}
// Patch 请求
func (ses *Session) Patch(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "PATCH"
return wf
}
// Options 请求
func (ses *Session) Options(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "OPTIONS"
return wf
}
// Delete 请求
func (ses *Session) Delete(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "DELETE"
return wf
}
// // CloseIdleConnections closes the idle connections that a session client may make use of
// // 从levigross/grequests 借鉴
// func (ses *Session) CloseIdleConnections() {
// ses.client.Transport.(*http.Transport).CloseIdleConnections()
// }
// EnsureTransporterFinalized will ensure that when the HTTP client is GCed
// the runtime will close the idle connections (so that they won't leak)
// this function was adopted from Hashicorp's go-cleanhttp package
func EnsureTransporterFinalized(httpTransport *http.Transport) {
runtime.SetFinalizer(&httpTransport, func(transportInt **http.Transport) {
(*transportInt).CloseIdleConnections()
})
}

View File

@ -1,56 +0,0 @@
package requests
import (
"io"
"log"
"os"
"path/filepath"
)
// UploadFile 上传文件的结构
type UploadFile struct {
FileName string
FieldName string
FileReaderCloser io.ReadCloser
}
// UploadFileFromPath 从本地文件获取上传文件
func UploadFileFromPath(fileName string) (*UploadFile, error) {
fd, err := os.Open(fileName)
if err != nil {
return nil, err
}
return &UploadFile{FileReaderCloser: fd, FileName: fileName}, nil
}
// UploadFileFromGlob 根据Glob从本地文件获取上传文件
func UploadFileFromGlob(glob string) ([]*UploadFile, error) {
files, err := filepath.Glob(glob)
if err != nil {
return nil, err
}
if len(files) == 0 {
log.Println("UploadFileFromGlob: len(files) == 0")
}
var ufiles []*UploadFile
for _, f := range files {
if s, err := os.Stat(f); err != nil || s.IsDir() {
continue
}
fd, err := os.Open(f)
if err != nil {
log.Println(fd.Name(), err)
} else {
ufiles = append(ufiles, &UploadFile{FileReaderCloser: fd, FileName: filepath.Base(fd.Name())})
}
}
return ufiles, nil
}

View File

@ -1,316 +0,0 @@
package requests
import (
"net/http"
"net/url"
"regexp"
"strings"
)
// Workflow 工作流
type Workflow struct {
session *Session
ParsedURL *url.URL
Method string
Body IBody
Header http.Header
Cookies map[string]*http.Cookie
}
// NewWorkflow new and init workflow
func NewWorkflow(ses *Session, u string) *Workflow {
wf := &Workflow{}
wf.SwitchSession(ses)
wf.SetRawURL(u)
wf.Body = NewBody()
wf.Header = make(http.Header)
wf.Cookies = make(map[string]*http.Cookie)
return wf
}
// SwitchSession 替换Session
func (wf *Workflow) SwitchSession(ses *Session) {
wf.session = ses
}
// AddHeader 添加头信息 Get方法从Header参数上获取
func (wf *Workflow) AddHeader(key, value string) *Workflow {
wf.Header.Add(key, value)
return wf
}
// SetHeader 设置完全替换原有Header
func (wf *Workflow) SetHeader(header http.Header) *Workflow {
wf.Header = make(http.Header)
for k, HValues := range header {
var newHValues []string
for _, value := range HValues {
newHValues = append(newHValues, value)
}
wf.Header[k] = newHValues
}
return wf
}
// GetHeader 获取Workflow Header
func (wf *Workflow) GetHeader() http.Header {
return wf.Header
}
// GetCombineHeader 获取后的Header信息
func (wf *Workflow) GetCombineHeader() http.Header {
return mergeMapList(wf.session.Header, wf.Header)
}
// DelHeader 添加头信息 Get方法从Header参数上获取
func (wf *Workflow) DelHeader(key string) *Workflow {
wf.Header.Del(key)
return wf
}
// AddCookie 添加Cookie
func (wf *Workflow) AddCookie(c *http.Cookie) *Workflow {
wf.Cookies[c.Name] = c
return wf
}
// AddCookies 添加[]*http.Cookie
func (wf *Workflow) AddCookies(cookies []*http.Cookie) *Workflow {
for _, c := range cookies {
wf.AddCookie(c)
}
return wf
}
// AddKVCookie 添加 以 key value 的 Cookie
func (wf *Workflow) AddKVCookie(name, value string) *Workflow {
wf.Cookies[name] = &http.Cookie{Name: name, Value: value}
return wf
}
// DelCookie 删除Cookie
func (wf *Workflow) DelCookie(name interface{}) *Workflow {
switch n := name.(type) {
case string:
if _, ok := wf.Cookies[n]; ok {
delete(wf.Cookies, n)
return wf
}
case *http.Cookie:
if _, ok := wf.Cookies[n.Name]; ok {
delete(wf.Cookies, n.Name)
return wf
}
default:
panic("name type is not support")
}
return nil
}
// GetParsedURL 获取url的string形式
func (wf *Workflow) GetParsedURL() *url.URL {
return wf.ParsedURL
}
// SetParsedURL 获取url的string形式
func (wf *Workflow) SetParsedURL(u *url.URL) *Workflow {
wf.ParsedURL = u
return wf
}
// GetRawURL 获取url的string形式
func (wf *Workflow) GetRawURL() string {
u := strings.Split(wf.ParsedURL.String(), "?")[0] + "?" + wf.GetCombineQuery().Encode()
return u
}
// SetRawURL 设置 url
func (wf *Workflow) SetRawURL(srcURL string) *Workflow {
purl, err := url.ParseRequestURI(srcURL)
if err != nil {
panic(err)
}
wf.ParsedURL = purl
return wf
}
// GetQuery 获取Query参数
func (wf *Workflow) GetQuery() url.Values {
return wf.ParsedURL.Query()
}
// GetCombineQuery 获取Query参数
func (wf *Workflow) GetCombineQuery() url.Values {
if wf.ParsedURL != nil {
vs := wf.ParsedURL.Query()
return mergeMapList(wf.session.GetQuery(), vs)
}
return nil
}
// SetQuery 设置Query参数
func (wf *Workflow) SetQuery(query url.Values) *Workflow {
if query == nil {
return wf
}
query = (url.Values)(mergeMapList(wf.session.Query, query))
wf.ParsedURL.RawQuery = query.Encode()
return wf
}
var regexGetPath = regexp.MustCompile("/[^/]*")
// GetURLPath 获取Path参数
func (wf *Workflow) GetURLPath() []string {
return regexGetPath.FindAllString(wf.ParsedURL.Path, -1)
}
// GetURLRawPath 获取未分解Path参数
func (wf *Workflow) GetURLRawPath() string {
return wf.ParsedURL.Path
}
// encodePath path格式每个item都必须以/开头
func encodePath(path []string) string {
rawpath := ""
for _, p := range path {
if p[0] != '/' {
p = "/" + p
}
rawpath += p
}
return rawpath
}
// SetURLPath 设置Path参数
func (wf *Workflow) SetURLPath(path []string) *Workflow {
if path == nil {
return wf
}
wf.ParsedURL.Path = encodePath(path)
return wf
}
// SetURLRawPath 设置Pa晚上参数
func (wf *Workflow) SetURLRawPath(path string) *Workflow {
wf.ParsedURL.Path = path
return wf
}
// SetBody 参数设置
func (wf *Workflow) SetBody(body IBody) *Workflow {
wf.Body = body
return wf
}
// GetBody 参数设置
func (wf *Workflow) GetBody(body IBody) IBody {
return wf.Body
}
// AutoSetBody 参数设置
func (wf *Workflow) AutoSetBody(params ...interface{}) *Workflow {
if params != nil {
plen := len(params)
defaultContentType := TypeURLENCODED
if plen >= 2 {
t := params[plen-1]
defaultContentType = t.(string)
}
wf.Body.SetPrefix(defaultContentType)
if defaultContentType == TypeFormData {
createMultipart(wf.Body, params)
} else {
var values url.Values
switch param := params[0].(type) {
case map[string]string:
values := make(url.Values)
for k, v := range param {
values.Set(k, v)
}
wf.Body.SetIOBody([]byte(values.Encode()))
case map[string][]string:
values = param
wf.Body.SetIOBody([]byte(values.Encode()))
case string:
wf.Body.SetIOBody([]byte(param))
case []byte:
wf.Body.SetIOBody(param)
}
}
}
return wf
}
func mergeMapList(headers ...map[string][]string) map[string][]string {
set := make(map[string]map[string]int)
merged := make(map[string][]string)
for _, header := range headers {
for key, values := range header {
for _, v := range values {
if vs, ok := set[key]; ok {
vs[v] = 1
} else {
set[key] = make(map[string]int)
set[key][v] = 1
}
}
}
}
for key, mvalue := range set {
for v := range mvalue {
// merged.Add(key, v)
if mergeValue, ok := merged[key]; ok {
merged[key] = append(mergeValue, v)
} else {
merged[key] = []string{v}
}
}
}
return merged
}
// setHeaderRequest 设置request的头
func setHeaderRequest(req *http.Request, wf *Workflow) {
req.Header = mergeMapList(req.Header, wf.session.Header, wf.Header)
}
// setHeaderRequest 设置request的临时Cookie, 永久需要在session上设置cookie
func setTempCookieRequest(req *http.Request, wf *Workflow) {
if wf.Cookies != nil {
for _, c := range wf.Cookies {
req.AddCookie(c)
}
}
}
// Execute 执行
func (wf *Workflow) Execute() (*Response, error) {
req := buildBodyRequest(wf)
setHeaderRequest(req, wf)
setTempCookieRequest(req, wf)
if wf.session.auth != nil {
req.SetBasicAuth(wf.session.auth.User, wf.session.auth.Password)
}
resp, err := wf.session.client.Do(req)
if err != nil {
return nil, err
}
return FromHTTPResponse(resp)
}

View File

@ -1,249 +0,0 @@
package clinked
import (
"log"
"strings"
"github.com/davecgh/go-spew/spew"
)
// CNode 循环链表 三色标记 不确定是否会清除循环引用, 网上说会
type CNode struct {
value interface{}
prev *CNode
next *CNode
}
// GetValue 获取到Node的值
func (node *CNode) GetValue() interface{} {
return node.value
}
// SetValue 获取到Node的值
func (node *CNode) SetValue(value interface{}) {
node.value = value
}
// CircularLinked 循环链表
type CircularLinked struct {
cursor *CNode
head *CNode
tail *CNode
size uint64
}
// NewCircularLinked create a CircularLinked
func NewCircularLinked(values ...interface{}) *CircularLinked {
list := &CircularLinked{}
if len(values) > 0 {
list.Append(values...)
}
return list
}
// Head 链表的首
func (list *CircularLinked) Head() *CNode {
return list.head
}
// Tail 链表的首
func (list *CircularLinked) Tail() *CNode {
return list.tail
}
// Cursor get current Cursor
func (list *CircularLinked) Cursor() *CNode {
if list.cursor == nil {
list.cursor = list.head
}
return list.cursor
}
// MoveNext get next Cursor
func (list *CircularLinked) MoveNext() *CNode {
if list.cursor == nil {
list.cursor = list.head
}
list.cursor = list.cursor.next
return list.cursor
}
// MovePrev get prev Cursor
func (list *CircularLinked) MovePrev() *CNode {
if list.cursor == nil {
list.cursor = list.head
}
list.cursor = list.cursor.prev
return list.cursor
}
// CursorToHead cursor move to head
func (list *CircularLinked) CursorToHead() *CNode {
list.cursor = list.head
return list.cursor
}
// CursorToTail cursor move to tail
func (list *CircularLinked) CursorToTail() *CNode {
list.cursor = list.tail
return list.cursor
}
// GetLoopValues 获取从头到尾的值
func (list *CircularLinked) GetLoopValues() []*CNode {
var result []*CNode
if list.head != nil {
result = append(result, list.head)
for cur := list.head.next; cur != list.head; cur = cur.next {
result = append(result, cur)
}
}
return result
}
// Append a value (one or more) at the end of the list (same as Append())
func (list *CircularLinked) Append(values ...interface{}) {
for _, value := range values {
node := &CNode{value: value}
if list.size == 0 {
list.head = node
list.tail = node
node.next = node
node.prev = node
} else {
list.tail.next = node
node.next = list.head
node.prev = list.tail
list.tail = node
}
list.size++
}
}
// Remove 移除一些节点
func (list *CircularLinked) Remove(node *CNode) {
switch list.size {
case 0:
list.errorNotInList(node)
case 1:
if list.head == node {
list.head = nil
list.tail = nil
node.next = nil
node.prev = nil
list.cursor = nil
list.size--
} else {
list.errorNotInList(node)
}
case 2:
node.prev = nil
node.next = nil
switch node {
case list.head:
list.head = list.tail
list.tail.prev = list.head
list.head.next = list.tail
list.cursor = list.head
list.size--
case list.tail:
list.tail = list.head
list.tail.prev = list.head
list.head.next = list.tail
list.cursor = list.head
list.size--
default:
list.errorNotInList(node)
}
default:
switch node {
case list.head:
_, next := list.cutAndSplice(node)
list.size--
list.head = next
if list.cursor == node {
list.cursor = next
}
case list.tail:
prev, _ := list.cutAndSplice(node)
list.size--
list.tail = prev
if list.cursor == node {
list.cursor = prev
}
default:
_, next := list.cutAndSplice(node)
list.size--
if list.cursor == node {
list.cursor = next
}
}
}
}
// LookCursor for list show
func (list *CircularLinked) LookCursor() string {
cursor := list.Cursor()
content := "->["
cur := list.head
if list.size != 0 {
for size := uint64(0); size < list.size; size++ {
if cursor == cur {
content += "(" + spew.Sprint(cur.value) + ")" + ", "
} else {
content += spew.Sprint(cur.value) + ", "
}
cur = cur.next
}
}
content = strings.TrimRight(content, ", ")
showlen := len(content)
if showlen >= 64 {
showlen = 64
}
content += "]" + content[0:showlen] + " ..."
return content
}
// Clear for list show
func (list *CircularLinked) Clear() {
if list.size != 0 {
list.head.prev = nil
list.tail.next = nil
list.head = nil
list.tail = nil
list.cursor = nil
list.size = 0
}
}
// Size for list show
func (list *CircularLinked) Size() uint64 {
return list.size
}
func (list *CircularLinked) errorNotInList(node *CNode) {
log.Println("the node value ", spew.Sprint(node), " is not in list")
}
// cutAndSplice (不考虑边界情况)
func (list *CircularLinked) cutAndSplice(node *CNode) (prev, next *CNode) {
prev = node.prev
next = node.next
prev.next = next
next.prev = prev
node.prev = nil
node.next = nil
return prev, next
}

View File

@ -1,18 +0,0 @@
package plist
// NodeInt plist 现成的Int节点, 可以作为例子
type NodeInt struct {
Node
}
// Compare plist 现成的Int节点Compare覆盖
func (fl *NodeInt) Compare(v INode) bool {
return fl.GetValue().(int) > v.GetValue().(int)
}
// NewNodeInt plist 现成的Int节点, New一个NodeInt
func NewNodeInt(v int) (fl *NodeInt) {
fl = new(NodeInt)
fl.SetValue(v)
return fl
}

View File

@ -1,270 +0,0 @@
package plist
import (
"github.com/davecgh/go-spew/spew"
)
// INode 比较的必须继承的接口
type INode interface {
Compare(v INode) bool
// GetValue 获取值
GetValue() interface{}
// SetValue 设置值
SetValue(v interface{})
GetPrev() INode
SetPrev(INode)
GetNext() INode
SetNext(INode)
}
// Node 优先链表的节点
type Node struct {
INode
prev, next INode
// isrelease bool
value interface{}
}
// NewNode 创建一个PriorityList 节点
func NewNode(v interface{}) *Node {
n := new(Node)
n.SetValue(v)
return n
}
// GetValue 获取值
func (node *Node) GetValue() interface{} {
return node.value
}
// SetValue 设置值
func (node *Node) SetValue(v interface{}) {
node.value = v
}
// GetPrev 获取left值
func (node *Node) GetPrev() INode {
return node.prev
}
// SetPrev 设置left值
func (node *Node) SetPrev(n INode) {
node.prev = n
}
// GetNext 设置right值
func (node *Node) GetNext() INode {
return node.next
}
// SetNext 获取left值
func (node *Node) SetNext(n INode) {
node.next = n
}
func (node *Node) String() string {
return spew.Sprint(node.value)
}
// PriorityList 优先列表
type PriorityList struct {
head INode
tail INode
size int
}
// Size 长度
func (pl *PriorityList) Size() int {
return pl.size
}
// Clear 清空链表, 如果外部有引用其中一个节点 要把节点Prev Next值为nil, 三色标记
func (pl *PriorityList) Clear() {
pl.head = nil
pl.tail = nil
pl.size = 0
}
// GetCompare 获取比较的节点 compare >= 11这个节点小的值 [15,12,9,8,6,1] = 9
func (pl *PriorityList) GetCompare(cNode INode) INode {
cur := pl.head
for cur != nil {
if !cur.Compare(cNode) {
return cur
}
cur = cur.GetNext()
}
return nil
}
// GetCompareReverse 逆序获取比较的节点 compare >= 11这个节点大的值 [15,12,9,8,6,1] = 12
func (pl *PriorityList) GetCompareReverse(cNode INode) INode {
cur := pl.tail
for cur != nil {
if cur.Compare(cNode) {
return cur
}
cur = cur.GetPrev()
}
return nil
}
// ForEach 顺序遍历
func (pl *PriorityList) ForEach(eachfunc func(INode) bool) {
cur := pl.head
for cur != nil {
if !eachfunc(cur) {
break
}
cur = cur.GetNext()
}
}
// ForEachReverse 逆遍历
func (pl *PriorityList) ForEachReverse(eachfunc func(INode) bool) {
cur := pl.tail
for cur != nil {
if !eachfunc(cur) {
break
}
cur = cur.GetPrev()
}
}
// Get 获取索引长度
func (pl *PriorityList) Get(idx int) INode {
if idx >= pl.size {
return nil
}
cur := pl.head
for i := 0; i < idx; i++ {
cur = cur.GetNext()
}
return cur
}
// GetReverse 腻序获取索引长度
func (pl *PriorityList) GetReverse(idx int) INode {
if idx >= pl.size {
return nil
}
cur := pl.tail
for i := 0; i < idx; i++ {
cur = cur.GetPrev()
}
return cur
}
// Remove 删除节点
func (pl *PriorityList) Remove(node INode) {
if pl.size <= 1 {
pl.head = nil
pl.tail = nil
if pl.size == 0 {
panic("the node is not in this list, node:" + spew.Sprint(node))
}
pl.size = 0
return
}
pl.size--
prev := node.GetPrev()
next := node.GetNext()
if prev == nil {
pl.head = next
next.SetPrev(nil)
node.SetNext(nil)
} else if next == nil {
pl.tail = prev
prev.SetNext(nil)
node.SetPrev(nil)
} else {
prev.SetNext(next)
next.SetPrev(prev)
node.SetPrev(nil)
node.SetNext(nil)
}
}
// RemoveIndex 删除节点 并获取该节点
func (pl *PriorityList) RemoveIndex(idx int) INode {
result := pl.Get(idx)
pl.Remove(result)
return result
}
// RemoveReverseIndex 逆顺序删除节点 并获取该节点
func (pl *PriorityList) RemoveReverseIndex(idx int) INode {
result := pl.GetReverse(idx)
pl.Remove(result)
return result
}
func (pl *PriorityList) String() string {
content := "["
cur := pl.head
for cur != nil {
content += spew.Sprint(cur.GetValue()) + ","
cur = cur.GetNext()
}
if content[len(content)-1:] == "," {
content = content[:len(content)-1]
}
content += "]"
return content
}
// Insert 插入节点
func (pl *PriorityList) Insert(node INode) {
pl.size++
if pl.head == nil {
pl.head = node
pl.tail = node
return
}
cur := pl.head
for {
if !cur.Compare(node) {
// 插入该值
prev := cur.GetPrev()
if prev == nil {
pl.head.SetPrev(node)
node.SetNext(pl.head)
pl.head = node
} else {
prev.SetNext(node)
node.SetNext(cur)
cur.SetPrev(node)
node.SetPrev(prev)
}
return
}
next := cur.GetNext()
if next == nil {
cur.SetNext(node)
node.SetPrev(cur)
pl.tail = node
return
}
cur = next
}
}
// New 创建 PriorityList
func New() *PriorityList {
return new(PriorityList)
}

View File

@ -1,223 +0,0 @@
package logdb
import (
"database/sql"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"sync"
"time"
"github.com/satori/go.uuid"
_ "github.com/go-sql-driver/mysql" // mysql驱动
yaml "gopkg.in/yaml.v2"
)
// LogDB 属性结构
type LogDB struct {
Charset string `yaml:"charset"`
DB string `yaml:"db"`
Hosts []string `yaml:"hosts"`
Password string `yaml:"password"`
Port string `yaml:"port"`
User string `yaml:"user"`
pid int
hostid int
nextCheck int64
checkLimit int64
driver *sql.DB
mutex sync.Mutex
}
// New 创建一个logdb的配置
func New(filename string) *LogDB {
logdb := LogDB{}
logdb.checkLimit = 300
logdb.pid = os.Getpid()
data, err := ioutil.ReadFile(filename)
if err != nil {
panic(err)
}
err = yaml.Unmarshal(data, &logdb)
if err != nil {
panic(err)
}
logdb.hostid = 0
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?timeout=15s&charset=%s", logdb.User, logdb.Password, logdb.Hosts[logdb.hostid], logdb.Port, logdb.DB, logdb.Charset))
if err != nil {
log.Println("connect", err)
} else {
logdb.driver = db
}
return &logdb
}
// Ping 是否Ping通数据库
func (logdb *LogDB) Ping() (result bool) {
log.Println("Ping")
logdb.mutex.Lock()
defer logdb.mutex.Unlock()
defer func() {
if err := recover(); err != nil {
result = false
log.Println(err, logdb.Hosts[logdb.hostid], " is unconnect ")
hostlen := len(logdb.Hosts)
errorhid := logdb.hostid
for i := 0; i < hostlen; i++ {
curid := errorhid + 1 + i
if curid >= hostlen {
curid = curid - hostlen
}
curHost := logdb.Hosts[curid]
myurl := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?timeout=30s&charset=%s", logdb.User, logdb.Password, curHost, logdb.Port, logdb.DB, logdb.Charset)
db, err := sql.Open("mysql", myurl)
if err != nil {
log.Println(err, curHost, " is connect fail")
continue
}
if err := db.Ping(); err != nil {
log.Println(err, curHost, " is connect fail")
continue
}
logdb.driver = db
logdb.hostid = curid
result = true
}
}
}()
if err := logdb.driver.Ping(); err != nil {
panic(err)
}
return true
}
// ADInsert 插入数据
func (logdb *LogDB) ADInsert(uid, device, platform, area_cc, section_id, response string, spider_id, channel, media, catch_account_id, status, priority int, ts_crawl time.Time) {
logdb.mutex.Lock()
defer logdb.mutex.Unlock()
_, err := logdb.driver.Exec("insert into log_spider (uid, spider_id, device, platform, channel, media, area_cc, catch_account_id, section_id, response, error_msg, status, priority, ts_crawl) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ? ,?, ?, ?)", uid, spider_id, device, platform, channel, media, area_cc, catch_account_id, section_id, response, "", status, priority, ts_crawl)
if err != nil {
log.Println(err)
log.Printf("for save ad sql: insert into log_spider (uid, spider_id, device, platform, channel, media, area_cc, catch_account_id, section_id, response, error_msg, status, priority, ts_crawl) values(%s, %s, %s, %s, %d, %d, %s, %d, %s, %s, %s, %d, %d, %s)\n", uid, spider_id, device, platform, channel, media, area_cc, catch_account_id, section_id, response, "", status, priority, ts_crawl.Format("2006-01-02 15:04:05"))
}
}
type ADResonse struct {
UID string
Response string
}
// ADParserSelect adpase 根据自己的spider_id, 选择selcount条数据进行处理. 10- 100条最佳
func (logdb *LogDB) ADParserSelect(spider_id int, selcount int) []ADResonse {
puid, err := uuid.NewV4()
if err != nil {
panic(err)
}
logdb.adCheckRecover(spider_id, 5*time.Minute)
_, err = logdb.driver.Exec("update log_spider set status = 10000, parse_id = ? where spider_id = ? and status = 0 limit ?", puid.String(), spider_id, selcount)
if err != nil {
log.Println(err)
return nil
}
rows, err := logdb.driver.Query("select uid, response from log_spider where spider_id = ? and parse_id = ? and status = 10000", spider_id, puid.String())
if err != nil {
log.Println(err)
return nil
}
var adresponse []ADResonse
var uid, response string
for rows.Next() {
rows.Scan(&uid, &response)
adresponse = append(adresponse, ADResonse{UID: uid, Response: response})
// log.Println(uid, response)
}
return adresponse
}
// adCheckRecover 处理恢复错误, 或者没处理完的Select 出来的数据, 5分钟以上最佳. 例子: intervalTime := time.Minute * 5
// spider_id 对应 spider_id
// intervalTime 每隔多少时间去检查一次
func (logdb *LogDB) adCheckRecover(spider_id int, intervalTime time.Duration) {
now := time.Now()
if now.Unix() > logdb.nextCheck {
logdb.nextCheck = now.Unix() + logdb.checkLimit
tsUpdate := now.Add(-intervalTime) // tsUpdate := now.Add(-time.Minute * 5)
_, err := logdb.driver.Exec("update log_spider set status = 0, error_msg = CONCAT(error_msg, 'Parser Timeout ') where status = 10000 and spider_id = ? and ts_update <= ?", spider_id, tsUpdate)
if err != nil {
log.Println(err)
}
}
}
// ADParserSuccess 解析成功后处理该条数据
func (logdb *LogDB) ADParserSuccess(uid string, successData string) {
logdb.mutex.Lock()
defer logdb.mutex.Unlock()
ext := make(map[string]string)
ext["success_data"] = successData
data, err := json.Marshal(&ext)
if err != nil || successData == "" {
_, err := logdb.driver.Exec("update log_spider set status = 200 where uid = ?", uid)
if err != nil {
log.Println(err)
}
} else {
_, err := logdb.driver.Exec("update log_spider set status = 200, ext = ? where uid = ?", string(data), uid)
if err != nil {
log.Println(err)
}
}
}
// Select 插入数据
func (logdb *LogDB) Select(query string, args ...interface{}) *sql.Rows {
logdb.mutex.Lock()
defer logdb.mutex.Unlock()
Rows, err := logdb.driver.Query(query, args...)
if err != nil {
log.Println(err)
return nil
}
return Rows
}
// ADError 广告错误后更新
func (logdb *LogDB) ADError(uid, error_msg string) {
logdb.mutex.Lock()
defer logdb.mutex.Unlock()
_, err := logdb.driver.Exec("update log_spider set status = 1000, error_msg=? where uid =?;", error_msg, uid)
if err != nil {
log.Println(err)
}
}

View File

@ -1,6 +0,0 @@
charset: utf8mb4
db: test_log
hosts: [192.168.6.101,192.168.6.102,192.168.6.103,192.168.6.104,192.168.6.105]
password: ag-spider-log
port: 4000
user: spider

View File

@ -1,30 +0,0 @@
# git ignore for idea (IntelliJ, Gogland,...)
.idea/
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
# Testing specific
*.out
*.prof
*.test

View File

@ -1 +0,0 @@
language: go

View File

@ -1,26 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [1.1.0] - 2018-10-31
### Added
- Generate random locale strings
- Country localised street names
- Country localised provinces
### Fixed
- Generating dates will respect varying number of days in a month
## [1.0.0] - 2018-10-30
### Added
- This CHANGELOG file to hopefully serve as an evolving example of a
standardized open source project CHANGELOG.
- Enforcing Semver compatability in releases
### Changed
- Update README.md to include information about release strategy
- Update README.md to link to CHANGELOG.md

View File

@ -1,20 +0,0 @@
The MIT License (MIT)
Copyright (c) 2013 David Pallinder
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -1,200 +0,0 @@
# go-randomdata
[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/Pallinder/go-randomdata/issues)
[![GoDoc](https://godoc.org/github.com/Pallinder/go-randomdata?status.svg)](https://godoc.org/github.com/Pallinder/go-randomdata)
[![Build Status](https://travis-ci.org/Pallinder/go-randomdata.png)](https://travis-ci.org/Pallinder/go-randomdata)
[![Go Report Card](https://goreportcard.com/badge/github.com/Pallinder/go-randomdata)](https://goreportcard.com/report/github.com/Pallinder/go-randomdata)
randomdata is a tiny help suite for generating random data such as
* first names (male or female)
* last names
* full names (male or female)
* country names (full name or iso 3166.1 alpha-2 or alpha-3)
* locales / language tags (bcp-47)
* random email address
* city names
* American state names (two chars or full)
* random numbers (in an interval)
* random paragraphs
* random bool values
* postal- or zip-codes formatted for a range of different countries.
* american sounding addresses / street names
* silly names - suitable for names of things
* random days
* random months
* random full date
* random full profile
* random date inside range
* random phone number
## Installation
```go get github.com/Pallinder/go-randomdata```
## Usage
```go
package main
import (
"fmt"
"github.com/Pallinder/go-randomdata"
)
func main() {
// Print a random silly name
fmt.Println(randomdata.SillyName())
// Print a male title
fmt.Println(randomdata.Title(randomdata.Male))
// Print a female title
fmt.Println(randomdata.Title(randomdata.Female))
// Print a title with random gender
fmt.Println(randomdata.Title(randomdata.RandomGender))
// Print a male first name
fmt.Println(randomdata.FirstName(randomdata.Male))
// Print a female first name
fmt.Println(randomdata.FirstName(randomdata.Female))
// Print a last name
fmt.Println(randomdata.LastName())
// Print a male name
fmt.Println(randomdata.FullName(randomdata.Male))
// Print a female name
fmt.Println(randomdata.FullName(randomdata.Female))
// Print a name with random gender
fmt.Println(randomdata.FullName(randomdata.RandomGender))
// Print an email
fmt.Println(randomdata.Email())
// Print a country with full text representation
fmt.Println(randomdata.Country(randomdata.FullCountry))
// Print a country using ISO 3166-1 alpha-2
fmt.Println(randomdata.Country(randomdata.TwoCharCountry))
// Print a country using ISO 3166-1 alpha-3
fmt.Println(randomdata.Country(randomdata.ThreeCharCountry))
// Print BCP 47 language tag
fmt.Println(randomdata.Locale())
// Print a currency using ISO 4217
fmt.Println(randomdata.Currency())
// Print the name of a random city
fmt.Println(randomdata.City())
// Print the name of a random american state
fmt.Println(randomdata.State(randomdata.Large))
// Print the name of a random american state using two chars
fmt.Println(randomdata.State(randomdata.Small))
// Print an american sounding street name
fmt.Println(randomdata.Street())
// Print an american sounding address
fmt.Println(randomdata.Address())
// Print a random number >= 10 and < 20
fmt.Println(randomdata.Number(10, 20))
// Print a number >= 0 and < 20
fmt.Println(randomdata.Number(20))
// Print a random float >= 0 and < 20 with decimal point 3
fmt.Println(randomdata.Decimal(0, 20, 3))
// Print a random float >= 10 and < 20
fmt.Println(randomdata.Decimal(10, 20))
// Print a random float >= 0 and < 20
fmt.Println(randomdata.Decimal(20))
// Print a bool
fmt.Println(randomdata.Boolean())
// Print a paragraph
fmt.Println(randomdata.Paragraph())
// Print a postal code
fmt.Println(randomdata.PostalCode("SE"))
// Print a set of 2 random numbers as a string
fmt.Println(randomdata.StringNumber(2, "-"))
// Print a set of 2 random 3-Digits numbers as a string
fmt.Println(randomdata.StringNumberExt(2, "-", 3))
// Print a random string sampled from a list of strings
fmt.Println(randomdata.StringSample("my string 1", "my string 2", "my string 3"))
// Print a valid random IPv4 address
fmt.Println(randomdata.IpV4Address())
// Print a valid random IPv6 address
fmt.Println(randomdata.IpV6Address())
// Print a browser's user agent string
fmt.Println(randomdata.UserAgentString())
// Print a day
fmt.Println(randomdata.Day())
// Print a month
fmt.Println(randomdata.Month())
// Print full date like Monday 22 Aug 2016
fmt.Println(randomdata.FullDate())
// Print full date <= Monday 22 Aug 2016
fmt.Println(randomdata.FullDateInRange("2016-08-22"))
// Print full date >= Monday 01 Aug 2016 and <= Monday 22 Aug 2016
fmt.Println(randomdata.FullDateInRange("2016-08-01", "2016-08-22"))
// Print phone number according to e.164
fmt.Println(randomdata.PhoneNumber())
// Get a complete and randomised profile of data generally used for users
// There are many fields in the profile to use check the Profile struct definition in fullprofile.go
profile := randomdata.GenerateProfile(randomdata.Male | randomdata.Female | randomdata.RandomGender)
fmt.Printf("The new profile's username is: %s and password (md5): %s\n", profile.Login.Username, profile.Login.Md5)
// Get a random country-localised street name for Great Britain
fmt.Println(randomdata.StreetForCountry("GB"))
// Get a random country-localised street name for USA
fmt.Println(randomdata.StreetForCountry("US"))
// Get a random country-localised province for Great Britain
fmt.Println(randomdata.ProvinceForCountry("GB"))
// Get a random country-localised province for USA
fmt.Println(randomdata.ProvinceForCountry("US"))
}
```
## Versioning / Release Strategy
Go-Randomdata follows [Semver](https://www.semver.org)
You can find current releases tagged under the [releases section](https://github.com/Pallinder/go-randomdata/releases).
The [CHANGELOG.md](CHANGELOG.md) file contains the changelog of the project.
## Contributors
* [jteeuwen](https://github.com/jteeuwen)
* [n1try](https://github.com/n1try)
All the other contributors are listed [here](https://github.com/Pallinder/go-randomdata/graphs/contributors).

View File

@ -1,139 +0,0 @@
package randomdata
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"strconv"
"strings"
)
var letterRunes = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var portraitDirs = []string{"men", "women"}
type Profile struct {
Gender string `json:"gender"`
Name struct {
First string `json:"first"`
Last string `json:"last"`
Title string `json:"title"`
} `json:"name"`
Location struct {
Street string `json:"street"`
City string `json:"city"`
State string `json:"state"`
Postcode int `json:"postcode"`
} `json:"location"`
Email string `json:"email"`
Login struct {
Username string `json:"username"`
Password string `json:"password"`
Salt string `json:"salt"`
Md5 string `json:"md5"`
Sha1 string `json:"sha1"`
Sha256 string `json:"sha256"`
} `json:"login"`
Dob string `json:"dob"`
Registered string `json:"registered"`
Phone string `json:"phone"`
Cell string `json:"cell"`
ID struct {
Name string `json:"name"`
Value interface{} `json:"value"`
} `json:"id"`
Picture struct {
Large string `json:"large"`
Medium string `json:"medium"`
Thumbnail string `json:"thumbnail"`
} `json:"picture"`
Nat string `json:"nat"`
}
func RandStringRunes(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = letterRunes[privateRand.Intn(len(letterRunes))]
}
return string(b)
}
func getMD5Hash(text string) string {
hasher := md5.New()
hasher.Write([]byte(text))
return hex.EncodeToString(hasher.Sum(nil))
}
func getSha1(text string) string {
hasher := sha1.New()
hasher.Write([]byte(text))
sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
return sha
}
func getSha256(text string) string {
hasher := sha256.New()
hasher.Write([]byte(text))
sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
return sha
}
func GenerateProfile(gender int) *Profile {
profile := &Profile{}
if gender == Male {
profile.Gender = "male"
} else if gender == Female {
profile.Gender = "female"
} else {
gender = privateRand.Intn(2)
if gender == Male {
profile.Gender = "male"
} else {
profile.Gender = "female"
}
}
profile.Name.Title = Title(gender)
profile.Name.First = FirstName(gender)
profile.Name.Last = LastName()
profile.ID.Name = "SSN"
profile.ID.Value = fmt.Sprintf("%d-%d-%d",
Number(101, 999),
Number(01, 99),
Number(100, 9999),
)
profile.Email = strings.ToLower(profile.Name.First) + "." + strings.ToLower(profile.Name.Last) + "@example.com"
profile.Cell = PhoneNumber()
profile.Phone = PhoneNumber()
profile.Dob = FullDate()
profile.Registered = FullDate()
profile.Nat = "US"
profile.Location.City = City()
i, _ := strconv.Atoi(PostalCode("US"))
profile.Location.Postcode = i
profile.Location.State = State(2)
profile.Location.Street = StringNumber(1, "") + " " + Street()
profile.Login.Username = SillyName()
pass := SillyName()
salt := RandStringRunes(16)
profile.Login.Password = pass
profile.Login.Salt = salt
profile.Login.Md5 = getMD5Hash(pass + salt)
profile.Login.Sha1 = getSha1(pass + salt)
profile.Login.Sha256 = getSha256(pass + salt)
pic := privateRand.Intn(35)
profile.Picture.Large = fmt.Sprintf("https://randomuser.me/api/portraits/%s/%d.jpg", portraitDirs[gender], pic)
profile.Picture.Medium = fmt.Sprintf("https://randomuser.me/api/portraits/med/%s/%d.jpg", portraitDirs[gender], pic)
profile.Picture.Thumbnail = fmt.Sprintf("https://randomuser.me/api/portraits/thumb/%s/%d.jpg", portraitDirs[gender], pic)
return profile
}

File diff suppressed because it is too large Load Diff

View File

@ -1,243 +0,0 @@
package randomdata
import (
"fmt"
"math"
"strings"
)
// Supported formats obtained from:
// * http://www.geopostcodes.com/GeoPC_Postal_codes_formats
// PostalCode yields a random postal/zip code for the given 2-letter country code.
//
// These codes are not guaranteed to refer to actually locations.
// They merely follow the correct format as far as letters and digits goes.
// Where possible, the function enforces valid ranges of letters and digits.
func PostalCode(countrycode string) string {
switch strings.ToUpper(countrycode) {
case "LS", "MG", "IS", "OM", "PG":
return Digits(3)
case "AM", "GE", "NZ", "NE", "NO", "PY", "ZA", "MZ", "SJ", "LI", "AL",
"BD", "CV", "GL":
return Digits(4)
case "DZ", "BA", "KH", "DO", "EG", "EE", "GP", "GT", "ID", "IL", "JO",
"KW", "MQ", "MX", "LK", "SD", "TR", "UA", "US", "CR", "IQ", "KV", "MY",
"MN", "ME", "PK", "SM", "MA", "UY", "EH", "ZM":
return Digits(5)
case "BY", "CN", "IN", "KZ", "KG", "NG", "RO", "RU", "SG", "TJ", "TM", "UZ", "VN":
return Digits(6)
case "CL":
return Digits(7)
case "IR":
return Digits(10)
case "FO":
return "FO " + Digits(3)
case "AF":
return BoundedDigits(2, 10, 43) + BoundedDigits(2, 1, 99)
case "AU", "AT", "BE", "BG", "CY", "DK", "ET", "GW", "HU", "LR", "MK", "PH",
"CH", "TN", "VE":
return BoundedDigits(4, 1000, 9999)
case "SV":
return "CP " + BoundedDigits(4, 1000, 9999)
case "HT":
return "HT" + Digits(4)
case "LB":
return Digits(4) + " " + Digits(4)
case "LU":
return BoundedDigits(4, 6600, 6999)
case "MD":
return "MD-" + BoundedDigits(4, 1000, 9999)
case "HR":
return "HR-" + Digits(5)
case "CU":
return "CP " + BoundedDigits(5, 10000, 99999)
case "FI":
// Last digit is usually 0 but can, in some cases, be 1 or 5.
switch privateRand.Intn(2) {
case 0:
return Digits(4) + "0"
case 1:
return Digits(4) + "1"
}
return Digits(4) + "5"
case "FR", "GF", "PF", "YT", "MC", "RE", "BL", "MF", "PM", "RS", "TH":
return BoundedDigits(5, 10000, 99999)
case "DE":
return BoundedDigits(5, 1000, 99999)
case "GR":
return BoundedDigits(3, 100, 999) + " " + Digits(2)
case "HN":
return "CM" + Digits(4)
case "IT", "VA":
return BoundedDigits(5, 10, 99999)
case "KE":
return BoundedDigits(5, 100, 99999)
case "LA":
return BoundedDigits(5, 1000, 99999)
case "MH":
return BoundedDigits(5, 96960, 96970)
case "FM":
return "FM" + BoundedDigits(5, 96941, 96944)
case "MM":
return BoundedDigits(2, 1, 14) + Digits(3)
case "NP":
return BoundedDigits(5, 10700, 56311)
case "NC":
return "98" + Digits(3)
case "PW":
return "PW96940"
case "PR":
return "PR " + Digits(5)
case "SA":
return BoundedDigits(5, 10000, 99999) + "-" + BoundedDigits(4, 1000, 9999)
case "ES":
return BoundedDigits(2, 1, 52) + BoundedDigits(3, 100, 999)
case "WF":
return "986" + Digits(2)
case "SZ":
return Letters(1) + Digits(3)
case "BM":
return Letters(2) + Digits(2)
case "AD":
return Letters(2) + Digits(3)
case "BN", "AZ", "VG", "PE":
return Letters(2) + Digits(4)
case "BB":
return Letters(2) + Digits(5)
case "EC":
return Letters(2) + Digits(6)
case "MT":
return Letters(3) + Digits(4)
case "JM":
return "JM" + Letters(3) + Digits(2)
case "AR":
return Letters(1) + Digits(4) + Letters(3)
case "CA":
return Letters(1) + Digits(1) + Letters(1) + Digits(1) + Letters(1) + Digits(1)
case "FK", "TC":
return Letters(4) + Digits(1) + Letters(2)
case "GG", "IM", "JE", "GB":
return Letters(2) + Digits(2) + Letters(2)
case "KY":
return Letters(2) + Digits(1) + "-" + Digits(4)
case "JP":
return Digits(3) + "-" + Digits(4)
case "LV", "SI":
return Letters(2) + "-" + Digits(4)
case "LT":
return Letters(2) + "-" + Digits(5)
case "SE", "TW":
return Digits(5)
case "MV":
return Digits(2) + "-" + Digits(2)
case "PL":
return Digits(2) + "-" + Digits(3)
case "NI":
return Digits(3) + "-" + Digits(3) + "-" + Digits(1)
case "KR":
return Digits(3) + "-" + Digits(3)
case "PT":
return Digits(4) + "-" + Digits(3)
case "NL":
return Digits(4) + Letters(2)
case "BR":
return Digits(5) + "-" + Digits(3)
}
return ""
}
// Letters generates a string of N random leters (A-Z).
func Letters(letters int) string {
list := make([]byte, letters)
for i := range list {
list[i] = byte(privateRand.Intn('Z'-'A') + 'A')
}
return string(list)
}
// Digits generates a string of N random digits, padded with zeros if necessary.
func Digits(digits int) string {
max := int(math.Pow10(digits)) - 1
num := privateRand.Intn(max)
format := fmt.Sprintf("%%0%dd", digits)
return fmt.Sprintf(format, num)
}
// BoundedDigits generates a string of N random digits, padded with zeros if necessary.
// The output is restricted to the given range.
func BoundedDigits(digits, low, high int) string {
if low > high {
low, high = high, low
}
max := int(math.Pow10(digits)) - 1
if high > max {
high = max
}
num := privateRand.Intn(high-low+1) + low
format := fmt.Sprintf("%%0%dd", digits)
return fmt.Sprintf(format, num)
}

View File

@ -1,417 +0,0 @@
// Package randomdata implements a bunch of simple ways to generate (pseudo) random data
package randomdata
import (
"encoding/json"
"fmt"
"log"
"math/rand"
"net"
"strconv"
"strings"
"time"
"unicode"
)
const (
Male int = 0
Female int = 1
RandomGender int = 2
)
const (
Small int = 0
Large int = 1
)
const (
FullCountry = 0
TwoCharCountry = 1
ThreeCharCountry = 2
)
const (
DateInputLayout = "2006-01-02"
DateOutputLayout = "Monday 2 Jan 2006"
)
type jsonContent struct {
Adjectives []string `json:"adjectives"`
Nouns []string `json:"nouns"`
FirstNamesFemale []string `json:"firstNamesFemale"`
FirstNamesMale []string `json:"firstNamesMale"`
LastNames []string `json:"lastNames"`
Domains []string `json:"domains"`
People []string `json:"people"`
StreetTypes []string `json:"streetTypes"` // Taken from https://github.com/tomharris/random_data/blob/master/lib/random_data/locations.rb
Paragraphs []string `json:"paragraphs"` // Taken from feedbooks.com and www.gutenberg.org
Countries []string `json:"countries"` // Fetched from the world bank at http://siteresources.worldbank.org/DATASTATISTICS/Resources/CLASS.XLS
CountriesThreeChars []string `json:"countriesThreeChars"`
CountriesTwoChars []string `json:"countriesTwoChars"`
Currencies []string `json:"currencies"` //https://github.com/OpenBookPrices/country-data
Cities []string `json:"cities"`
States []string `json:"states"`
StatesSmall []string `json:"statesSmall"`
Days []string `json:"days"`
Months []string `json:"months"`
FemaleTitles []string `json:"femaleTitles"`
MaleTitles []string `json:"maleTitles"`
Timezones []string `json:"timezones"` // https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
Locales []string `json:"locales"` // https://tools.ietf.org/html/bcp47
UserAgents []string `json:"userAgents"` // http://techpatterns.com/downloads/firefox/useragentswitcher.xml
CountryCallingCodes []string `json:"countryCallingCodes"` // from https://github.com/datasets/country-codes/blob/master/data/country-codes.csv
ProvincesGB []string `json:"provincesGB"`
StreetNameGB []string `json:"streetNameGB"`
StreetTypesGB []string `json:"streetTypesGB"`
}
var jsonData = jsonContent{}
var privateRand *rand.Rand
func init() {
privateRand = rand.New(rand.NewSource(time.Now().UnixNano()))
jsonData = jsonContent{}
err := json.Unmarshal(data, &jsonData)
if err != nil {
log.Fatal(err)
}
}
func CustomRand(randToUse *rand.Rand) {
privateRand = randToUse
}
// Returns a random part of a slice
func randomFrom(source []string) string {
return source[privateRand.Intn(len(source))]
}
// Title returns a random title, gender decides the gender of the name
func Title(gender int) string {
var title = ""
switch gender {
case Male:
title = randomFrom(jsonData.MaleTitles)
break
case Female:
title = randomFrom(jsonData.FemaleTitles)
break
default:
title = FirstName(privateRand.Intn(2))
break
}
return title
}
// FirstName returns a random first name, gender decides the gender of the name
func FirstName(gender int) string {
var name = ""
switch gender {
case Male:
name = randomFrom(jsonData.FirstNamesMale)
break
case Female:
name = randomFrom(jsonData.FirstNamesFemale)
break
default:
name = FirstName(rand.Intn(2))
break
}
return name
}
// LastName returns a random last name
func LastName() string {
return randomFrom(jsonData.LastNames)
}
// FullName returns a combination of FirstName LastName randomized, gender decides the gender of the name
func FullName(gender int) string {
return FirstName(gender) + " " + LastName()
}
// Email returns a random email
func Email() string {
return strings.ToLower(FirstName(RandomGender)+LastName()) + StringNumberExt(1, "", 3) + "@" + randomFrom(jsonData.Domains)
}
// Country returns a random country, countryStyle decides what kind of format the returned country will have
func Country(countryStyle int64) string {
country := ""
switch countryStyle {
default:
case FullCountry:
country = randomFrom(jsonData.Countries)
break
case TwoCharCountry:
country = randomFrom(jsonData.CountriesTwoChars)
break
case ThreeCharCountry:
country = randomFrom(jsonData.CountriesThreeChars)
break
}
return country
}
// Currency returns a random currency under ISO 4217 format
func Currency() string {
return randomFrom(jsonData.Currencies)
}
// City returns a random city
func City() string {
return randomFrom(jsonData.Cities)
}
// ProvinceForCountry returns a randomly selected province (state, county,subdivision ) name for a supplied country.
// If the country is not supported it will return an empty string.
func ProvinceForCountry(countrycode string) string {
switch countrycode {
case "US":
return randomFrom(jsonData.States)
case "GB":
return randomFrom(jsonData.ProvincesGB)
}
return ""
}
// State returns a random american state
func State(typeOfState int) string {
if typeOfState == Small {
return randomFrom(jsonData.StatesSmall)
}
return randomFrom(jsonData.States)
}
// Street returns a random fake street name
func Street() string {
return fmt.Sprintf("%s %s", randomFrom(jsonData.People), randomFrom(jsonData.StreetTypes))
}
// StreetForCountry returns a random fake street name typical to the supplied country.
// If the country is not supported it will return an empty string.
func StreetForCountry(countrycode string) string {
switch countrycode {
case "US":
return Street()
case "GB":
return fmt.Sprintf("%s %s", randomFrom(jsonData.StreetNameGB), randomFrom(jsonData.StreetTypesGB))
}
return ""
}
// Address returns an american style address
func Address() string {
return fmt.Sprintf("%d %s,\n%s, %s, %s", Number(100), Street(), City(), State(Small), PostalCode("US"))
}
// Paragraph returns a random paragraph
func Paragraph() string {
return randomFrom(jsonData.Paragraphs)
}
// Number returns a random number, if only one integer (n1) is supplied it returns a number in [0,n1)
// if a second argument is supplied it returns a number in [n1,n2)
func Number(numberRange ...int) int {
nr := 0
if len(numberRange) > 1 {
nr = 1
nr = privateRand.Intn(numberRange[1]-numberRange[0]) + numberRange[0]
} else {
nr = privateRand.Intn(numberRange[0])
}
return nr
}
func Decimal(numberRange ...int) float64 {
nr := 0.0
if len(numberRange) > 1 {
nr = 1.0
nr = privateRand.Float64()*(float64(numberRange[1])-float64(numberRange[0])) + float64(numberRange[0])
} else {
nr = privateRand.Float64() * float64(numberRange[0])
}
if len(numberRange) > 2 {
sf := strconv.FormatFloat(nr, 'f', numberRange[2], 64)
nr, _ = strconv.ParseFloat(sf, 64)
}
return nr
}
func StringNumberExt(numberPairs int, separator string, numberOfDigits int) string {
numberString := ""
for i := 0; i < numberPairs; i++ {
for d := 0; d < numberOfDigits; d++ {
numberString += fmt.Sprintf("%d", Number(0, 9))
}
if i+1 != numberPairs {
numberString += separator
}
}
return numberString
}
// StringNumber returns a random number as a string
func StringNumber(numberPairs int, separator string) string {
return StringNumberExt(numberPairs, separator, 2)
}
// StringSample returns a random string from a list of strings
func StringSample(stringList ...string) string {
str := ""
if len(stringList) > 0 {
str = stringList[Number(0, len(stringList))]
}
return str
}
func Boolean() bool {
nr := privateRand.Intn(2)
return nr != 0
}
// Noun returns a random noun
func Noun() string {
return randomFrom(jsonData.Nouns)
}
// Adjective returns a random adjective
func Adjective() string {
return randomFrom(jsonData.Adjectives)
}
func uppercaseFirstLetter(word string) string {
a := []rune(word)
a[0] = unicode.ToUpper(a[0])
return string(a)
}
func lowercaseFirstLetter(word string) string {
a := []rune(word)
a[0] = unicode.ToLower(a[0])
return string(a)
}
// SillyName returns a silly name, useful for randomizing naming of things
func SillyName() string {
return uppercaseFirstLetter(Noun()) + Adjective()
}
// IpV4Address returns a valid IPv4 address as string
func IpV4Address() string {
blocks := []string{}
for i := 0; i < 4; i++ {
number := privateRand.Intn(255)
blocks = append(blocks, strconv.Itoa(number))
}
return strings.Join(blocks, ".")
}
// IpV6Address returns a valid IPv6 address as net.IP
func IpV6Address() string {
var ip net.IP
for i := 0; i < net.IPv6len; i++ {
number := uint8(privateRand.Intn(255))
ip = append(ip, number)
}
return ip.String()
}
// MacAddress returns an mac address string
func MacAddress() string {
blocks := []string{}
for i := 0; i < 6; i++ {
number := fmt.Sprintf("%02x", privateRand.Intn(255))
blocks = append(blocks, number)
}
return strings.Join(blocks, ":")
}
// Day returns random day
func Day() string {
return randomFrom(jsonData.Days)
}
// Month returns random month
func Month() string {
return randomFrom(jsonData.Months)
}
// FullDate returns full date
func FullDate() string {
timestamp := time.Now()
year := timestamp.Year()
month := Number(1, 13)
maxDay := time.Date(year, time.Month(month+1), 0, 0, 0, 0, 0, time.UTC).Day()
day := Number(1, maxDay+1)
date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
return date.Format(DateOutputLayout)
}
// FullDateInRange returns a date string within a given range, given in the format "2006-01-02".
// If no argument is supplied it will return the result of randomdata.FullDate().
// If only one argument is supplied it is treated as the max date to return.
// If a second argument is supplied it returns a date between (and including) the two dates.
// Returned date is in format "Monday 2 Jan 2006".
func FullDateInRange(dateRange ...string) string {
var (
min time.Time
max time.Time
duration int
dateString string
)
if len(dateRange) == 1 {
max, _ = time.Parse(DateInputLayout, dateRange[0])
} else if len(dateRange) == 2 {
min, _ = time.Parse(DateInputLayout, dateRange[0])
max, _ = time.Parse(DateInputLayout, dateRange[1])
}
if !max.IsZero() && max.After(min) {
duration = Number(int(max.Sub(min))) * -1
dateString = max.Add(time.Duration(duration)).Format(DateOutputLayout)
} else if !max.IsZero() && !max.After(min) {
dateString = max.Format(DateOutputLayout)
} else {
dateString = FullDate()
}
return dateString
}
func Timezone() string {
return randomFrom(jsonData.Timezones)
}
func Locale() string {
return randomFrom(jsonData.Locales)
}
func UserAgentString() string {
return randomFrom(jsonData.UserAgents)
}
func PhoneNumber() string {
str := randomFrom(jsonData.CountryCallingCodes) + " "
str += Digits(privateRand.Intn(3) + 1)
for {
// max 15 chars
remaining := 15 - (len(str) - strings.Count(str, " "))
if remaining < 2 {
return "+" + str
}
str += " " + Digits(privateRand.Intn(remaining-1)+1)
}
}

View File

@ -1,15 +0,0 @@
ISC License
Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

View File

@ -1,145 +0,0 @@
// Copyright (c) 2015-2016 Dave Collins <dave@davec.name>
//
// Permission to use, copy, modify, and distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
// NOTE: Due to the following build constraints, this file will only be compiled
// when the code is not running on Google App Engine, compiled by GopherJS, and
// "-tags safe" is not added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// Go versions prior to 1.4 are disabled because they use a different layout
// for interfaces which make the implementation of unsafeReflectValue more complex.
// +build !js,!appengine,!safe,!disableunsafe,go1.4
package spew
import (
"reflect"
"unsafe"
)
const (
// UnsafeDisabled is a build-time constant which specifies whether or
// not access to the unsafe package is available.
UnsafeDisabled = false
// ptrSize is the size of a pointer on the current arch.
ptrSize = unsafe.Sizeof((*byte)(nil))
)
type flag uintptr
var (
// flagRO indicates whether the value field of a reflect.Value
// is read-only.
flagRO flag
// flagAddr indicates whether the address of the reflect.Value's
// value may be taken.
flagAddr flag
)
// flagKindMask holds the bits that make up the kind
// part of the flags field. In all the supported versions,
// it is in the lower 5 bits.
const flagKindMask = flag(0x1f)
// Different versions of Go have used different
// bit layouts for the flags type. This table
// records the known combinations.
var okFlags = []struct {
ro, addr flag
}{{
// From Go 1.4 to 1.5
ro: 1 << 5,
addr: 1 << 7,
}, {
// Up to Go tip.
ro: 1<<5 | 1<<6,
addr: 1 << 8,
}}
var flagValOffset = func() uintptr {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
return field.Offset
}()
// flagField returns a pointer to the flag field of a reflect.Value.
func flagField(v *reflect.Value) *flag {
return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset))
}
// unsafeReflectValue converts the passed reflect.Value into a one that bypasses
// the typical safety restrictions preventing access to unaddressable and
// unexported data. It works by digging the raw pointer to the underlying
// value out of the protected value and generating a new unprotected (unsafe)
// reflect.Value to it.
//
// This allows us to check for implementations of the Stringer and error
// interfaces to be used for pretty printing ordinarily unaddressable and
// inaccessible values such as unexported struct fields.
func unsafeReflectValue(v reflect.Value) reflect.Value {
if !v.IsValid() || (v.CanInterface() && v.CanAddr()) {
return v
}
flagFieldPtr := flagField(&v)
*flagFieldPtr &^= flagRO
*flagFieldPtr |= flagAddr
return v
}
// Sanity checks against future reflect package changes
// to the type or semantics of the Value.flag field.
func init() {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() {
panic("reflect.Value flag field has changed kind")
}
type t0 int
var t struct {
A t0
// t0 will have flagEmbedRO set.
t0
// a will have flagStickyRO set
a t0
}
vA := reflect.ValueOf(t).FieldByName("A")
va := reflect.ValueOf(t).FieldByName("a")
vt0 := reflect.ValueOf(t).FieldByName("t0")
// Infer flagRO from the difference between the flags
// for the (otherwise identical) fields in t.
flagPublic := *flagField(&vA)
flagWithRO := *flagField(&va) | *flagField(&vt0)
flagRO = flagPublic ^ flagWithRO
// Infer flagAddr from the difference between a value
// taken from a pointer and not.
vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A")
flagNoPtr := *flagField(&vA)
flagPtr := *flagField(&vPtrA)
flagAddr = flagNoPtr ^ flagPtr
// Check that the inferred flags tally with one of the known versions.
for _, f := range okFlags {
if flagRO == f.ro && flagAddr == f.addr {
return
}
}
panic("reflect.Value read-only flag has changed semantics")
}

View File

@ -1,38 +0,0 @@
// Copyright (c) 2015-2016 Dave Collins <dave@davec.name>
//
// Permission to use, copy, modify, and distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
// NOTE: Due to the following build constraints, this file will only be compiled
// when the code is running on Google App Engine, compiled by GopherJS, or
// "-tags safe" is added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build js appengine safe disableunsafe !go1.4
package spew
import "reflect"
const (
// UnsafeDisabled is a build-time constant which specifies whether or
// not access to the unsafe package is available.
UnsafeDisabled = true
)
// unsafeReflectValue typically converts the passed reflect.Value into a one
// that bypasses the typical safety restrictions preventing access to
// unaddressable and unexported data. However, doing this relies on access to
// the unsafe package. This is a stub version which simply returns the passed
// reflect.Value when the unsafe package is not available.
func unsafeReflectValue(v reflect.Value) reflect.Value {
return v
}

View File

@ -1,341 +0,0 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"fmt"
"io"
"reflect"
"sort"
"strconv"
)
// Some constants in the form of bytes to avoid string overhead. This mirrors
// the technique used in the fmt package.
var (
panicBytes = []byte("(PANIC=")
plusBytes = []byte("+")
iBytes = []byte("i")
trueBytes = []byte("true")
falseBytes = []byte("false")
interfaceBytes = []byte("(interface {})")
commaNewlineBytes = []byte(",\n")
newlineBytes = []byte("\n")
openBraceBytes = []byte("{")
openBraceNewlineBytes = []byte("{\n")
closeBraceBytes = []byte("}")
asteriskBytes = []byte("*")
colonBytes = []byte(":")
colonSpaceBytes = []byte(": ")
openParenBytes = []byte("(")
closeParenBytes = []byte(")")
spaceBytes = []byte(" ")
pointerChainBytes = []byte("->")
nilAngleBytes = []byte("<nil>")
maxNewlineBytes = []byte("<max depth reached>\n")
maxShortBytes = []byte("<max>")
circularBytes = []byte("<already shown>")
circularShortBytes = []byte("<shown>")
invalidAngleBytes = []byte("<invalid>")
openBracketBytes = []byte("[")
closeBracketBytes = []byte("]")
percentBytes = []byte("%")
precisionBytes = []byte(".")
openAngleBytes = []byte("<")
closeAngleBytes = []byte(">")
openMapBytes = []byte("map[")
closeMapBytes = []byte("]")
lenEqualsBytes = []byte("len=")
capEqualsBytes = []byte("cap=")
)
// hexDigits is used to map a decimal value to a hex digit.
var hexDigits = "0123456789abcdef"
// catchPanic handles any panics that might occur during the handleMethods
// calls.
func catchPanic(w io.Writer, v reflect.Value) {
if err := recover(); err != nil {
w.Write(panicBytes)
fmt.Fprintf(w, "%v", err)
w.Write(closeParenBytes)
}
}
// handleMethods attempts to call the Error and String methods on the underlying
// type the passed reflect.Value represents and outputes the result to Writer w.
//
// It handles panics in any called methods by catching and displaying the error
// as the formatted value.
func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) {
// We need an interface to check if the type implements the error or
// Stringer interface. However, the reflect package won't give us an
// interface on certain things like unexported struct fields in order
// to enforce visibility rules. We use unsafe, when it's available,
// to bypass these restrictions since this package does not mutate the
// values.
if !v.CanInterface() {
if UnsafeDisabled {
return false
}
v = unsafeReflectValue(v)
}
// Choose whether or not to do error and Stringer interface lookups against
// the base type or a pointer to the base type depending on settings.
// Technically calling one of these methods with a pointer receiver can
// mutate the value, however, types which choose to satisify an error or
// Stringer interface with a pointer receiver should not be mutating their
// state inside these interface methods.
if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() {
v = unsafeReflectValue(v)
}
if v.CanAddr() {
v = v.Addr()
}
// Is it an error or Stringer?
switch iface := v.Interface().(type) {
case error:
defer catchPanic(w, v)
if cs.ContinueOnMethod {
w.Write(openParenBytes)
w.Write([]byte(iface.Error()))
w.Write(closeParenBytes)
w.Write(spaceBytes)
return false
}
w.Write([]byte(iface.Error()))
return true
case fmt.Stringer:
defer catchPanic(w, v)
if cs.ContinueOnMethod {
w.Write(openParenBytes)
w.Write([]byte(iface.String()))
w.Write(closeParenBytes)
w.Write(spaceBytes)
return false
}
w.Write([]byte(iface.String()))
return true
}
return false
}
// printBool outputs a boolean value as true or false to Writer w.
func printBool(w io.Writer, val bool) {
if val {
w.Write(trueBytes)
} else {
w.Write(falseBytes)
}
}
// printInt outputs a signed integer value to Writer w.
func printInt(w io.Writer, val int64, base int) {
w.Write([]byte(strconv.FormatInt(val, base)))
}
// printUint outputs an unsigned integer value to Writer w.
func printUint(w io.Writer, val uint64, base int) {
w.Write([]byte(strconv.FormatUint(val, base)))
}
// printFloat outputs a floating point value using the specified precision,
// which is expected to be 32 or 64bit, to Writer w.
func printFloat(w io.Writer, val float64, precision int) {
w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision)))
}
// printComplex outputs a complex value using the specified float precision
// for the real and imaginary parts to Writer w.
func printComplex(w io.Writer, c complex128, floatPrecision int) {
r := real(c)
w.Write(openParenBytes)
w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision)))
i := imag(c)
if i >= 0 {
w.Write(plusBytes)
}
w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision)))
w.Write(iBytes)
w.Write(closeParenBytes)
}
// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x'
// prefix to Writer w.
func printHexPtr(w io.Writer, p uintptr) {
// Null pointer.
num := uint64(p)
if num == 0 {
w.Write(nilAngleBytes)
return
}
// Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix
buf := make([]byte, 18)
// It's simpler to construct the hex string right to left.
base := uint64(16)
i := len(buf) - 1
for num >= base {
buf[i] = hexDigits[num%base]
num /= base
i--
}
buf[i] = hexDigits[num]
// Add '0x' prefix.
i--
buf[i] = 'x'
i--
buf[i] = '0'
// Strip unused leading bytes.
buf = buf[i:]
w.Write(buf)
}
// valuesSorter implements sort.Interface to allow a slice of reflect.Value
// elements to be sorted.
type valuesSorter struct {
values []reflect.Value
strings []string // either nil or same len and values
cs *ConfigState
}
// newValuesSorter initializes a valuesSorter instance, which holds a set of
// surrogate keys on which the data should be sorted. It uses flags in
// ConfigState to decide if and how to populate those surrogate keys.
func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface {
vs := &valuesSorter{values: values, cs: cs}
if canSortSimply(vs.values[0].Kind()) {
return vs
}
if !cs.DisableMethods {
vs.strings = make([]string, len(values))
for i := range vs.values {
b := bytes.Buffer{}
if !handleMethods(cs, &b, vs.values[i]) {
vs.strings = nil
break
}
vs.strings[i] = b.String()
}
}
if vs.strings == nil && cs.SpewKeys {
vs.strings = make([]string, len(values))
for i := range vs.values {
vs.strings[i] = Sprintf("%#v", vs.values[i].Interface())
}
}
return vs
}
// canSortSimply tests whether a reflect.Kind is a primitive that can be sorted
// directly, or whether it should be considered for sorting by surrogate keys
// (if the ConfigState allows it).
func canSortSimply(kind reflect.Kind) bool {
// This switch parallels valueSortLess, except for the default case.
switch kind {
case reflect.Bool:
return true
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
return true
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
return true
case reflect.Float32, reflect.Float64:
return true
case reflect.String:
return true
case reflect.Uintptr:
return true
case reflect.Array:
return true
}
return false
}
// Len returns the number of values in the slice. It is part of the
// sort.Interface implementation.
func (s *valuesSorter) Len() int {
return len(s.values)
}
// Swap swaps the values at the passed indices. It is part of the
// sort.Interface implementation.
func (s *valuesSorter) Swap(i, j int) {
s.values[i], s.values[j] = s.values[j], s.values[i]
if s.strings != nil {
s.strings[i], s.strings[j] = s.strings[j], s.strings[i]
}
}
// valueSortLess returns whether the first value should sort before the second
// value. It is used by valueSorter.Less as part of the sort.Interface
// implementation.
func valueSortLess(a, b reflect.Value) bool {
switch a.Kind() {
case reflect.Bool:
return !a.Bool() && b.Bool()
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
return a.Int() < b.Int()
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
return a.Uint() < b.Uint()
case reflect.Float32, reflect.Float64:
return a.Float() < b.Float()
case reflect.String:
return a.String() < b.String()
case reflect.Uintptr:
return a.Uint() < b.Uint()
case reflect.Array:
// Compare the contents of both arrays.
l := a.Len()
for i := 0; i < l; i++ {
av := a.Index(i)
bv := b.Index(i)
if av.Interface() == bv.Interface() {
continue
}
return valueSortLess(av, bv)
}
}
return a.String() < b.String()
}
// Less returns whether the value at index i should sort before the
// value at index j. It is part of the sort.Interface implementation.
func (s *valuesSorter) Less(i, j int) bool {
if s.strings == nil {
return valueSortLess(s.values[i], s.values[j])
}
return s.strings[i] < s.strings[j]
}
// sortValues is a sort function that handles both native types and any type that
// can be converted to error or Stringer. Other inputs are sorted according to
// their Value.String() value to ensure display stability.
func sortValues(values []reflect.Value, cs *ConfigState) {
if len(values) == 0 {
return
}
sort.Sort(newValuesSorter(values, cs))
}

View File

@ -1,306 +0,0 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"fmt"
"io"
"os"
)
// ConfigState houses the configuration options used by spew to format and
// display values. There is a global instance, Config, that is used to control
// all top-level Formatter and Dump functionality. Each ConfigState instance
// provides methods equivalent to the top-level functions.
//
// The zero value for ConfigState provides no indentation. You would typically
// want to set it to a space or a tab.
//
// Alternatively, you can use NewDefaultConfig to get a ConfigState instance
// with default settings. See the documentation of NewDefaultConfig for default
// values.
type ConfigState struct {
// Indent specifies the string to use for each indentation level. The
// global config instance that all top-level functions use set this to a
// single space by default. If you would like more indentation, you might
// set this to a tab with "\t" or perhaps two spaces with " ".
Indent string
// MaxDepth controls the maximum number of levels to descend into nested
// data structures. The default, 0, means there is no limit.
//
// NOTE: Circular data structures are properly detected, so it is not
// necessary to set this value unless you specifically want to limit deeply
// nested data structures.
MaxDepth int
// DisableMethods specifies whether or not error and Stringer interfaces are
// invoked for types that implement them.
DisableMethods bool
// DisablePointerMethods specifies whether or not to check for and invoke
// error and Stringer interfaces on types which only accept a pointer
// receiver when the current type is not a pointer.
//
// NOTE: This might be an unsafe action since calling one of these methods
// with a pointer receiver could technically mutate the value, however,
// in practice, types which choose to satisify an error or Stringer
// interface with a pointer receiver should not be mutating their state
// inside these interface methods. As a result, this option relies on
// access to the unsafe package, so it will not have any effect when
// running in environments without access to the unsafe package such as
// Google App Engine or with the "safe" build tag specified.
DisablePointerMethods bool
// DisablePointerAddresses specifies whether to disable the printing of
// pointer addresses. This is useful when diffing data structures in tests.
DisablePointerAddresses bool
// DisableCapacities specifies whether to disable the printing of capacities
// for arrays, slices, maps and channels. This is useful when diffing
// data structures in tests.
DisableCapacities bool
// ContinueOnMethod specifies whether or not recursion should continue once
// a custom error or Stringer interface is invoked. The default, false,
// means it will print the results of invoking the custom error or Stringer
// interface and return immediately instead of continuing to recurse into
// the internals of the data type.
//
// NOTE: This flag does not have any effect if method invocation is disabled
// via the DisableMethods or DisablePointerMethods options.
ContinueOnMethod bool
// SortKeys specifies map keys should be sorted before being printed. Use
// this to have a more deterministic, diffable output. Note that only
// native types (bool, int, uint, floats, uintptr and string) and types
// that support the error or Stringer interfaces (if methods are
// enabled) are supported, with other types sorted according to the
// reflect.Value.String() output which guarantees display stability.
SortKeys bool
// SpewKeys specifies that, as a last resort attempt, map keys should
// be spewed to strings and sorted by those strings. This is only
// considered if SortKeys is true.
SpewKeys bool
}
// Config is the active configuration of the top-level functions.
// The configuration can be changed by modifying the contents of spew.Config.
var Config = ConfigState{Indent: " "}
// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the formatted string as a value that satisfies error. See NewFormatter
// for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) {
return fmt.Errorf(format, c.convertArgs(a)...)
}
// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprint(w, c.convertArgs(a)...)
}
// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(w, format, c.convertArgs(a)...)
}
// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it
// passed with a Formatter interface returned by c.NewFormatter. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprintln(w, c.convertArgs(a)...)
}
// Print is a wrapper for fmt.Print that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Print(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Print(a ...interface{}) (n int, err error) {
return fmt.Print(c.convertArgs(a)...)
}
// Printf is a wrapper for fmt.Printf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Printf(format, c.convertArgs(a)...)
}
// Println is a wrapper for fmt.Println that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Println(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Println(a ...interface{}) (n int, err error) {
return fmt.Println(c.convertArgs(a)...)
}
// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Sprint(a ...interface{}) string {
return fmt.Sprint(c.convertArgs(a)...)
}
// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Sprintf(format string, a ...interface{}) string {
return fmt.Sprintf(format, c.convertArgs(a)...)
}
// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it
// were passed with a Formatter interface returned by c.NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Sprintln(a ...interface{}) string {
return fmt.Sprintln(c.convertArgs(a)...)
}
/*
NewFormatter returns a custom formatter that satisfies the fmt.Formatter
interface. As a result, it integrates cleanly with standard fmt package
printing functions. The formatter is useful for inline printing of smaller data
types similar to the standard %v format specifier.
The custom formatter only responds to the %v (most compact), %+v (adds pointer
addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb
combinations. Any other verbs such as %x and %q will be sent to the the
standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
Typically this function shouldn't be called directly. It is much easier to make
use of the custom formatter by calling one of the convenience functions such as
c.Printf, c.Println, or c.Printf.
*/
func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter {
return newFormatter(c, v)
}
// Fdump formats and displays the passed arguments to io.Writer w. It formats
// exactly the same as Dump.
func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) {
fdump(c, w, a...)
}
/*
Dump displays the passed parameters to standard out with newlines, customizable
indentation, and additional debug information such as complete types and all
pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by modifying the public members
of c. See ConfigState for options documentation.
See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to
get the formatted result as a string.
*/
func (c *ConfigState) Dump(a ...interface{}) {
fdump(c, os.Stdout, a...)
}
// Sdump returns a string with the passed arguments formatted exactly the same
// as Dump.
func (c *ConfigState) Sdump(a ...interface{}) string {
var buf bytes.Buffer
fdump(c, &buf, a...)
return buf.String()
}
// convertArgs accepts a slice of arguments and returns a slice of the same
// length with each argument converted to a spew Formatter interface using
// the ConfigState associated with s.
func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) {
formatters = make([]interface{}, len(args))
for index, arg := range args {
formatters[index] = newFormatter(c, arg)
}
return formatters
}
// NewDefaultConfig returns a ConfigState with the following default settings.
//
// Indent: " "
// MaxDepth: 0
// DisableMethods: false
// DisablePointerMethods: false
// ContinueOnMethod: false
// SortKeys: false
func NewDefaultConfig() *ConfigState {
return &ConfigState{Indent: " "}
}

View File

@ -1,211 +0,0 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
Package spew implements a deep pretty printer for Go data structures to aid in
debugging.
A quick overview of the additional features spew provides over the built-in
printing facilities for Go data types are as follows:
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output (only when using
Dump style)
There are two different approaches spew allows for dumping Go data structures:
* Dump style which prints with newlines, customizable indentation,
and additional debug information such as types and all pointer addresses
used to indirect to the final value
* A custom Formatter interface that integrates cleanly with the standard fmt
package and replaces %v, %+v, %#v, and %#+v to provide inline printing
similar to the default %v while providing the additional functionality
outlined above and passing unsupported format verbs such as %x and %q
along to fmt
Quick Start
This section demonstrates how to quickly get started with spew. See the
sections below for further details on formatting and configuration options.
To dump a variable with full newlines, indentation, type, and pointer
information use Dump, Fdump, or Sdump:
spew.Dump(myVar1, myVar2, ...)
spew.Fdump(someWriter, myVar1, myVar2, ...)
str := spew.Sdump(myVar1, myVar2, ...)
Alternatively, if you would prefer to use format strings with a compacted inline
printing style, use the convenience wrappers Printf, Fprintf, etc with
%v (most compact), %+v (adds pointer addresses), %#v (adds types), or
%#+v (adds types and pointer addresses):
spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
Configuration Options
Configuration of spew is handled by fields in the ConfigState type. For
convenience, all of the top-level functions use a global state available
via the spew.Config global.
It is also possible to create a ConfigState instance that provides methods
equivalent to the top-level functions. This allows concurrent configuration
options. See the ConfigState documentation for more details.
The following configuration options are available:
* Indent
String to use for each indentation level for Dump functions.
It is a single space by default. A popular alternative is "\t".
* MaxDepth
Maximum number of levels to descend into nested data structures.
There is no limit by default.
* DisableMethods
Disables invocation of error and Stringer interface methods.
Method invocation is enabled by default.
* DisablePointerMethods
Disables invocation of error and Stringer interface methods on types
which only accept pointer receivers from non-pointer variables.
Pointer method invocation is enabled by default.
* DisablePointerAddresses
DisablePointerAddresses specifies whether to disable the printing of
pointer addresses. This is useful when diffing data structures in tests.
* DisableCapacities
DisableCapacities specifies whether to disable the printing of
capacities for arrays, slices, maps and channels. This is useful when
diffing data structures in tests.
* ContinueOnMethod
Enables recursion into types after invoking error and Stringer interface
methods. Recursion after method invocation is disabled by default.
* SortKeys
Specifies map keys should be sorted before being printed. Use
this to have a more deterministic, diffable output. Note that
only native types (bool, int, uint, floats, uintptr and string)
and types which implement error or Stringer interfaces are
supported with other types sorted according to the
reflect.Value.String() output which guarantees display
stability. Natural map order is used by default.
* SpewKeys
Specifies that, as a last resort attempt, map keys should be
spewed to strings and sorted by those strings. This is only
considered if SortKeys is true.
Dump Usage
Simply call spew.Dump with a list of variables you want to dump:
spew.Dump(myVar1, myVar2, ...)
You may also call spew.Fdump if you would prefer to output to an arbitrary
io.Writer. For example, to dump to standard error:
spew.Fdump(os.Stderr, myVar1, myVar2, ...)
A third option is to call spew.Sdump to get the formatted output as a string:
str := spew.Sdump(myVar1, myVar2, ...)
Sample Dump Output
See the Dump example for details on the setup of the types and variables being
shown here.
(main.Foo) {
unexportedField: (*main.Bar)(0xf84002e210)({
flag: (main.Flag) flagTwo,
data: (uintptr) <nil>
}),
ExportedField: (map[interface {}]interface {}) (len=1) {
(string) (len=3) "one": (bool) true
}
}
Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C
command as shown.
([]uint8) (len=32 cap=32) {
00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... |
00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0|
00000020 31 32 |12|
}
Custom Formatter
Spew provides a custom formatter that implements the fmt.Formatter interface
so that it integrates cleanly with standard fmt package printing functions. The
formatter is useful for inline printing of smaller data types similar to the
standard %v format specifier.
The custom formatter only responds to the %v (most compact), %+v (adds pointer
addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb
combinations. Any other verbs such as %x and %q will be sent to the the
standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
Custom Formatter Usage
The simplest way to make use of the spew custom formatter is to call one of the
convenience functions such as spew.Printf, spew.Println, or spew.Printf. The
functions have syntax you are most likely already familiar with:
spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
spew.Println(myVar, myVar2)
spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
See the Index for the full list convenience functions.
Sample Formatter Output
Double pointer to a uint8:
%v: <**>5
%+v: <**>(0xf8400420d0->0xf8400420c8)5
%#v: (**uint8)5
%#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5
Pointer to circular struct with a uint8 field and a pointer to itself:
%v: <*>{1 <*><shown>}
%+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)<shown>}
%#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)<shown>}
%#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)<shown>}
See the Printf example for details on the setup of variables being shown
here.
Errors
Since it is possible for custom Stringer/error interfaces to panic, spew
detects them and handles them internally by printing the panic information
inline with the output. Since spew is intended to provide deep pretty printing
capabilities on structures, it intentionally does not return any errors.
*/
package spew

View File

@ -1,509 +0,0 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"os"
"reflect"
"regexp"
"strconv"
"strings"
)
var (
// uint8Type is a reflect.Type representing a uint8. It is used to
// convert cgo types to uint8 slices for hexdumping.
uint8Type = reflect.TypeOf(uint8(0))
// cCharRE is a regular expression that matches a cgo char.
// It is used to detect character arrays to hexdump them.
cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`)
// cUnsignedCharRE is a regular expression that matches a cgo unsigned
// char. It is used to detect unsigned character arrays to hexdump
// them.
cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`)
// cUint8tCharRE is a regular expression that matches a cgo uint8_t.
// It is used to detect uint8_t arrays to hexdump them.
cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`)
)
// dumpState contains information about the state of a dump operation.
type dumpState struct {
w io.Writer
depth int
pointers map[uintptr]int
ignoreNextType bool
ignoreNextIndent bool
cs *ConfigState
}
// indent performs indentation according to the depth level and cs.Indent
// option.
func (d *dumpState) indent() {
if d.ignoreNextIndent {
d.ignoreNextIndent = false
return
}
d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth))
}
// unpackValue returns values inside of non-nil interfaces when possible.
// This is useful for data types like structs, arrays, slices, and maps which
// can contain varying types packed inside an interface.
func (d *dumpState) unpackValue(v reflect.Value) reflect.Value {
if v.Kind() == reflect.Interface && !v.IsNil() {
v = v.Elem()
}
return v
}
// dumpPtr handles formatting of pointers by indirecting them as necessary.
func (d *dumpState) dumpPtr(v reflect.Value) {
// Remove pointers at or below the current depth from map used to detect
// circular refs.
for k, depth := range d.pointers {
if depth >= d.depth {
delete(d.pointers, k)
}
}
// Keep list of all dereferenced pointers to show later.
pointerChain := make([]uintptr, 0)
// Figure out how many levels of indirection there are by dereferencing
// pointers and unpacking interfaces down the chain while detecting circular
// references.
nilFound := false
cycleFound := false
indirects := 0
ve := v
for ve.Kind() == reflect.Ptr {
if ve.IsNil() {
nilFound = true
break
}
indirects++
addr := ve.Pointer()
pointerChain = append(pointerChain, addr)
if pd, ok := d.pointers[addr]; ok && pd < d.depth {
cycleFound = true
indirects--
break
}
d.pointers[addr] = d.depth
ve = ve.Elem()
if ve.Kind() == reflect.Interface {
if ve.IsNil() {
nilFound = true
break
}
ve = ve.Elem()
}
}
// Display type information.
d.w.Write(openParenBytes)
d.w.Write(bytes.Repeat(asteriskBytes, indirects))
d.w.Write([]byte(ve.Type().String()))
d.w.Write(closeParenBytes)
// Display pointer information.
if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 {
d.w.Write(openParenBytes)
for i, addr := range pointerChain {
if i > 0 {
d.w.Write(pointerChainBytes)
}
printHexPtr(d.w, addr)
}
d.w.Write(closeParenBytes)
}
// Display dereferenced value.
d.w.Write(openParenBytes)
switch {
case nilFound:
d.w.Write(nilAngleBytes)
case cycleFound:
d.w.Write(circularBytes)
default:
d.ignoreNextType = true
d.dump(ve)
}
d.w.Write(closeParenBytes)
}
// dumpSlice handles formatting of arrays and slices. Byte (uint8 under
// reflection) arrays and slices are dumped in hexdump -C fashion.
func (d *dumpState) dumpSlice(v reflect.Value) {
// Determine whether this type should be hex dumped or not. Also,
// for types which should be hexdumped, try to use the underlying data
// first, then fall back to trying to convert them to a uint8 slice.
var buf []uint8
doConvert := false
doHexDump := false
numEntries := v.Len()
if numEntries > 0 {
vt := v.Index(0).Type()
vts := vt.String()
switch {
// C types that need to be converted.
case cCharRE.MatchString(vts):
fallthrough
case cUnsignedCharRE.MatchString(vts):
fallthrough
case cUint8tCharRE.MatchString(vts):
doConvert = true
// Try to use existing uint8 slices and fall back to converting
// and copying if that fails.
case vt.Kind() == reflect.Uint8:
// We need an addressable interface to convert the type
// to a byte slice. However, the reflect package won't
// give us an interface on certain things like
// unexported struct fields in order to enforce
// visibility rules. We use unsafe, when available, to
// bypass these restrictions since this package does not
// mutate the values.
vs := v
if !vs.CanInterface() || !vs.CanAddr() {
vs = unsafeReflectValue(vs)
}
if !UnsafeDisabled {
vs = vs.Slice(0, numEntries)
// Use the existing uint8 slice if it can be
// type asserted.
iface := vs.Interface()
if slice, ok := iface.([]uint8); ok {
buf = slice
doHexDump = true
break
}
}
// The underlying data needs to be converted if it can't
// be type asserted to a uint8 slice.
doConvert = true
}
// Copy and convert the underlying type if needed.
if doConvert && vt.ConvertibleTo(uint8Type) {
// Convert and copy each element into a uint8 byte
// slice.
buf = make([]uint8, numEntries)
for i := 0; i < numEntries; i++ {
vv := v.Index(i)
buf[i] = uint8(vv.Convert(uint8Type).Uint())
}
doHexDump = true
}
}
// Hexdump the entire slice as needed.
if doHexDump {
indent := strings.Repeat(d.cs.Indent, d.depth)
str := indent + hex.Dump(buf)
str = strings.Replace(str, "\n", "\n"+indent, -1)
str = strings.TrimRight(str, d.cs.Indent)
d.w.Write([]byte(str))
return
}
// Recursively call dump for each item.
for i := 0; i < numEntries; i++ {
d.dump(d.unpackValue(v.Index(i)))
if i < (numEntries - 1) {
d.w.Write(commaNewlineBytes)
} else {
d.w.Write(newlineBytes)
}
}
}
// dump is the main workhorse for dumping a value. It uses the passed reflect
// value to figure out what kind of object we are dealing with and formats it
// appropriately. It is a recursive function, however circular data structures
// are detected and handled properly.
func (d *dumpState) dump(v reflect.Value) {
// Handle invalid reflect values immediately.
kind := v.Kind()
if kind == reflect.Invalid {
d.w.Write(invalidAngleBytes)
return
}
// Handle pointers specially.
if kind == reflect.Ptr {
d.indent()
d.dumpPtr(v)
return
}
// Print type information unless already handled elsewhere.
if !d.ignoreNextType {
d.indent()
d.w.Write(openParenBytes)
d.w.Write([]byte(v.Type().String()))
d.w.Write(closeParenBytes)
d.w.Write(spaceBytes)
}
d.ignoreNextType = false
// Display length and capacity if the built-in len and cap functions
// work with the value's kind and the len/cap itself is non-zero.
valueLen, valueCap := 0, 0
switch v.Kind() {
case reflect.Array, reflect.Slice, reflect.Chan:
valueLen, valueCap = v.Len(), v.Cap()
case reflect.Map, reflect.String:
valueLen = v.Len()
}
if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 {
d.w.Write(openParenBytes)
if valueLen != 0 {
d.w.Write(lenEqualsBytes)
printInt(d.w, int64(valueLen), 10)
}
if !d.cs.DisableCapacities && valueCap != 0 {
if valueLen != 0 {
d.w.Write(spaceBytes)
}
d.w.Write(capEqualsBytes)
printInt(d.w, int64(valueCap), 10)
}
d.w.Write(closeParenBytes)
d.w.Write(spaceBytes)
}
// Call Stringer/error interfaces if they exist and the handle methods flag
// is enabled
if !d.cs.DisableMethods {
if (kind != reflect.Invalid) && (kind != reflect.Interface) {
if handled := handleMethods(d.cs, d.w, v); handled {
return
}
}
}
switch kind {
case reflect.Invalid:
// Do nothing. We should never get here since invalid has already
// been handled above.
case reflect.Bool:
printBool(d.w, v.Bool())
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
printInt(d.w, v.Int(), 10)
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
printUint(d.w, v.Uint(), 10)
case reflect.Float32:
printFloat(d.w, v.Float(), 32)
case reflect.Float64:
printFloat(d.w, v.Float(), 64)
case reflect.Complex64:
printComplex(d.w, v.Complex(), 32)
case reflect.Complex128:
printComplex(d.w, v.Complex(), 64)
case reflect.Slice:
if v.IsNil() {
d.w.Write(nilAngleBytes)
break
}
fallthrough
case reflect.Array:
d.w.Write(openBraceNewlineBytes)
d.depth++
if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
d.indent()
d.w.Write(maxNewlineBytes)
} else {
d.dumpSlice(v)
}
d.depth--
d.indent()
d.w.Write(closeBraceBytes)
case reflect.String:
d.w.Write([]byte(strconv.Quote(v.String())))
case reflect.Interface:
// The only time we should get here is for nil interfaces due to
// unpackValue calls.
if v.IsNil() {
d.w.Write(nilAngleBytes)
}
case reflect.Ptr:
// Do nothing. We should never get here since pointers have already
// been handled above.
case reflect.Map:
// nil maps should be indicated as different than empty maps
if v.IsNil() {
d.w.Write(nilAngleBytes)
break
}
d.w.Write(openBraceNewlineBytes)
d.depth++
if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
d.indent()
d.w.Write(maxNewlineBytes)
} else {
numEntries := v.Len()
keys := v.MapKeys()
if d.cs.SortKeys {
sortValues(keys, d.cs)
}
for i, key := range keys {
d.dump(d.unpackValue(key))
d.w.Write(colonSpaceBytes)
d.ignoreNextIndent = true
d.dump(d.unpackValue(v.MapIndex(key)))
if i < (numEntries - 1) {
d.w.Write(commaNewlineBytes)
} else {
d.w.Write(newlineBytes)
}
}
}
d.depth--
d.indent()
d.w.Write(closeBraceBytes)
case reflect.Struct:
d.w.Write(openBraceNewlineBytes)
d.depth++
if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
d.indent()
d.w.Write(maxNewlineBytes)
} else {
vt := v.Type()
numFields := v.NumField()
for i := 0; i < numFields; i++ {
d.indent()
vtf := vt.Field(i)
d.w.Write([]byte(vtf.Name))
d.w.Write(colonSpaceBytes)
d.ignoreNextIndent = true
d.dump(d.unpackValue(v.Field(i)))
if i < (numFields - 1) {
d.w.Write(commaNewlineBytes)
} else {
d.w.Write(newlineBytes)
}
}
}
d.depth--
d.indent()
d.w.Write(closeBraceBytes)
case reflect.Uintptr:
printHexPtr(d.w, uintptr(v.Uint()))
case reflect.UnsafePointer, reflect.Chan, reflect.Func:
printHexPtr(d.w, v.Pointer())
// There were not any other types at the time this code was written, but
// fall back to letting the default fmt package handle it in case any new
// types are added.
default:
if v.CanInterface() {
fmt.Fprintf(d.w, "%v", v.Interface())
} else {
fmt.Fprintf(d.w, "%v", v.String())
}
}
}
// fdump is a helper function to consolidate the logic from the various public
// methods which take varying writers and config states.
func fdump(cs *ConfigState, w io.Writer, a ...interface{}) {
for _, arg := range a {
if arg == nil {
w.Write(interfaceBytes)
w.Write(spaceBytes)
w.Write(nilAngleBytes)
w.Write(newlineBytes)
continue
}
d := dumpState{w: w, cs: cs}
d.pointers = make(map[uintptr]int)
d.dump(reflect.ValueOf(arg))
d.w.Write(newlineBytes)
}
}
// Fdump formats and displays the passed arguments to io.Writer w. It formats
// exactly the same as Dump.
func Fdump(w io.Writer, a ...interface{}) {
fdump(&Config, w, a...)
}
// Sdump returns a string with the passed arguments formatted exactly the same
// as Dump.
func Sdump(a ...interface{}) string {
var buf bytes.Buffer
fdump(&Config, &buf, a...)
return buf.String()
}
/*
Dump displays the passed parameters to standard out with newlines, customizable
indentation, and additional debug information such as complete types and all
pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by an exported package global,
spew.Config. See ConfigState for options documentation.
See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to
get the formatted result as a string.
*/
func Dump(a ...interface{}) {
fdump(&Config, os.Stdout, a...)
}

View File

@ -1,419 +0,0 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"fmt"
"reflect"
"strconv"
"strings"
)
// supportedFlags is a list of all the character flags supported by fmt package.
const supportedFlags = "0-+# "
// formatState implements the fmt.Formatter interface and contains information
// about the state of a formatting operation. The NewFormatter function can
// be used to get a new Formatter which can be used directly as arguments
// in standard fmt package printing calls.
type formatState struct {
value interface{}
fs fmt.State
depth int
pointers map[uintptr]int
ignoreNextType bool
cs *ConfigState
}
// buildDefaultFormat recreates the original format string without precision
// and width information to pass in to fmt.Sprintf in the case of an
// unrecognized type. Unless new types are added to the language, this
// function won't ever be called.
func (f *formatState) buildDefaultFormat() (format string) {
buf := bytes.NewBuffer(percentBytes)
for _, flag := range supportedFlags {
if f.fs.Flag(int(flag)) {
buf.WriteRune(flag)
}
}
buf.WriteRune('v')
format = buf.String()
return format
}
// constructOrigFormat recreates the original format string including precision
// and width information to pass along to the standard fmt package. This allows
// automatic deferral of all format strings this package doesn't support.
func (f *formatState) constructOrigFormat(verb rune) (format string) {
buf := bytes.NewBuffer(percentBytes)
for _, flag := range supportedFlags {
if f.fs.Flag(int(flag)) {
buf.WriteRune(flag)
}
}
if width, ok := f.fs.Width(); ok {
buf.WriteString(strconv.Itoa(width))
}
if precision, ok := f.fs.Precision(); ok {
buf.Write(precisionBytes)
buf.WriteString(strconv.Itoa(precision))
}
buf.WriteRune(verb)
format = buf.String()
return format
}
// unpackValue returns values inside of non-nil interfaces when possible and
// ensures that types for values which have been unpacked from an interface
// are displayed when the show types flag is also set.
// This is useful for data types like structs, arrays, slices, and maps which
// can contain varying types packed inside an interface.
func (f *formatState) unpackValue(v reflect.Value) reflect.Value {
if v.Kind() == reflect.Interface {
f.ignoreNextType = false
if !v.IsNil() {
v = v.Elem()
}
}
return v
}
// formatPtr handles formatting of pointers by indirecting them as necessary.
func (f *formatState) formatPtr(v reflect.Value) {
// Display nil if top level pointer is nil.
showTypes := f.fs.Flag('#')
if v.IsNil() && (!showTypes || f.ignoreNextType) {
f.fs.Write(nilAngleBytes)
return
}
// Remove pointers at or below the current depth from map used to detect
// circular refs.
for k, depth := range f.pointers {
if depth >= f.depth {
delete(f.pointers, k)
}
}
// Keep list of all dereferenced pointers to possibly show later.
pointerChain := make([]uintptr, 0)
// Figure out how many levels of indirection there are by derferencing
// pointers and unpacking interfaces down the chain while detecting circular
// references.
nilFound := false
cycleFound := false
indirects := 0
ve := v
for ve.Kind() == reflect.Ptr {
if ve.IsNil() {
nilFound = true
break
}
indirects++
addr := ve.Pointer()
pointerChain = append(pointerChain, addr)
if pd, ok := f.pointers[addr]; ok && pd < f.depth {
cycleFound = true
indirects--
break
}
f.pointers[addr] = f.depth
ve = ve.Elem()
if ve.Kind() == reflect.Interface {
if ve.IsNil() {
nilFound = true
break
}
ve = ve.Elem()
}
}
// Display type or indirection level depending on flags.
if showTypes && !f.ignoreNextType {
f.fs.Write(openParenBytes)
f.fs.Write(bytes.Repeat(asteriskBytes, indirects))
f.fs.Write([]byte(ve.Type().String()))
f.fs.Write(closeParenBytes)
} else {
if nilFound || cycleFound {
indirects += strings.Count(ve.Type().String(), "*")
}
f.fs.Write(openAngleBytes)
f.fs.Write([]byte(strings.Repeat("*", indirects)))
f.fs.Write(closeAngleBytes)
}
// Display pointer information depending on flags.
if f.fs.Flag('+') && (len(pointerChain) > 0) {
f.fs.Write(openParenBytes)
for i, addr := range pointerChain {
if i > 0 {
f.fs.Write(pointerChainBytes)
}
printHexPtr(f.fs, addr)
}
f.fs.Write(closeParenBytes)
}
// Display dereferenced value.
switch {
case nilFound:
f.fs.Write(nilAngleBytes)
case cycleFound:
f.fs.Write(circularShortBytes)
default:
f.ignoreNextType = true
f.format(ve)
}
}
// format is the main workhorse for providing the Formatter interface. It
// uses the passed reflect value to figure out what kind of object we are
// dealing with and formats it appropriately. It is a recursive function,
// however circular data structures are detected and handled properly.
func (f *formatState) format(v reflect.Value) {
// Handle invalid reflect values immediately.
kind := v.Kind()
if kind == reflect.Invalid {
f.fs.Write(invalidAngleBytes)
return
}
// Handle pointers specially.
if kind == reflect.Ptr {
f.formatPtr(v)
return
}
// Print type information unless already handled elsewhere.
if !f.ignoreNextType && f.fs.Flag('#') {
f.fs.Write(openParenBytes)
f.fs.Write([]byte(v.Type().String()))
f.fs.Write(closeParenBytes)
}
f.ignoreNextType = false
// Call Stringer/error interfaces if they exist and the handle methods
// flag is enabled.
if !f.cs.DisableMethods {
if (kind != reflect.Invalid) && (kind != reflect.Interface) {
if handled := handleMethods(f.cs, f.fs, v); handled {
return
}
}
}
switch kind {
case reflect.Invalid:
// Do nothing. We should never get here since invalid has already
// been handled above.
case reflect.Bool:
printBool(f.fs, v.Bool())
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
printInt(f.fs, v.Int(), 10)
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
printUint(f.fs, v.Uint(), 10)
case reflect.Float32:
printFloat(f.fs, v.Float(), 32)
case reflect.Float64:
printFloat(f.fs, v.Float(), 64)
case reflect.Complex64:
printComplex(f.fs, v.Complex(), 32)
case reflect.Complex128:
printComplex(f.fs, v.Complex(), 64)
case reflect.Slice:
if v.IsNil() {
f.fs.Write(nilAngleBytes)
break
}
fallthrough
case reflect.Array:
f.fs.Write(openBracketBytes)
f.depth++
if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
f.fs.Write(maxShortBytes)
} else {
numEntries := v.Len()
for i := 0; i < numEntries; i++ {
if i > 0 {
f.fs.Write(spaceBytes)
}
f.ignoreNextType = true
f.format(f.unpackValue(v.Index(i)))
}
}
f.depth--
f.fs.Write(closeBracketBytes)
case reflect.String:
f.fs.Write([]byte(v.String()))
case reflect.Interface:
// The only time we should get here is for nil interfaces due to
// unpackValue calls.
if v.IsNil() {
f.fs.Write(nilAngleBytes)
}
case reflect.Ptr:
// Do nothing. We should never get here since pointers have already
// been handled above.
case reflect.Map:
// nil maps should be indicated as different than empty maps
if v.IsNil() {
f.fs.Write(nilAngleBytes)
break
}
f.fs.Write(openMapBytes)
f.depth++
if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
f.fs.Write(maxShortBytes)
} else {
keys := v.MapKeys()
if f.cs.SortKeys {
sortValues(keys, f.cs)
}
for i, key := range keys {
if i > 0 {
f.fs.Write(spaceBytes)
}
f.ignoreNextType = true
f.format(f.unpackValue(key))
f.fs.Write(colonBytes)
f.ignoreNextType = true
f.format(f.unpackValue(v.MapIndex(key)))
}
}
f.depth--
f.fs.Write(closeMapBytes)
case reflect.Struct:
numFields := v.NumField()
f.fs.Write(openBraceBytes)
f.depth++
if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
f.fs.Write(maxShortBytes)
} else {
vt := v.Type()
for i := 0; i < numFields; i++ {
if i > 0 {
f.fs.Write(spaceBytes)
}
vtf := vt.Field(i)
if f.fs.Flag('+') || f.fs.Flag('#') {
f.fs.Write([]byte(vtf.Name))
f.fs.Write(colonBytes)
}
f.format(f.unpackValue(v.Field(i)))
}
}
f.depth--
f.fs.Write(closeBraceBytes)
case reflect.Uintptr:
printHexPtr(f.fs, uintptr(v.Uint()))
case reflect.UnsafePointer, reflect.Chan, reflect.Func:
printHexPtr(f.fs, v.Pointer())
// There were not any other types at the time this code was written, but
// fall back to letting the default fmt package handle it if any get added.
default:
format := f.buildDefaultFormat()
if v.CanInterface() {
fmt.Fprintf(f.fs, format, v.Interface())
} else {
fmt.Fprintf(f.fs, format, v.String())
}
}
}
// Format satisfies the fmt.Formatter interface. See NewFormatter for usage
// details.
func (f *formatState) Format(fs fmt.State, verb rune) {
f.fs = fs
// Use standard formatting for verbs that are not v.
if verb != 'v' {
format := f.constructOrigFormat(verb)
fmt.Fprintf(fs, format, f.value)
return
}
if f.value == nil {
if fs.Flag('#') {
fs.Write(interfaceBytes)
}
fs.Write(nilAngleBytes)
return
}
f.format(reflect.ValueOf(f.value))
}
// newFormatter is a helper function to consolidate the logic from the various
// public methods which take varying config states.
func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter {
fs := &formatState{value: v, cs: cs}
fs.pointers = make(map[uintptr]int)
return fs
}
/*
NewFormatter returns a custom formatter that satisfies the fmt.Formatter
interface. As a result, it integrates cleanly with standard fmt package
printing functions. The formatter is useful for inline printing of smaller data
types similar to the standard %v format specifier.
The custom formatter only responds to the %v (most compact), %+v (adds pointer
addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb
combinations. Any other verbs such as %x and %q will be sent to the the
standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
Typically this function shouldn't be called directly. It is much easier to make
use of the custom formatter by calling one of the convenience functions such as
Printf, Println, or Fprintf.
*/
func NewFormatter(v interface{}) fmt.Formatter {
return newFormatter(&Config, v)
}

View File

@ -1,148 +0,0 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"fmt"
"io"
)
// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the formatted string as a value that satisfies error. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b))
func Errorf(format string, a ...interface{}) (err error) {
return fmt.Errorf(format, convertArgs(a)...)
}
// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b))
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprint(w, convertArgs(a)...)
}
// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b))
func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(w, format, convertArgs(a)...)
}
// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it
// passed with a default Formatter interface returned by NewFormatter. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b))
func Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprintln(w, convertArgs(a)...)
}
// Print is a wrapper for fmt.Print that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b))
func Print(a ...interface{}) (n int, err error) {
return fmt.Print(convertArgs(a)...)
}
// Printf is a wrapper for fmt.Printf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b))
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Printf(format, convertArgs(a)...)
}
// Println is a wrapper for fmt.Println that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b))
func Println(a ...interface{}) (n int, err error) {
return fmt.Println(convertArgs(a)...)
}
// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b))
func Sprint(a ...interface{}) string {
return fmt.Sprint(convertArgs(a)...)
}
// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b))
func Sprintf(format string, a ...interface{}) string {
return fmt.Sprintf(format, convertArgs(a)...)
}
// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it
// were passed with a default Formatter interface returned by NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b))
func Sprintln(a ...interface{}) string {
return fmt.Sprintln(convertArgs(a)...)
}
// convertArgs accepts a slice of arguments and returns a slice of the same
// length with each argument converted to a default spew Formatter interface.
func convertArgs(args []interface{}) (formatters []interface{}) {
formatters = make([]interface{}, len(args))
for index, arg := range args {
formatters[index] = NewFormatter(arg)
}
return formatters
}

View File

@ -1,9 +0,0 @@
.DS_Store
.DS_Store?
._*
.Spotlight-V100
.Trashes
Icon?
ehthumbs.db
Thumbs.db
.idea

View File

@ -1,107 +0,0 @@
sudo: false
language: go
go:
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- master
before_install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
before_script:
- echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB" | sudo tee -a /etc/mysql/my.cnf
- sudo service mysql restart
- .travis/wait_mysql.sh
- mysql -e 'create database gotest;'
matrix:
include:
- env: DB=MYSQL8
sudo: required
dist: trusty
go: 1.10.x
services:
- docker
before_install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- docker pull mysql:8.0
- docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret
mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1
- cp .travis/docker.cnf ~/.my.cnf
- .travis/wait_mysql.sh
before_script:
- export MYSQL_TEST_USER=gotest
- export MYSQL_TEST_PASS=secret
- export MYSQL_TEST_ADDR=127.0.0.1:3307
- export MYSQL_TEST_CONCURRENT=1
- env: DB=MYSQL57
sudo: required
dist: trusty
go: 1.10.x
services:
- docker
before_install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- docker pull mysql:5.7
- docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret
mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1
- cp .travis/docker.cnf ~/.my.cnf
- .travis/wait_mysql.sh
before_script:
- export MYSQL_TEST_USER=gotest
- export MYSQL_TEST_PASS=secret
- export MYSQL_TEST_ADDR=127.0.0.1:3307
- export MYSQL_TEST_CONCURRENT=1
- env: DB=MARIA55
sudo: required
dist: trusty
go: 1.10.x
services:
- docker
before_install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- docker pull mariadb:5.5
- docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret
mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1
- cp .travis/docker.cnf ~/.my.cnf
- .travis/wait_mysql.sh
before_script:
- export MYSQL_TEST_USER=gotest
- export MYSQL_TEST_PASS=secret
- export MYSQL_TEST_ADDR=127.0.0.1:3307
- export MYSQL_TEST_CONCURRENT=1
- env: DB=MARIA10_1
sudo: required
dist: trusty
go: 1.10.x
services:
- docker
before_install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- docker pull mariadb:10.1
- docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret
mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1
- cp .travis/docker.cnf ~/.my.cnf
- .travis/wait_mysql.sh
before_script:
- export MYSQL_TEST_USER=gotest
- export MYSQL_TEST_PASS=secret
- export MYSQL_TEST_ADDR=127.0.0.1:3307
- export MYSQL_TEST_CONCURRENT=1
script:
- go test -v -covermode=count -coverprofile=coverage.out
- go vet ./...
- .travis/gofmt.sh
after_script:
- $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci

View File

@ -1,89 +0,0 @@
# This is the official list of Go-MySQL-Driver authors for copyright purposes.
# If you are submitting a patch, please add your name or the name of the
# organization which holds the copyright to this list in alphabetical order.
# Names should be added to this file as
# Name <email address>
# The email address is not required for organizations.
# Please keep the list sorted.
# Individual Persons
Aaron Hopkins <go-sql-driver at die.net>
Achille Roussel <achille.roussel at gmail.com>
Alexey Palazhchenko <alexey.palazhchenko at gmail.com>
Andrew Reid <andrew.reid at tixtrack.com>
Arne Hormann <arnehormann at gmail.com>
Asta Xie <xiemengjun at gmail.com>
Bulat Gaifullin <gaifullinbf at gmail.com>
Carlos Nieto <jose.carlos at menteslibres.net>
Chris Moos <chris at tech9computers.com>
Craig Wilson <craiggwilson at gmail.com>
Daniel Montoya <dsmontoyam at gmail.com>
Daniel Nichter <nil at codenode.com>
Daniël van Eeden <git at myname.nl>
Dave Protasowski <dprotaso at gmail.com>
DisposaBoy <disposaboy at dby.me>
Egor Smolyakov <egorsmkv at gmail.com>
Evan Shaw <evan at vendhq.com>
Frederick Mayle <frederickmayle at gmail.com>
Gustavo Kristic <gkristic at gmail.com>
Hajime Nakagami <nakagami at gmail.com>
Hanno Braun <mail at hannobraun.com>
Henri Yandell <flamefew at gmail.com>
Hirotaka Yamamoto <ymmt2005 at gmail.com>
ICHINOSE Shogo <shogo82148 at gmail.com>
INADA Naoki <songofacandy at gmail.com>
Jacek Szwec <szwec.jacek at gmail.com>
James Harr <james.harr at gmail.com>
Jeff Hodges <jeff at somethingsimilar.com>
Jeffrey Charles <jeffreycharles at gmail.com>
Jian Zhen <zhenjl at gmail.com>
Joshua Prunier <joshua.prunier at gmail.com>
Julien Lefevre <julien.lefevr at gmail.com>
Julien Schmidt <go-sql-driver at julienschmidt.com>
Justin Li <jli at j-li.net>
Justin Nuß <nuss.justin at gmail.com>
Kamil Dziedzic <kamil at klecza.pl>
Kevin Malachowski <kevin at chowski.com>
Kieron Woodhouse <kieron.woodhouse at infosum.com>
Lennart Rudolph <lrudolph at hmc.edu>
Leonardo YongUk Kim <dalinaum at gmail.com>
Linh Tran Tuan <linhduonggnu at gmail.com>
Lion Yang <lion at aosc.xyz>
Luca Looz <luca.looz92 at gmail.com>
Lucas Liu <extrafliu at gmail.com>
Luke Scott <luke at webconnex.com>
Maciej Zimnoch <maciej.zimnoch at codilime.com>
Michael Woolnough <michael.woolnough at gmail.com>
Nicola Peduzzi <thenikso at gmail.com>
Olivier Mengué <dolmen at cpan.org>
oscarzhao <oscarzhaosl at gmail.com>
Paul Bonser <misterpib at gmail.com>
Peter Schultz <peter.schultz at classmarkets.com>
Rebecca Chin <rchin at pivotal.io>
Reed Allman <rdallman10 at gmail.com>
Richard Wilkes <wilkes at me.com>
Robert Russell <robert at rrbrussell.com>
Runrioter Wung <runrioter at gmail.com>
Shuode Li <elemount at qq.com>
Soroush Pour <me at soroushjp.com>
Stan Putrya <root.vagner at gmail.com>
Stanley Gunawan <gunawan.stanley at gmail.com>
Xiangyu Hu <xiangyu.hu at outlook.com>
Xiaobing Jiang <s7v7nislands at gmail.com>
Xiuming Chen <cc at cxm.cc>
Zhenye Xie <xiezhenye at gmail.com>
# Organizations
Barracuda Networks, Inc.
Counting Ltd.
Google Inc.
InfoSum Ltd.
Keybase Inc.
Percona LLC
Pivotal Inc.
Stripe Inc.

View File

@ -1,178 +0,0 @@
## Version 1.4.1 (2018-11-14)
Bugfixes:
- Fix TIME format for binary columns (#818)
- Fix handling of empty auth plugin names (#835)
- Fix caching_sha2_password with empty password (#826)
- Fix canceled context broke mysqlConn (#862)
- Fix OldAuthSwitchRequest support (#870)
- Fix Auth Response packet for cleartext password (#887)
## Version 1.4 (2018-06-03)
Changes:
- Documentation fixes (#530, #535, #567)
- Refactoring (#575, #579, #580, #581, #603, #615, #704)
- Cache column names (#444)
- Sort the DSN parameters in DSNs generated from a config (#637)
- Allow native password authentication by default (#644)
- Use the default port if it is missing in the DSN (#668)
- Removed the `strict` mode (#676)
- Do not query `max_allowed_packet` by default (#680)
- Dropped support Go 1.6 and lower (#696)
- Updated `ConvertValue()` to match the database/sql/driver implementation (#760)
- Document the usage of `0000-00-00T00:00:00` as the time.Time zero value (#783)
- Improved the compatibility of the authentication system (#807)
New Features:
- Multi-Results support (#537)
- `rejectReadOnly` DSN option (#604)
- `context.Context` support (#608, #612, #627, #761)
- Transaction isolation level support (#619, #744)
- Read-Only transactions support (#618, #634)
- `NewConfig` function which initializes a config with default values (#679)
- Implemented the `ColumnType` interfaces (#667, #724)
- Support for custom string types in `ConvertValue` (#623)
- Implemented `NamedValueChecker`, improving support for uint64 with high bit set (#690, #709, #710)
- `caching_sha2_password` authentication plugin support (#794, #800, #801, #802)
- Implemented `driver.SessionResetter` (#779)
- `sha256_password` authentication plugin support (#808)
Bugfixes:
- Use the DSN hostname as TLS default ServerName if `tls=true` (#564, #718)
- Fixed LOAD LOCAL DATA INFILE for empty files (#590)
- Removed columns definition cache since it sometimes cached invalid data (#592)
- Don't mutate registered TLS configs (#600)
- Make RegisterTLSConfig concurrency-safe (#613)
- Handle missing auth data in the handshake packet correctly (#646)
- Do not retry queries when data was written to avoid data corruption (#302, #736)
- Cache the connection pointer for error handling before invalidating it (#678)
- Fixed imports for appengine/cloudsql (#700)
- Fix sending STMT_LONG_DATA for 0 byte data (#734)
- Set correct capacity for []bytes read from length-encoded strings (#766)
- Make RegisterDial concurrency-safe (#773)
## Version 1.3 (2016-12-01)
Changes:
- Go 1.1 is no longer supported
- Use decimals fields in MySQL to format time types (#249)
- Buffer optimizations (#269)
- TLS ServerName defaults to the host (#283)
- Refactoring (#400, #410, #437)
- Adjusted documentation for second generation CloudSQL (#485)
- Documented DSN system var quoting rules (#502)
- Made statement.Close() calls idempotent to avoid errors in Go 1.6+ (#512)
New Features:
- Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249)
- Support for returning table alias on Columns() (#289, #359, #382)
- Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490)
- Support for uint64 parameters with high bit set (#332, #345)
- Cleartext authentication plugin support (#327)
- Exported ParseDSN function and the Config struct (#403, #419, #429)
- Read / Write timeouts (#401)
- Support for JSON field type (#414)
- Support for multi-statements and multi-results (#411, #431)
- DSN parameter to set the driver-side max_allowed_packet value manually (#489)
- Native password authentication plugin support (#494, #524)
Bugfixes:
- Fixed handling of queries without columns and rows (#255)
- Fixed a panic when SetKeepAlive() failed (#298)
- Handle ERR packets while reading rows (#321)
- Fixed reading NULL length-encoded integers in MySQL 5.6+ (#349)
- Fixed absolute paths support in LOAD LOCAL DATA INFILE (#356)
- Actually zero out bytes in handshake response (#378)
- Fixed race condition in registering LOAD DATA INFILE handler (#383)
- Fixed tests with MySQL 5.7.9+ (#380)
- QueryUnescape TLS config names (#397)
- Fixed "broken pipe" error by writing to closed socket (#390)
- Fixed LOAD LOCAL DATA INFILE buffering (#424)
- Fixed parsing of floats into float64 when placeholders are used (#434)
- Fixed DSN tests with Go 1.7+ (#459)
- Handle ERR packets while waiting for EOF (#473)
- Invalidate connection on error while discarding additional results (#513)
- Allow terminating packets of length 0 (#516)
## Version 1.2 (2014-06-03)
Changes:
- We switched back to a "rolling release". `go get` installs the current master branch again
- Version v1 of the driver will not be maintained anymore. Go 1.0 is no longer supported by this driver
- Exported errors to allow easy checking from application code
- Enabled TCP Keepalives on TCP connections
- Optimized INFILE handling (better buffer size calculation, lazy init, ...)
- The DSN parser also checks for a missing separating slash
- Faster binary date / datetime to string formatting
- Also exported the MySQLWarning type
- mysqlConn.Close returns the first error encountered instead of ignoring all errors
- writePacket() automatically writes the packet size to the header
- readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets
New Features:
- `RegisterDial` allows the usage of a custom dial function to establish the network connection
- Setting the connection collation is possible with the `collation` DSN parameter. This parameter should be preferred over the `charset` parameter
- Logging of critical errors is configurable with `SetLogger`
- Google CloudSQL support
Bugfixes:
- Allow more than 32 parameters in prepared statements
- Various old_password fixes
- Fixed TestConcurrent test to pass Go's race detection
- Fixed appendLengthEncodedInteger for large numbers
- Renamed readLengthEnodedString to readLengthEncodedString and skipLengthEnodedString to skipLengthEncodedString (fixed typo)
## Version 1.1 (2013-11-02)
Changes:
- Go-MySQL-Driver now requires Go 1.1
- Connections now use the collation `utf8_general_ci` by default. Adding `&charset=UTF8` to the DSN should not be necessary anymore
- Made closing rows and connections error tolerant. This allows for example deferring rows.Close() without checking for errors
- `[]byte(nil)` is now treated as a NULL value. Before, it was treated like an empty string / `[]byte("")`
- DSN parameter values must now be url.QueryEscape'ed. This allows text values to contain special characters, such as '&'.
- Use the IO buffer also for writing. This results in zero allocations (by the driver) for most queries
- Optimized the buffer for reading
- stmt.Query now caches column metadata
- New Logo
- Changed the copyright header to include all contributors
- Improved the LOAD INFILE documentation
- The driver struct is now exported to make the driver directly accessible
- Refactored the driver tests
- Added more benchmarks and moved all to a separate file
- Other small refactoring
New Features:
- Added *old_passwords* support: Required in some cases, but must be enabled by adding `allowOldPasswords=true` to the DSN since it is insecure
- Added a `clientFoundRows` parameter: Return the number of matching rows instead of the number of rows changed on UPDATEs
- Added TLS/SSL support: Use a TLS/SSL encrypted connection to the server. Custom TLS configs can be registered and used
Bugfixes:
- Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification
- Convert to DB timezone when inserting `time.Time`
- Splitted packets (more than 16MB) are now merged correctly
- Fixed false positive `io.EOF` errors when the data was fully read
- Avoid panics on reuse of closed connections
- Fixed empty string producing false nil values
- Fixed sign byte for positive TIME fields
## Version 1.0 (2013-05-14)
Initial Release

View File

@ -1,23 +0,0 @@
# Contributing Guidelines
## Reporting Issues
Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed).
## Contributing Code
By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file.
Don't forget to add yourself to the AUTHORS file.
### Code Review
Everyone is invited to review and comment on pull requests.
If it looks fine to you, comment with "LGTM" (Looks good to me).
If changes are required, notice the reviewers with "PTAL" (Please take another look) after committing the fixes.
Before merging the Pull Request, at least one [team member](https://github.com/go-sql-driver?tab=members) must have commented with "LGTM".
## Development Ideas
If you are looking for ideas for code contributions, please check our [Development Ideas](https://github.com/go-sql-driver/mysql/wiki/Development-Ideas) Wiki page.

View File

@ -1,373 +0,0 @@
Mozilla Public License Version 2.0
==================================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to
the creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used
by a Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached
the notice in Exhibit A, the Executable Form of such Source Code
Form, and Modifications of such Source Code Form, in each case
including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
(a) that the initial Contributor has attached the notice described
in Exhibit B to the Covered Software; or
(b) that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the
terms of a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible,
whether at the time of the initial grant or subsequently, any and
all of the rights conveyed by this License.
1.10. "Modifications"
means any of the following:
(a) any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered
Software; or
(b) any new file in Source Code Form that contains any Covered
Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the
License, by the making, using, selling, offering for sale, having
made, import, or transfer of either its Contributions or its
Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU
Lesser General Public License, Version 2.1, the GNU Affero General
Public License, Version 3.0, or any later versions of those
licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that
controls, is controlled by, or is under common control with You. For
purposes of this definition, "control" means (a) the power, direct
or indirect, to cause the direction or management of such entity,
whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial
ownership of such entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
(b) under Patent Claims of such Contributor to make, use, sell, offer
for sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
(a) for any code that a Contributor has removed from Covered Software;
or
(b) for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
(c) under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights
to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
in Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
(a) such Covered Software must also be made available in Source Code
Form, as described in Section 3.1, and You must inform recipients of
the Executable Form how they can obtain a copy of such Source Code
Form by reasonable means in a timely manner, at a charge no more
than the cost of distribution to the recipient; and
(b) You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter
the recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty,
or limitations of liability) contained within the Source Code Form of
the Covered Software, except that You may alter any license notices to
the extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Software due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description must
be placed in a text file included with all distributions of the Covered
Software under this License. Except to the extent prohibited by statute
or regulation, such description must be sufficiently detailed for a
recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically
if You fail to comply with any of its terms. However, if You become
compliant, then the rights granted under this License from a particular
Contributor are reinstated (a) provisionally, unless and until such
Contributor explicitly and finally terminates Your grants, and (b) on an
ongoing basis, if such Contributor fails to notify You of the
non-compliance by some reasonable means prior to 60 days after You have
come back into compliance. Moreover, Your grants from a particular
Contributor are reinstated on an ongoing basis if such Contributor
notifies You of the non-compliance by some reasonable means, this is the
first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after
Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
end user license agreements (excluding distributors and resellers) which
have been validly granted by You or Your distributors under this License
prior to termination shall survive termination.
************************************************************************
* *
* 6. Disclaimer of Warranty *
* ------------------------- *
* *
* Covered Software is provided under this License on an "as is" *
* basis, without warranty of any kind, either expressed, implied, or *
* statutory, including, without limitation, warranties that the *
* Covered Software is free of defects, merchantable, fit for a *
* particular purpose or non-infringing. The entire risk as to the *
* quality and performance of the Covered Software is with You. *
* Should any Covered Software prove defective in any respect, You *
* (not any Contributor) assume the cost of any necessary servicing, *
* repair, or correction. This disclaimer of warranty constitutes an *
* essential part of this License. No use of any Covered Software is *
* authorized under this License except under this disclaimer. *
* *
************************************************************************
************************************************************************
* *
* 7. Limitation of Liability *
* -------------------------- *
* *
* Under no circumstances and under no legal theory, whether tort *
* (including negligence), contract, or otherwise, shall any *
* Contributor, or anyone who distributes Covered Software as *
* permitted above, be liable to You for any direct, indirect, *
* special, incidental, or consequential damages of any character *
* including, without limitation, damages for lost profits, loss of *
* goodwill, work stoppage, computer failure or malfunction, or any *
* and all other commercial damages or losses, even if such party *
* shall have been informed of the possibility of such damages. This *
* limitation of liability shall not apply to liability for death or *
* personal injury resulting from such party's negligence to the *
* extent applicable law prohibits such limitation. Some *
* jurisdictions do not allow the exclusion or limitation of *
* incidental or consequential damages, so this exclusion and *
* limitation may not apply to You. *
* *
************************************************************************
8. Litigation
-------------
Any litigation relating to this License may be brought only in the
courts of a jurisdiction where the defendant maintains its principal
place of business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions.
Nothing in this Section shall prevent a party's ability to bring
cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides
that the language of a contract shall be construed against the drafter
shall not be used to construe this License against a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the
notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to look
for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.

View File

@ -1,490 +0,0 @@
# Go-MySQL-Driver
A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package
![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin")
---------------------------------------
* [Features](#features)
* [Requirements](#requirements)
* [Installation](#installation)
* [Usage](#usage)
* [DSN (Data Source Name)](#dsn-data-source-name)
* [Password](#password)
* [Protocol](#protocol)
* [Address](#address)
* [Parameters](#parameters)
* [Examples](#examples)
* [Connection pool and timeouts](#connection-pool-and-timeouts)
* [context.Context Support](#contextcontext-support)
* [ColumnType Support](#columntype-support)
* [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support)
* [time.Time support](#timetime-support)
* [Unicode support](#unicode-support)
* [Testing / Development](#testing--development)
* [License](#license)
---------------------------------------
## Features
* Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance")
* Native Go implementation. No C-bindings, just pure Go
* Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](https://godoc.org/github.com/go-sql-driver/mysql#DialFunc)
* Automatic handling of broken connections
* Automatic Connection Pooling *(by database/sql package)*
* Supports queries larger than 16MB
* Full [`sql.RawBytes`](https://golang.org/pkg/database/sql/#RawBytes) support.
* Intelligent `LONG DATA` handling in prepared statements
* Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support
* Optional `time.Time` parsing
* Optional placeholder interpolation
## Requirements
* Go 1.7 or higher. We aim to support the 3 latest versions of Go.
* MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+)
---------------------------------------
## Installation
Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell:
```bash
$ go get -u github.com/go-sql-driver/mysql
```
Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`.
## Usage
_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](https://golang.org/pkg/database/sql/) API then.
Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`:
```go
import "database/sql"
import _ "github.com/go-sql-driver/mysql"
db, err := sql.Open("mysql", "user:password@/dbname")
```
[Examples are available in our Wiki](https://github.com/go-sql-driver/mysql/wiki/Examples "Go-MySQL-Driver Examples").
### DSN (Data Source Name)
The Data Source Name has a common format, like e.g. [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php) uses it, but without type-prefix (optional parts marked by squared brackets):
```
[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
```
A DSN in its fullest form:
```
username:password@protocol(address)/dbname?param=value
```
Except for the databasename, all values are optional. So the minimal DSN is:
```
/dbname
```
If you do not want to preselect a database, leave `dbname` empty:
```
/
```
This has the same effect as an empty DSN string:
```
```
Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct.
#### Password
Passwords can consist of any character. Escaping is **not** necessary.
#### Protocol
See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available.
In general you should use an Unix domain socket if available and TCP otherwise for best performance.
#### Address
For TCP and UDP networks, addresses have the form `host[:port]`.
If `port` is omitted, the default port will be used.
If `host` is a literal IPv6 address, it must be enclosed in square brackets.
The functions [net.JoinHostPort](https://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](https://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form.
For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`.
#### Parameters
*Parameters are case-sensitive!*
Notice that any of `true`, `TRUE`, `True` or `1` is accepted to stand for a true boolean value. Not surprisingly, false can be specified as any of: `false`, `FALSE`, `False` or `0`.
##### `allowAllFiles`
```
Type: bool
Valid Values: true, false
Default: false
```
`allowAllFiles=true` disables the file Whitelist for `LOAD DATA LOCAL INFILE` and allows *all* files.
[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)
##### `allowCleartextPasswords`
```
Type: bool
Valid Values: true, false
Default: false
```
`allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network.
##### `allowNativePasswords`
```
Type: bool
Valid Values: true, false
Default: true
```
`allowNativePasswords=false` disallows the usage of MySQL native password method.
##### `allowOldPasswords`
```
Type: bool
Valid Values: true, false
Default: false
```
`allowOldPasswords=true` allows the usage of the insecure old password method. This should be avoided, but is necessary in some cases. See also [the old_passwords wiki page](https://github.com/go-sql-driver/mysql/wiki/old_passwords).
##### `charset`
```
Type: string
Valid Values: <name>
Default: none
```
Sets the charset used for client-server interaction (`"SET NAMES <value>"`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`).
Usage of the `charset` parameter is discouraged because it issues additional queries to the server.
Unless you need the fallback behavior, please use `collation` instead.
##### `collation`
```
Type: string
Valid Values: <name>
Default: utf8_general_ci
```
Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail.
A list of valid charsets for a server is retrievable with `SHOW COLLATION`.
##### `clientFoundRows`
```
Type: bool
Valid Values: true, false
Default: false
```
`clientFoundRows=true` causes an UPDATE to return the number of matching rows instead of the number of rows changed.
##### `columnsWithAlias`
```
Type: bool
Valid Values: true, false
Default: false
```
When `columnsWithAlias` is true, calls to `sql.Rows.Columns()` will return the table alias and the column name separated by a dot. For example:
```
SELECT u.id FROM users as u
```
will return `u.id` instead of just `id` if `columnsWithAlias=true`.
##### `interpolateParams`
```
Type: bool
Valid Values: true, false
Default: false
```
If `interpolateParams` is true, placeholders (`?`) in calls to `db.Query()` and `db.Exec()` are interpolated into a single query string with given parameters. This reduces the number of roundtrips, since the driver has to prepare a statement, execute it with given parameters and close the statement again with `interpolateParams=false`.
*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are blacklisted as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!*
##### `loc`
```
Type: string
Valid Values: <escaped name>
Default: UTC
```
Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) for details.
Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter.
Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`.
##### `maxAllowedPacket`
```
Type: decimal number
Default: 4194304
```
Max packet size allowed in bytes. The default value is 4 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*.
##### `multiStatements`
```
Type: bool
Valid Values: true, false
Default: false
```
Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded.
When `multiStatements` is used, `?` parameters must only be used in the first statement.
##### `parseTime`
```
Type: bool
Valid Values: true, false
Default: false
```
`parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string`
The date or datetime like `0000-00-00 00:00:00` is converted into zero value of `time.Time`.
##### `readTimeout`
```
Type: duration
Default: 0
```
I/O read timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*.
##### `rejectReadOnly`
```
Type: bool
Valid Values: true, false
Default: false
```
`rejectReadOnly=true` causes the driver to reject read-only connections. This
is for a possible race condition during an automatic failover, where the mysql
client gets connected to a read-only replica after the failover.
Note that this should be a fairly rare case, as an automatic failover normally
happens when the primary is down, and the race condition shouldn't happen
unless it comes back up online as soon as the failover is kicked off. On the
other hand, when this happens, a MySQL application can get stuck on a
read-only connection until restarted. It is however fairly easy to reproduce,
for example, using a manual failover on AWS Aurora's MySQL-compatible cluster.
If you are not relying on read-only transactions to reject writes that aren't
supposed to happen, setting this on some MySQL providers (such as AWS Aurora)
is safer for failovers.
Note that ERROR 1290 can be returned for a `read-only` server and this option will
cause a retry for that error. However the same error number is used for some
other cases. You should ensure your application will never cause an ERROR 1290
except for `read-only` mode when enabling this option.
##### `serverPubKey`
```
Type: string
Valid Values: <name>
Default: none
```
Server public keys can be registered with [`mysql.RegisterServerPubKey`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterServerPubKey), which can then be used by the assigned name in the DSN.
Public keys are used to transmit encrypted data, e.g. for authentication.
If the server's public key is known, it should be set manually to avoid expensive and potentially insecure transmissions of the public key from the server to the client each time it is required.
##### `timeout`
```
Type: duration
Default: OS default
```
Timeout for establishing connections, aka dial timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*.
##### `tls`
```
Type: bool / string
Valid Values: true, false, skip-verify, <name>
Default: false
```
`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig).
##### `writeTimeout`
```
Type: duration
Default: 0
```
I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*.
##### System Variables
Any other parameters are interpreted as system variables:
* `<boolean_var>=<value>`: `SET <boolean_var>=<value>`
* `<enum_var>=<value>`: `SET <enum_var>=<value>`
* `<string_var>=%27<value>%27`: `SET <string_var>='<value>'`
Rules:
* The values for string variables must be quoted with `'`.
* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed!
(which implies values of string variables must be wrapped with `%27`).
Examples:
* `autocommit=1`: `SET autocommit=1`
* [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'`
* [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'`
#### Examples
```
user@unix(/path/to/socket)/dbname
```
```
root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local
```
```
user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true
```
Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html):
```
user:password@/dbname?sql_mode=TRADITIONAL
```
TCP via IPv6:
```
user:password@tcp([de:ad:be:ef::ca:fe]:80)/dbname?timeout=90s&collation=utf8mb4_unicode_ci
```
TCP on a remote host, e.g. Amazon RDS:
```
id:password@tcp(your-amazonaws-uri.com:3306)/dbname
```
Google Cloud SQL on App Engine (First Generation MySQL Server):
```
user@cloudsql(project-id:instance-name)/dbname
```
Google Cloud SQL on App Engine (Second Generation MySQL Server):
```
user@cloudsql(project-id:regionname:instance-name)/dbname
```
TCP using default port (3306) on localhost:
```
user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped
```
Use the default protocol (tcp) and host (localhost:3306):
```
user:password@/dbname
```
No Database preselected:
```
user:password@/
```
### Connection pool and timeouts
The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively.
## `ColumnType` Support
This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported.
## `context.Context` Support
Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts.
See [context support in the database/sql package](https://golang.org/doc/go1.8#database_sql) for more details.
### `LOAD DATA LOCAL INFILE` support
For this feature you need direct access to the package. Therefore you must change the import path (no `_`):
```go
import "github.com/go-sql-driver/mysql"
```
Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)).
To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::<name>` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore.
See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details.
### `time.Time` support
The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program.
However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter.
**Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes).
Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`.
### Unicode support
Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default.
Other collations / charsets can be set using the [`collation`](#collation) DSN parameter.
Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default.
See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support.
## Testing / Development
To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details.
Go-MySQL-Driver is not feature-complete yet. Your help is very appreciated.
If you want to contribute, you can work on an [open issue](https://github.com/go-sql-driver/mysql/issues?state=open) or review a [pull request](https://github.com/go-sql-driver/mysql/pulls).
See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/CONTRIBUTING.md) for details.
---------------------------------------
## License
Go-MySQL-Driver is licensed under the [Mozilla Public License Version 2.0](https://raw.github.com/go-sql-driver/mysql/master/LICENSE)
Mozilla summarizes the license scope as follows:
> MPL: The copyleft applies to any files containing MPLed code.
That means:
* You can **use** the **unchanged** source code both in private and commercially.
* When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0).
* You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged**.
Please read the [MPL 2.0 FAQ](https://www.mozilla.org/en-US/MPL/2.0/FAQ/) if you have further questions regarding the license.
You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE).
![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow")

View File

@ -1,19 +0,0 @@
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build appengine
package mysql
import (
"google.golang.org/appengine/cloudsql"
)
func init() {
RegisterDial("cloudsql", cloudsql.Dial)
}

View File

@ -1,420 +0,0 @@
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"crypto/rand"
"crypto/rsa"
"crypto/sha1"
"crypto/sha256"
"crypto/x509"
"encoding/pem"
"sync"
)
// server pub keys registry
var (
serverPubKeyLock sync.RWMutex
serverPubKeyRegistry map[string]*rsa.PublicKey
)
// RegisterServerPubKey registers a server RSA public key which can be used to
// send data in a secure manner to the server without receiving the public key
// in a potentially insecure way from the server first.
// Registered keys can afterwards be used adding serverPubKey=<name> to the DSN.
//
// Note: The provided rsa.PublicKey instance is exclusively owned by the driver
// after registering it and may not be modified.
//
// data, err := ioutil.ReadFile("mykey.pem")
// if err != nil {
// log.Fatal(err)
// }
//
// block, _ := pem.Decode(data)
// if block == nil || block.Type != "PUBLIC KEY" {
// log.Fatal("failed to decode PEM block containing public key")
// }
//
// pub, err := x509.ParsePKIXPublicKey(block.Bytes)
// if err != nil {
// log.Fatal(err)
// }
//
// if rsaPubKey, ok := pub.(*rsa.PublicKey); ok {
// mysql.RegisterServerPubKey("mykey", rsaPubKey)
// } else {
// log.Fatal("not a RSA public key")
// }
//
func RegisterServerPubKey(name string, pubKey *rsa.PublicKey) {
serverPubKeyLock.Lock()
if serverPubKeyRegistry == nil {
serverPubKeyRegistry = make(map[string]*rsa.PublicKey)
}
serverPubKeyRegistry[name] = pubKey
serverPubKeyLock.Unlock()
}
// DeregisterServerPubKey removes the public key registered with the given name.
func DeregisterServerPubKey(name string) {
serverPubKeyLock.Lock()
if serverPubKeyRegistry != nil {
delete(serverPubKeyRegistry, name)
}
serverPubKeyLock.Unlock()
}
func getServerPubKey(name string) (pubKey *rsa.PublicKey) {
serverPubKeyLock.RLock()
if v, ok := serverPubKeyRegistry[name]; ok {
pubKey = v
}
serverPubKeyLock.RUnlock()
return
}
// Hash password using pre 4.1 (old password) method
// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c
type myRnd struct {
seed1, seed2 uint32
}
const myRndMaxVal = 0x3FFFFFFF
// Pseudo random number generator
func newMyRnd(seed1, seed2 uint32) *myRnd {
return &myRnd{
seed1: seed1 % myRndMaxVal,
seed2: seed2 % myRndMaxVal,
}
}
// Tested to be equivalent to MariaDB's floating point variant
// http://play.golang.org/p/QHvhd4qved
// http://play.golang.org/p/RG0q4ElWDx
func (r *myRnd) NextByte() byte {
r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal
r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal
return byte(uint64(r.seed1) * 31 / myRndMaxVal)
}
// Generate binary hash from byte string using insecure pre 4.1 method
func pwHash(password []byte) (result [2]uint32) {
var add uint32 = 7
var tmp uint32
result[0] = 1345345333
result[1] = 0x12345671
for _, c := range password {
// skip spaces and tabs in password
if c == ' ' || c == '\t' {
continue
}
tmp = uint32(c)
result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8)
result[1] += (result[1] << 8) ^ result[0]
add += tmp
}
// Remove sign bit (1<<31)-1)
result[0] &= 0x7FFFFFFF
result[1] &= 0x7FFFFFFF
return
}
// Hash password using insecure pre 4.1 method
func scrambleOldPassword(scramble []byte, password string) []byte {
if len(password) == 0 {
return nil
}
scramble = scramble[:8]
hashPw := pwHash([]byte(password))
hashSc := pwHash(scramble)
r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1])
var out [8]byte
for i := range out {
out[i] = r.NextByte() + 64
}
mask := r.NextByte()
for i := range out {
out[i] ^= mask
}
return out[:]
}
// Hash password using 4.1+ method (SHA1)
func scramblePassword(scramble []byte, password string) []byte {
if len(password) == 0 {
return nil
}
// stage1Hash = SHA1(password)
crypt := sha1.New()
crypt.Write([]byte(password))
stage1 := crypt.Sum(nil)
// scrambleHash = SHA1(scramble + SHA1(stage1Hash))
// inner Hash
crypt.Reset()
crypt.Write(stage1)
hash := crypt.Sum(nil)
// outer Hash
crypt.Reset()
crypt.Write(scramble)
crypt.Write(hash)
scramble = crypt.Sum(nil)
// token = scrambleHash XOR stage1Hash
for i := range scramble {
scramble[i] ^= stage1[i]
}
return scramble
}
// Hash password using MySQL 8+ method (SHA256)
func scrambleSHA256Password(scramble []byte, password string) []byte {
if len(password) == 0 {
return nil
}
// XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble))
crypt := sha256.New()
crypt.Write([]byte(password))
message1 := crypt.Sum(nil)
crypt.Reset()
crypt.Write(message1)
message1Hash := crypt.Sum(nil)
crypt.Reset()
crypt.Write(message1Hash)
crypt.Write(scramble)
message2 := crypt.Sum(nil)
for i := range message1 {
message1[i] ^= message2[i]
}
return message1
}
func encryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, error) {
plain := make([]byte, len(password)+1)
copy(plain, password)
for i := range plain {
j := i % len(seed)
plain[i] ^= seed[j]
}
sha1 := sha1.New()
return rsa.EncryptOAEP(sha1, rand.Reader, pub, plain, nil)
}
func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) error {
enc, err := encryptPassword(mc.cfg.Passwd, seed, pub)
if err != nil {
return err
}
return mc.writeAuthSwitchPacket(enc)
}
func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) {
switch plugin {
case "caching_sha2_password":
authResp := scrambleSHA256Password(authData, mc.cfg.Passwd)
return authResp, nil
case "mysql_old_password":
if !mc.cfg.AllowOldPasswords {
return nil, ErrOldPassword
}
// Note: there are edge cases where this should work but doesn't;
// this is currently "wontfix":
// https://github.com/go-sql-driver/mysql/issues/184
authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0)
return authResp, nil
case "mysql_clear_password":
if !mc.cfg.AllowCleartextPasswords {
return nil, ErrCleartextPassword
}
// http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html
// http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html
return append([]byte(mc.cfg.Passwd), 0), nil
case "mysql_native_password":
if !mc.cfg.AllowNativePasswords {
return nil, ErrNativePassword
}
// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html
// Native password authentication only need and will need 20-byte challenge.
authResp := scramblePassword(authData[:20], mc.cfg.Passwd)
return authResp, nil
case "sha256_password":
if len(mc.cfg.Passwd) == 0 {
return []byte{0}, nil
}
if mc.cfg.tls != nil || mc.cfg.Net == "unix" {
// write cleartext auth packet
return append([]byte(mc.cfg.Passwd), 0), nil
}
pubKey := mc.cfg.pubKey
if pubKey == nil {
// request public key from server
return []byte{1}, nil
}
// encrypted password
enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey)
return enc, err
default:
errLog.Print("unknown auth plugin:", plugin)
return nil, ErrUnknownPlugin
}
}
func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error {
// Read Result Packet
authData, newPlugin, err := mc.readAuthResult()
if err != nil {
return err
}
// handle auth plugin switch, if requested
if newPlugin != "" {
// If CLIENT_PLUGIN_AUTH capability is not supported, no new cipher is
// sent and we have to keep using the cipher sent in the init packet.
if authData == nil {
authData = oldAuthData
} else {
// copy data from read buffer to owned slice
copy(oldAuthData, authData)
}
plugin = newPlugin
authResp, err := mc.auth(authData, plugin)
if err != nil {
return err
}
if err = mc.writeAuthSwitchPacket(authResp); err != nil {
return err
}
// Read Result Packet
authData, newPlugin, err = mc.readAuthResult()
if err != nil {
return err
}
// Do not allow to change the auth plugin more than once
if newPlugin != "" {
return ErrMalformPkt
}
}
switch plugin {
// https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/
case "caching_sha2_password":
switch len(authData) {
case 0:
return nil // auth successful
case 1:
switch authData[0] {
case cachingSha2PasswordFastAuthSuccess:
if err = mc.readResultOK(); err == nil {
return nil // auth successful
}
case cachingSha2PasswordPerformFullAuthentication:
if mc.cfg.tls != nil || mc.cfg.Net == "unix" {
// write cleartext auth packet
err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0))
if err != nil {
return err
}
} else {
pubKey := mc.cfg.pubKey
if pubKey == nil {
// request public key from server
data := mc.buf.takeSmallBuffer(4 + 1)
data[4] = cachingSha2PasswordRequestPublicKey
mc.writePacket(data)
// parse public key
data, err := mc.readPacket()
if err != nil {
return err
}
block, _ := pem.Decode(data[1:])
pkix, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
return err
}
pubKey = pkix.(*rsa.PublicKey)
}
// send encrypted password
err = mc.sendEncryptedPassword(oldAuthData, pubKey)
if err != nil {
return err
}
}
return mc.readResultOK()
default:
return ErrMalformPkt
}
default:
return ErrMalformPkt
}
case "sha256_password":
switch len(authData) {
case 0:
return nil // auth successful
default:
block, _ := pem.Decode(authData)
pub, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
return err
}
// send encrypted password
err = mc.sendEncryptedPassword(oldAuthData, pub.(*rsa.PublicKey))
if err != nil {
return err
}
return mc.readResultOK()
}
default:
return nil // auth successful
}
return err
}

View File

@ -1,147 +0,0 @@
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"io"
"net"
"time"
)
const defaultBufSize = 4096
// A buffer which is used for both reading and writing.
// This is possible since communication on each connection is synchronous.
// In other words, we can't write and read simultaneously on the same connection.
// The buffer is similar to bufio.Reader / Writer but zero-copy-ish
// Also highly optimized for this particular use case.
type buffer struct {
buf []byte
nc net.Conn
idx int
length int
timeout time.Duration
}
func newBuffer(nc net.Conn) buffer {
var b [defaultBufSize]byte
return buffer{
buf: b[:],
nc: nc,
}
}
// fill reads into the buffer until at least _need_ bytes are in it
func (b *buffer) fill(need int) error {
n := b.length
// move existing data to the beginning
if n > 0 && b.idx > 0 {
copy(b.buf[0:n], b.buf[b.idx:])
}
// grow buffer if necessary
// TODO: let the buffer shrink again at some point
// Maybe keep the org buf slice and swap back?
if need > len(b.buf) {
// Round up to the next multiple of the default size
newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize)
copy(newBuf, b.buf)
b.buf = newBuf
}
b.idx = 0
for {
if b.timeout > 0 {
if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil {
return err
}
}
nn, err := b.nc.Read(b.buf[n:])
n += nn
switch err {
case nil:
if n < need {
continue
}
b.length = n
return nil
case io.EOF:
if n >= need {
b.length = n
return nil
}
return io.ErrUnexpectedEOF
default:
return err
}
}
}
// returns next N bytes from buffer.
// The returned slice is only guaranteed to be valid until the next read
func (b *buffer) readNext(need int) ([]byte, error) {
if b.length < need {
// refill
if err := b.fill(need); err != nil {
return nil, err
}
}
offset := b.idx
b.idx += need
b.length -= need
return b.buf[offset:b.idx], nil
}
// returns a buffer with the requested size.
// If possible, a slice from the existing buffer is returned.
// Otherwise a bigger buffer is made.
// Only one buffer (total) can be used at a time.
func (b *buffer) takeBuffer(length int) []byte {
if b.length > 0 {
return nil
}
// test (cheap) general case first
if length <= defaultBufSize || length <= cap(b.buf) {
return b.buf[:length]
}
if length < maxPacketSize {
b.buf = make([]byte, length)
return b.buf
}
return make([]byte, length)
}
// shortcut which can be used if the requested buffer is guaranteed to be
// smaller than defaultBufSize
// Only one buffer (total) can be used at a time.
func (b *buffer) takeSmallBuffer(length int) []byte {
if b.length > 0 {
return nil
}
return b.buf[:length]
}
// takeCompleteBuffer returns the complete existing buffer.
// This can be used if the necessary buffer size is unknown.
// Only one buffer (total) can be used at a time.
func (b *buffer) takeCompleteBuffer() []byte {
if b.length > 0 {
return nil
}
return b.buf
}

View File

@ -1,251 +0,0 @@
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2014 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
const defaultCollation = "utf8_general_ci"
const binaryCollation = "binary"
// A list of available collations mapped to the internal ID.
// To update this map use the following MySQL query:
// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS
var collations = map[string]byte{
"big5_chinese_ci": 1,
"latin2_czech_cs": 2,
"dec8_swedish_ci": 3,
"cp850_general_ci": 4,
"latin1_german1_ci": 5,
"hp8_english_ci": 6,
"koi8r_general_ci": 7,
"latin1_swedish_ci": 8,
"latin2_general_ci": 9,
"swe7_swedish_ci": 10,
"ascii_general_ci": 11,
"ujis_japanese_ci": 12,
"sjis_japanese_ci": 13,
"cp1251_bulgarian_ci": 14,
"latin1_danish_ci": 15,
"hebrew_general_ci": 16,
"tis620_thai_ci": 18,
"euckr_korean_ci": 19,
"latin7_estonian_cs": 20,
"latin2_hungarian_ci": 21,
"koi8u_general_ci": 22,
"cp1251_ukrainian_ci": 23,
"gb2312_chinese_ci": 24,
"greek_general_ci": 25,
"cp1250_general_ci": 26,
"latin2_croatian_ci": 27,
"gbk_chinese_ci": 28,
"cp1257_lithuanian_ci": 29,
"latin5_turkish_ci": 30,
"latin1_german2_ci": 31,
"armscii8_general_ci": 32,
"utf8_general_ci": 33,
"cp1250_czech_cs": 34,
"ucs2_general_ci": 35,
"cp866_general_ci": 36,
"keybcs2_general_ci": 37,
"macce_general_ci": 38,
"macroman_general_ci": 39,
"cp852_general_ci": 40,
"latin7_general_ci": 41,
"latin7_general_cs": 42,
"macce_bin": 43,
"cp1250_croatian_ci": 44,
"utf8mb4_general_ci": 45,
"utf8mb4_bin": 46,
"latin1_bin": 47,
"latin1_general_ci": 48,
"latin1_general_cs": 49,
"cp1251_bin": 50,
"cp1251_general_ci": 51,
"cp1251_general_cs": 52,
"macroman_bin": 53,
"utf16_general_ci": 54,
"utf16_bin": 55,
"utf16le_general_ci": 56,
"cp1256_general_ci": 57,
"cp1257_bin": 58,
"cp1257_general_ci": 59,
"utf32_general_ci": 60,
"utf32_bin": 61,
"utf16le_bin": 62,
"binary": 63,
"armscii8_bin": 64,
"ascii_bin": 65,
"cp1250_bin": 66,
"cp1256_bin": 67,
"cp866_bin": 68,
"dec8_bin": 69,
"greek_bin": 70,
"hebrew_bin": 71,
"hp8_bin": 72,
"keybcs2_bin": 73,
"koi8r_bin": 74,
"koi8u_bin": 75,
"latin2_bin": 77,
"latin5_bin": 78,
"latin7_bin": 79,
"cp850_bin": 80,
"cp852_bin": 81,
"swe7_bin": 82,
"utf8_bin": 83,
"big5_bin": 84,
"euckr_bin": 85,
"gb2312_bin": 86,
"gbk_bin": 87,
"sjis_bin": 88,
"tis620_bin": 89,
"ucs2_bin": 90,
"ujis_bin": 91,
"geostd8_general_ci": 92,
"geostd8_bin": 93,
"latin1_spanish_ci": 94,
"cp932_japanese_ci": 95,
"cp932_bin": 96,
"eucjpms_japanese_ci": 97,
"eucjpms_bin": 98,
"cp1250_polish_ci": 99,
"utf16_unicode_ci": 101,
"utf16_icelandic_ci": 102,
"utf16_latvian_ci": 103,
"utf16_romanian_ci": 104,
"utf16_slovenian_ci": 105,
"utf16_polish_ci": 106,
"utf16_estonian_ci": 107,
"utf16_spanish_ci": 108,
"utf16_swedish_ci": 109,
"utf16_turkish_ci": 110,
"utf16_czech_ci": 111,
"utf16_danish_ci": 112,
"utf16_lithuanian_ci": 113,
"utf16_slovak_ci": 114,
"utf16_spanish2_ci": 115,
"utf16_roman_ci": 116,
"utf16_persian_ci": 117,
"utf16_esperanto_ci": 118,
"utf16_hungarian_ci": 119,
"utf16_sinhala_ci": 120,
"utf16_german2_ci": 121,
"utf16_croatian_ci": 122,
"utf16_unicode_520_ci": 123,
"utf16_vietnamese_ci": 124,
"ucs2_unicode_ci": 128,
"ucs2_icelandic_ci": 129,
"ucs2_latvian_ci": 130,
"ucs2_romanian_ci": 131,
"ucs2_slovenian_ci": 132,
"ucs2_polish_ci": 133,
"ucs2_estonian_ci": 134,
"ucs2_spanish_ci": 135,
"ucs2_swedish_ci": 136,
"ucs2_turkish_ci": 137,
"ucs2_czech_ci": 138,
"ucs2_danish_ci": 139,
"ucs2_lithuanian_ci": 140,
"ucs2_slovak_ci": 141,
"ucs2_spanish2_ci": 142,
"ucs2_roman_ci": 143,
"ucs2_persian_ci": 144,
"ucs2_esperanto_ci": 145,
"ucs2_hungarian_ci": 146,
"ucs2_sinhala_ci": 147,
"ucs2_german2_ci": 148,
"ucs2_croatian_ci": 149,
"ucs2_unicode_520_ci": 150,
"ucs2_vietnamese_ci": 151,
"ucs2_general_mysql500_ci": 159,
"utf32_unicode_ci": 160,
"utf32_icelandic_ci": 161,
"utf32_latvian_ci": 162,
"utf32_romanian_ci": 163,
"utf32_slovenian_ci": 164,
"utf32_polish_ci": 165,
"utf32_estonian_ci": 166,
"utf32_spanish_ci": 167,
"utf32_swedish_ci": 168,
"utf32_turkish_ci": 169,
"utf32_czech_ci": 170,
"utf32_danish_ci": 171,
"utf32_lithuanian_ci": 172,
"utf32_slovak_ci": 173,
"utf32_spanish2_ci": 174,
"utf32_roman_ci": 175,
"utf32_persian_ci": 176,
"utf32_esperanto_ci": 177,
"utf32_hungarian_ci": 178,
"utf32_sinhala_ci": 179,
"utf32_german2_ci": 180,
"utf32_croatian_ci": 181,
"utf32_unicode_520_ci": 182,
"utf32_vietnamese_ci": 183,
"utf8_unicode_ci": 192,
"utf8_icelandic_ci": 193,
"utf8_latvian_ci": 194,
"utf8_romanian_ci": 195,
"utf8_slovenian_ci": 196,
"utf8_polish_ci": 197,
"utf8_estonian_ci": 198,
"utf8_spanish_ci": 199,
"utf8_swedish_ci": 200,
"utf8_turkish_ci": 201,
"utf8_czech_ci": 202,
"utf8_danish_ci": 203,
"utf8_lithuanian_ci": 204,
"utf8_slovak_ci": 205,
"utf8_spanish2_ci": 206,
"utf8_roman_ci": 207,
"utf8_persian_ci": 208,
"utf8_esperanto_ci": 209,
"utf8_hungarian_ci": 210,
"utf8_sinhala_ci": 211,
"utf8_german2_ci": 212,
"utf8_croatian_ci": 213,
"utf8_unicode_520_ci": 214,
"utf8_vietnamese_ci": 215,
"utf8_general_mysql500_ci": 223,
"utf8mb4_unicode_ci": 224,
"utf8mb4_icelandic_ci": 225,
"utf8mb4_latvian_ci": 226,
"utf8mb4_romanian_ci": 227,
"utf8mb4_slovenian_ci": 228,
"utf8mb4_polish_ci": 229,
"utf8mb4_estonian_ci": 230,
"utf8mb4_spanish_ci": 231,
"utf8mb4_swedish_ci": 232,
"utf8mb4_turkish_ci": 233,
"utf8mb4_czech_ci": 234,
"utf8mb4_danish_ci": 235,
"utf8mb4_lithuanian_ci": 236,
"utf8mb4_slovak_ci": 237,
"utf8mb4_spanish2_ci": 238,
"utf8mb4_roman_ci": 239,
"utf8mb4_persian_ci": 240,
"utf8mb4_esperanto_ci": 241,
"utf8mb4_hungarian_ci": 242,
"utf8mb4_sinhala_ci": 243,
"utf8mb4_german2_ci": 244,
"utf8mb4_croatian_ci": 245,
"utf8mb4_unicode_520_ci": 246,
"utf8mb4_vietnamese_ci": 247,
}
// A blacklist of collations which is unsafe to interpolate parameters.
// These multibyte encodings may contains 0x5c (`\`) in their trailing bytes.
var unsafeCollations = map[string]bool{
"big5_chinese_ci": true,
"sjis_japanese_ci": true,
"gbk_chinese_ci": true,
"big5_bin": true,
"gb2312_bin": true,
"gbk_bin": true,
"sjis_bin": true,
"cp932_japanese_ci": true,
"cp932_bin": true,
}

View File

@ -1,461 +0,0 @@
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"database/sql/driver"
"io"
"net"
"strconv"
"strings"
"time"
)
// a copy of context.Context for Go 1.7 and earlier
type mysqlContext interface {
Done() <-chan struct{}
Err() error
// defined in context.Context, but not used in this driver:
// Deadline() (deadline time.Time, ok bool)
// Value(key interface{}) interface{}
}
type mysqlConn struct {
buf buffer
netConn net.Conn
affectedRows uint64
insertId uint64
cfg *Config
maxAllowedPacket int
maxWriteSize int
writeTimeout time.Duration
flags clientFlag
status statusFlag
sequence uint8
parseTime bool
// for context support (Go 1.8+)
watching bool
watcher chan<- mysqlContext
closech chan struct{}
finished chan<- struct{}
canceled atomicError // set non-nil if conn is canceled
closed atomicBool // set when conn is closed, before closech is closed
}
// Handles parameters set in DSN after the connection is established
func (mc *mysqlConn) handleParams() (err error) {
for param, val := range mc.cfg.Params {
switch param {
// Charset
case "charset":
charsets := strings.Split(val, ",")
for i := range charsets {
// ignore errors here - a charset may not exist
err = mc.exec("SET NAMES " + charsets[i])
if err == nil {
break
}
}
if err != nil {
return
}
// System Vars
default:
err = mc.exec("SET " + param + "=" + val + "")
if err != nil {
return
}
}
}
return
}
func (mc *mysqlConn) markBadConn(err error) error {
if mc == nil {
return err
}
if err != errBadConnNoWrite {
return err
}
return driver.ErrBadConn
}
func (mc *mysqlConn) Begin() (driver.Tx, error) {
return mc.begin(false)
}
func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) {
if mc.closed.IsSet() {
errLog.Print(ErrInvalidConn)
return nil, driver.ErrBadConn
}
var q string
if readOnly {
q = "START TRANSACTION READ ONLY"
} else {
q = "START TRANSACTION"
}
err := mc.exec(q)
if err == nil {
return &mysqlTx{mc}, err
}
return nil, mc.markBadConn(err)
}
func (mc *mysqlConn) Close() (err error) {
// Makes Close idempotent
if !mc.closed.IsSet() {
err = mc.writeCommandPacket(comQuit)
}
mc.cleanup()
return
}
// Closes the network connection and unsets internal variables. Do not call this
// function after successfully authentication, call Close instead. This function
// is called before auth or on auth failure because MySQL will have already
// closed the network connection.
func (mc *mysqlConn) cleanup() {
if !mc.closed.TrySet(true) {
return
}
// Makes cleanup idempotent
close(mc.closech)
if mc.netConn == nil {
return
}
if err := mc.netConn.Close(); err != nil {
errLog.Print(err)
}
}
func (mc *mysqlConn) error() error {
if mc.closed.IsSet() {
if err := mc.canceled.Value(); err != nil {
return err
}
return ErrInvalidConn
}
return nil
}
func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
if mc.closed.IsSet() {
errLog.Print(ErrInvalidConn)
return nil, driver.ErrBadConn
}
// Send command
err := mc.writeCommandPacketStr(comStmtPrepare, query)
if err != nil {
return nil, mc.markBadConn(err)
}
stmt := &mysqlStmt{
mc: mc,
}
// Read Result
columnCount, err := stmt.readPrepareResultPacket()
if err == nil {
if stmt.paramCount > 0 {
if err = mc.readUntilEOF(); err != nil {
return nil, err
}
}
if columnCount > 0 {
err = mc.readUntilEOF()
}
}
return stmt, err
}
func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) {
// Number of ? should be same to len(args)
if strings.Count(query, "?") != len(args) {
return "", driver.ErrSkip
}
buf := mc.buf.takeCompleteBuffer()
if buf == nil {
// can not take the buffer. Something must be wrong with the connection
errLog.Print(ErrBusyBuffer)
return "", ErrInvalidConn
}
buf = buf[:0]
argPos := 0
for i := 0; i < len(query); i++ {
q := strings.IndexByte(query[i:], '?')
if q == -1 {
buf = append(buf, query[i:]...)
break
}
buf = append(buf, query[i:i+q]...)
i += q
arg := args[argPos]
argPos++
if arg == nil {
buf = append(buf, "NULL"...)
continue
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
v := v.In(mc.cfg.Loc)
v = v.Add(time.Nanosecond * 500) // To round under microsecond
year := v.Year()
year100 := year / 100
year1 := year % 100
month := v.Month()
day := v.Day()
hour := v.Hour()
minute := v.Minute()
second := v.Second()
micro := v.Nanosecond() / 1000
buf = append(buf, []byte{
'\'',
digits10[year100], digits01[year100],
digits10[year1], digits01[year1],
'-',
digits10[month], digits01[month],
'-',
digits10[day], digits01[day],
' ',
digits10[hour], digits01[hour],
':',
digits10[minute], digits01[minute],
':',
digits10[second], digits01[second],
}...)
if micro != 0 {
micro10000 := micro / 10000
micro100 := micro / 100 % 100
micro1 := micro % 100
buf = append(buf, []byte{
'.',
digits10[micro10000], digits01[micro10000],
digits10[micro100], digits01[micro100],
digits10[micro1], digits01[micro1],
}...)
}
buf = append(buf, '\'')
}
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
buf = append(buf, "_binary'"...)
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
} else {
buf = escapeBytesQuotes(buf, v)
}
buf = append(buf, '\'')
}
case string:
buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeStringBackslash(buf, v)
} else {
buf = escapeStringQuotes(buf, v)
}
buf = append(buf, '\'')
default:
return "", driver.ErrSkip
}
if len(buf)+4 > mc.maxAllowedPacket {
return "", driver.ErrSkip
}
}
if argPos != len(args) {
return "", driver.ErrSkip
}
return string(buf), nil
}
func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) {
if mc.closed.IsSet() {
errLog.Print(ErrInvalidConn)
return nil, driver.ErrBadConn
}
if len(args) != 0 {
if !mc.cfg.InterpolateParams {
return nil, driver.ErrSkip
}
// try to interpolate the parameters to save extra roundtrips for preparing and closing a statement
prepared, err := mc.interpolateParams(query, args)
if err != nil {
return nil, err
}
query = prepared
}
mc.affectedRows = 0
mc.insertId = 0
err := mc.exec(query)
if err == nil {
return &mysqlResult{
affectedRows: int64(mc.affectedRows),
insertId: int64(mc.insertId),
}, err
}
return nil, mc.markBadConn(err)
}
// Internal function to execute commands
func (mc *mysqlConn) exec(query string) error {
// Send command
if err := mc.writeCommandPacketStr(comQuery, query); err != nil {
return mc.markBadConn(err)
}
// Read Result
resLen, err := mc.readResultSetHeaderPacket()
if err != nil {
return err
}
if resLen > 0 {
// columns
if err := mc.readUntilEOF(); err != nil {
return err
}
// rows
if err := mc.readUntilEOF(); err != nil {
return err
}
}
return mc.discardResults()
}
func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) {
return mc.query(query, args)
}
func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) {
if mc.closed.IsSet() {
errLog.Print(ErrInvalidConn)
return nil, driver.ErrBadConn
}
if len(args) != 0 {
if !mc.cfg.InterpolateParams {
return nil, driver.ErrSkip
}
// try client-side prepare to reduce roundtrip
prepared, err := mc.interpolateParams(query, args)
if err != nil {
return nil, err
}
query = prepared
}
// Send command
err := mc.writeCommandPacketStr(comQuery, query)
if err == nil {
// Read Result
var resLen int
resLen, err = mc.readResultSetHeaderPacket()
if err == nil {
rows := new(textRows)
rows.mc = mc
if resLen == 0 {
rows.rs.done = true
switch err := rows.NextResultSet(); err {
case nil, io.EOF:
return rows, nil
default:
return nil, err
}
}
// Columns
rows.rs.columns, err = mc.readColumns(resLen)
return rows, err
}
}
return nil, mc.markBadConn(err)
}
// Gets the value of the given MySQL System Variable
// The returned byte slice is only valid until the next read
func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
// Send command
if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil {
return nil, err
}
// Read Result
resLen, err := mc.readResultSetHeaderPacket()
if err == nil {
rows := new(textRows)
rows.mc = mc
rows.rs.columns = []mysqlField{{fieldType: fieldTypeVarChar}}
if resLen > 0 {
// Columns
if err := mc.readUntilEOF(); err != nil {
return nil, err
}
}
dest := make([]driver.Value, resLen)
if err = rows.readRow(dest); err == nil {
return dest[0].([]byte), mc.readUntilEOF()
}
}
return nil, err
}
// finish is called when the query has canceled.
func (mc *mysqlConn) cancel(err error) {
mc.canceled.Set(err)
mc.cleanup()
}
// finish is called when the query has succeeded.
func (mc *mysqlConn) finish() {
if !mc.watching || mc.finished == nil {
return
}
select {
case mc.finished <- struct{}{}:
mc.watching = false
case <-mc.closech:
}
}

Some files were not shown because too many files have changed in this diff Show More