save
This commit is contained in:
parent
2c26cae619
commit
48b62576c4
10
go.mod
10
go.mod
|
@ -2,10 +2,13 @@ module moneymoney
|
|||
|
||||
go 1.18
|
||||
|
||||
require github.com/tidwall/gjson v1.14.1
|
||||
require (
|
||||
github.com/go-rod/rod v0.107.2
|
||||
github.com/tidwall/gjson v1.14.1
|
||||
go.mongodb.org/mongo-driver v1.9.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/go-rod/rod v0.107.2 // indirect
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/golang/snappy v0.0.1 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
|
@ -17,13 +20,12 @@ require (
|
|||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/gson v0.7.1 // indirect
|
||||
github.com/ysmood/leakless v0.7.0 // indirect
|
||||
go.mongodb.org/mongo-driver v1.9.1 // indirect
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/474420502/gcurl v0.5.1 // indirect
|
||||
github.com/474420502/gcurl v0.5.1
|
||||
github.com/474420502/requests v1.11.1 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
|
|
9
go.sum
9
go.sum
|
@ -3,7 +3,9 @@ github.com/474420502/gcurl v0.5.1/go.mod h1:bYc/86ynsx28WNFd6MDt/8gy0xRvuqhspmEq
|
|||
github.com/474420502/requests v1.11.1 h1:BLI95klIGRjTLeQ7U7CJLkkniVsL2tK0PLJnUTSTyfY=
|
||||
github.com/474420502/requests v1.11.1/go.mod h1:epMXb90Z7hmiOu+hMLFj8eFbkqOXWThNDYHuh2ThTlE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4 h1:lS3P5Nw3oPO05Lk2gFiYUOL3QPaH+fRoI1wFOc4G1UY=
|
||||
github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
|
||||
|
@ -13,6 +15,7 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
|||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
|
@ -22,9 +25,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
|||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
|
||||
|
@ -44,7 +49,9 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM
|
|||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/got v0.29.5 h1:+wMnm8UjoyYFMfeAsr57a1bahWTkloysc0Hxsu2gmnM=
|
||||
github.com/ysmood/got v0.29.5/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
|
||||
github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
|
@ -75,7 +82,9 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
|||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
256
main.go
256
main.go
|
@ -13,7 +13,6 @@ import (
|
|||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
|
@ -22,6 +21,7 @@ import (
|
|||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/tidwall/gjson"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
|
@ -77,17 +77,78 @@ type StockBase struct {
|
|||
NO float64 `json:"NO"`
|
||||
}
|
||||
|
||||
var DefaultPage *rod.Page
|
||||
|
||||
func GetDefaultPage() *rod.Page {
|
||||
|
||||
if DefaultPage != nil {
|
||||
return DefaultPage
|
||||
}
|
||||
|
||||
screen := devices.Device{
|
||||
Title: "Laptop with MDPI screen",
|
||||
Capabilities: []string{"touch", "mobile"},
|
||||
UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
|
||||
Screen: devices.Screen{
|
||||
DevicePixelRatio: 1,
|
||||
Horizontal: devices.ScreenSize{
|
||||
Width: 1920,
|
||||
Height: 1080,
|
||||
},
|
||||
},
|
||||
}
|
||||
port := GetPort()
|
||||
log.Println("get port:", port)
|
||||
rodlauncher := launcher.New().
|
||||
Bin(`google-chrome`).
|
||||
RemoteDebuggingPort(port).
|
||||
Set("user-data-dir", fmt.Sprintf("/tmp/%s_rod", "money-money")).
|
||||
Delete("headless")
|
||||
|
||||
//debug url
|
||||
launchers := rodlauncher.MustLaunch()
|
||||
fmt.Printf("debug url: %s\n", launchers)
|
||||
//连接浏览器
|
||||
browser := rod.New().ControlURL(launchers).MustConnect()
|
||||
page := browser.DefaultDevice(screen).MustPage()
|
||||
|
||||
// p := page.Timeout(time.Second * 15)
|
||||
DefaultPage = page
|
||||
return DefaultPage
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
log.SetFlags(log.Llongfile | log.LstdFlags)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
|
||||
// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
|
||||
// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cur, err := client.Database("money").Collection("stock").Aggregate(context.TODO(), bson.A{
|
||||
bson.M{"$group": bson.M{"_id": "$股票数字代码"}},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var skipMap map[string]bool = make(map[string]bool)
|
||||
|
||||
for cur.Next(context.TODO()) {
|
||||
var doc bson.M
|
||||
err = cur.Decode(&doc)
|
||||
if err == nil {
|
||||
skipMap[strconv.Itoa(int(doc["_id"].(int32)))] = true
|
||||
} else {
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \
|
||||
-H 'Accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
|
@ -95,9 +156,7 @@ func main() {
|
|||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
|
||||
-H 'Referer: http://quotes.money.163.com/old/' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
|
||||
-H 'X-Requested-With: XMLHttpRequest' \
|
||||
--compressed \
|
||||
--insecure`
|
||||
-H 'X-Requested-With: XMLHttpRequest'`
|
||||
|
||||
tp := gcurl.Parse(murl).Temporary()
|
||||
page := tp.QueryParam(`page=\d+`)
|
||||
|
@ -152,6 +211,10 @@ func main() {
|
|||
}
|
||||
|
||||
for _, code := range stockCodes {
|
||||
if _, ok := skipMap[code.SYMBOL]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if code.MCAP >= 50000000000 {
|
||||
DownloadDataFromCode(client, code)
|
||||
}
|
||||
|
@ -160,136 +223,119 @@ func main() {
|
|||
|
||||
func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
|
||||
// 300731
|
||||
durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
|
||||
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; s_n_f_l_n3=90474b666b6678eb1655828892281; pgr_n_f_l_n3=90474b666b6678eb16558271774556486; vinfo_n_f_l_n3=90474b666b6678eb.1.6.1655737842842.1655828160869.1655828914287' \
|
||||
-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${SYMBOL}.html' \
|
||||
-H 'Upgrade-Insecure-Requests: 1' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' `
|
||||
// durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
|
||||
// -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
|
||||
// -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
// -H 'Connection: keep-alive' \
|
||||
// -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; s_n_f_l_n3=90474b666b6678eb1655828892281; pgr_n_f_l_n3=90474b666b6678eb16558271774556486; vinfo_n_f_l_n3=90474b666b6678eb.1.6.1655737842842.1655828160869.1655828914287' \
|
||||
// -H 'Referer: http://quotes.money.163.com/trade/lsjysj_${SYMBOL}.html' \
|
||||
// -H 'Upgrade-Insecure-Requests: 1' \
|
||||
// -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' `
|
||||
|
||||
// http://quotes.money.163.com/0601988.html
|
||||
|
||||
screen := devices.Device{
|
||||
Title: "Laptop with MDPI screen",
|
||||
Capabilities: []string{"touch", "mobile"},
|
||||
UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
|
||||
Screen: devices.Screen{
|
||||
DevicePixelRatio: 1,
|
||||
Horizontal: devices.ScreenSize{
|
||||
Width: 1920,
|
||||
Height: 1080,
|
||||
},
|
||||
},
|
||||
}
|
||||
port := 40000
|
||||
log.Println("get port:", port)
|
||||
rodlauncher := launcher.New().
|
||||
Bin(`google-chrome`).
|
||||
RemoteDebuggingPort(port).
|
||||
Set("user-data-dir", fmt.Sprintf("/tmp/%s_rod", "money-money")).
|
||||
Delete("headless")
|
||||
//debug url
|
||||
launchers := rodlauncher.MustLaunch()
|
||||
fmt.Printf("debug url: %s\n", launchers)
|
||||
//连接浏览器
|
||||
browser := rod.New().ControlURL(launchers).MustConnect()
|
||||
page := browser.DefaultDevice(screen).MustPage(fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE))
|
||||
page := GetDefaultPage()
|
||||
stockurl := fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE)
|
||||
log.Println(stockurl)
|
||||
|
||||
p := page.Timeout(time.Second * 5)
|
||||
ele, err := p.ElementsX("//a[@href='/trade/lsjysj_600096.html#01b07']/@href")
|
||||
page.Navigate(stockurl)
|
||||
page.WaitNavigation(proto.PageLifecycleEventNameFirstContentfulPaint)()
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
|
||||
ele, err := page.ElementsX(fmt.Sprintf("//a[contains(@href,'/trade/lsjysj_%s')]/@href", code.SYMBOL))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Second * 3)
|
||||
|
||||
if iter := ele.First(); iter != nil {
|
||||
iter.WaitEnabled()
|
||||
|
||||
urlpath, err := iter.HTML()
|
||||
log.Println("click", urlpath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.Println(page.MustInfo().URL)
|
||||
page.MustNavigate("http://quotes.money.163.com" + urlpath)
|
||||
|
||||
ahref := page.MustElementX("//a[@id='downloadData']")
|
||||
ahref.WaitEnabled()
|
||||
|
||||
ahref.Click(proto.InputMouseButtonLeft)
|
||||
time.Sleep(time.Second)
|
||||
e := page.MustElementX("//a[@class='blue_btn submit']")
|
||||
e.WaitEnabled()
|
||||
|
||||
w := page.Browser().MustWaitDownload()
|
||||
e.Click(proto.InputMouseButtonLeft)
|
||||
downloaddata := w()
|
||||
log.Println(len(downloaddata))
|
||||
|
||||
}
|
||||
// durl = strings.ReplaceAll(durl, `${SYMBOL}`, code.SYMBOL)
|
||||
// durl = strings.ReplaceAll(durl, `${CODE}`, code.CODE)
|
||||
|
||||
durl = strings.ReplaceAll(durl, `${SYMBOL}`, code.SYMBOL)
|
||||
durl = strings.ReplaceAll(durl, `${CODE}`, code.CODE)
|
||||
// resp, err := gcurl.Parse(durl).Temporary().Execute()
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
|
||||
resp, err := gcurl.Parse(durl).Temporary().Execute()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
reader := csv.NewReader(bytes.NewBuffer(resp.Content()))
|
||||
alls, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var jfield []string
|
||||
for _, field := range alls[0] {
|
||||
v, _ := GbkToUtf8([]byte(field))
|
||||
jfield = append(jfield, string(v))
|
||||
// log.Printf("%#v", string(v))
|
||||
}
|
||||
|
||||
re, _ := regexp.Compile(`\d+`)
|
||||
|
||||
var stocks []mongo.WriteModel
|
||||
for _, line := range alls[1:] {
|
||||
|
||||
var fields []string
|
||||
for _, field := range line {
|
||||
v, _ := GbkToUtf8([]byte(field))
|
||||
fields = append(fields, string(v))
|
||||
|
||||
}
|
||||
|
||||
code, err := strconv.Atoi(re.FindString(fields[1]))
|
||||
reader := csv.NewReader(bytes.NewBuffer(downloaddata))
|
||||
alls, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
s := &Stock{
|
||||
Date: fields[0],
|
||||
CodeStr: fields[1],
|
||||
Name: fields[2],
|
||||
ClosingPrice: ToFloat(fields[3]),
|
||||
MaxPrice: ToFloat(fields[4]),
|
||||
MinPrice: ToFloat(fields[5]),
|
||||
OpeningPrice: ToFloat(fields[6]),
|
||||
PreviousClosingPrice: ToFloat(fields[7]),
|
||||
UpsDowns: ToFloat(fields[8]),
|
||||
UpsDownsRatio: ToFloat(fields[9]),
|
||||
TurnoverRate: ToFloat(fields[10]),
|
||||
Volume: ToFloat(fields[11]),
|
||||
Turnover: ToFloat(fields[12]),
|
||||
MarketValue: ToFloat(fields[13]),
|
||||
CirculatingMarketValue: ToFloat(fields[14]),
|
||||
|
||||
Code: code,
|
||||
var jfield []string
|
||||
for _, field := range alls[0] {
|
||||
v, _ := GbkToUtf8([]byte(field))
|
||||
jfield = append(jfield, string(v))
|
||||
// log.Printf("%#v", string(v))
|
||||
}
|
||||
stocks = append(stocks, &mongo.InsertOneModel{Document: s})
|
||||
}
|
||||
|
||||
cstock := client.Database("money").Collection("stock")
|
||||
r, err := cstock.BulkWrite(context.TODO(), stocks)
|
||||
if err != nil {
|
||||
re, _ := regexp.Compile(`\d+`)
|
||||
|
||||
var stocks []mongo.WriteModel
|
||||
for _, line := range alls[1:] {
|
||||
|
||||
var fields []string
|
||||
for _, field := range line {
|
||||
v, _ := GbkToUtf8([]byte(field))
|
||||
fields = append(fields, string(v))
|
||||
|
||||
}
|
||||
|
||||
code, err := strconv.Atoi(re.FindString(fields[1]))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
s := &Stock{
|
||||
Date: fields[0],
|
||||
CodeStr: fields[1],
|
||||
Name: fields[2],
|
||||
ClosingPrice: ToFloat(fields[3]),
|
||||
MaxPrice: ToFloat(fields[4]),
|
||||
MinPrice: ToFloat(fields[5]),
|
||||
OpeningPrice: ToFloat(fields[6]),
|
||||
PreviousClosingPrice: ToFloat(fields[7]),
|
||||
UpsDowns: ToFloat(fields[8]),
|
||||
UpsDownsRatio: ToFloat(fields[9]),
|
||||
TurnoverRate: ToFloat(fields[10]),
|
||||
Volume: ToFloat(fields[11]),
|
||||
Turnover: ToFloat(fields[12]),
|
||||
MarketValue: ToFloat(fields[13]),
|
||||
CirculatingMarketValue: ToFloat(fields[14]),
|
||||
|
||||
Code: code,
|
||||
}
|
||||
stocks = append(stocks, &mongo.InsertOneModel{Document: s})
|
||||
}
|
||||
|
||||
cstock := client.Database("money").Collection("stock")
|
||||
r, err := cstock.BulkWrite(context.TODO(), stocks)
|
||||
if err != nil {
|
||||
|
||||
}
|
||||
log.Println(code.CODE, r)
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
log.Println(code.CODE, r)
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
|
||||
func ToFloat(s string) float64 {
|
||||
|
|
Loading…
Reference in New Issue
Block a user