TODO: hook Download

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require github.com/tidwall/gjson v1.14.1
 
 require (
+	github.com/go-rod/rod v0.107.2 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
@@ -13,6 +14,9 @@ require (
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	github.com/ysmood/goob v0.4.0 // indirect
+	github.com/ysmood/gson v0.7.1 // indirect
+	github.com/ysmood/leakless v0.7.0 // indirect
 	go.mongodb.org/mongo-driver v1.9.1 // indirect
 	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect

go.sum

@@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/elazarl/goproxy/ext v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/go-rod/rod v0.107.2 h1:T2nSPKBrHuM/nhGPS/rl9Uwz0dI7T2M9xYF5owqxjMQ=
+github.com/go-rod/rod v0.107.2/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
@@ -40,6 +42,14 @@ github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyh
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
+github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
+github.com/ysmood/got v0.29.5/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
+github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
+github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c=
 go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

main.go

@@ -5,8 +5,11 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/gob"
+	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
 	"regexp"
 	"strconv"
@@ -14,6 +17,10 @@ import (
 	"time"
 
 	"github.com/474420502/gcurl"
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/devices"
+	"github.com/go-rod/rod/lib/launcher"
+	"github.com/go-rod/rod/lib/proto"
 	"github.com/tidwall/gjson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
@@ -41,9 +48,33 @@ type Stock struct {
 	Code int `json:"股票数字代码" bson:"股票数字代码"`
 }
 
-type StockCode struct {
+type StockBase struct {
-	CodeStr string // 代地区码
+	// CodeStr string // 代地区码
-	Code    string // 不带地区码
+	// Code    string // 不带地区码
+
+	CODE        string  `json:"CODE"`
+	FIVE_MINUTE float64 `json:"FIVE_MINUTE"`
+	HIGH        float64 `json:"HIGH"`
+	HS          float64 `json:"HS"`
+	LB          float64 `json:"LB"`
+	LOW         float64 `json:"LOW"`
+	MCAP        float64 `json:"MCAP"`
+	MFSUM       float64 `json:"MFSUM"`
+	NAME        string  `json:"NAME"`
+	OPEN        float64 `json:"OPEN"`
+	PE          float64 `json:"PE"`
+	PERCENT     float64 `json:"PERCENT"`
+	PRICE       float64 `json:"PRICE"`
+	SNAME       string  `json:"SNAME"`
+	SYMBOL      string  `json:"SYMBOL"`
+	TCAP        float64 `json:"TCAP"`
+	TURNOVER    float64 `json:"TURNOVER"`
+	UPDOWN      float64 `json:"UPDOWN"`
+	VOLUME      float64 `json:"VOLUME"`
+	WB          float64 `json:"WB"`
+	YESTCLOSE   float64 `json:"YESTCLOSE"`
+	ZF          float64 `json:"ZF"`
+	NO          float64 `json:"NO"`
 }
 
 func main() {
@@ -52,8 +83,8 @@ func main() {
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
-	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
+	// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
 	if err != nil {
 		panic(err)
 	}
@@ -71,7 +102,7 @@ func main() {
 	tp := gcurl.Parse(murl).Temporary()
 	page := tp.QueryParam(`page=\d+`)
 	var stockCodesFile = "./stock_codes.gob"
-	var stockCodes []StockCode
+	var stockCodes []*StockBase
 	f, err := os.Open(stockCodesFile)
 	if err == nil {
 		err = gob.NewDecoder(f).Decode(&stockCodes)
@@ -87,11 +118,20 @@ func main() {
 			if err != nil {
 				panic(err)
 			}
+
 			jr := gjson.ParseBytes(resp.Content())
 			pagecount = jr.Get("pagecount").Int()
-			for _, ljr := range jr.Get("list").Array() {
+
-				stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str})
+			for _, s := range jr.Get("list").Array() {
+				var stockCode StockBase
+				err = json.Unmarshal([]byte(s.String()), &stockCode)
+				if err != nil {
+					panic(err)
+				}
+
+				stockCodes = append(stockCodes, &stockCode)
 			}
+
 			// log.Println(jr.String())
 		}
 		f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
@@ -112,23 +152,79 @@ func main() {
 	}
 
 	for _, code := range stockCodes {
-		DownloadDataFromCode(client, &code)
+		if code.MCAP >= 50000000000 {
+			DownloadDataFromCode(client, code)
+		}
 	}
 }
 
-func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
+func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
 	// 300731
-	durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
+	durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
-		-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
+	-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
-		-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
+	-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
-		-H 'Connection: keep-alive' \
+	-H 'Connection: keep-alive' \
-		-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
+	-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; s_n_f_l_n3=90474b666b6678eb1655828892281; pgr_n_f_l_n3=90474b666b6678eb16558271774556486; vinfo_n_f_l_n3=90474b666b6678eb.1.6.1655737842842.1655828160869.1655828914287' \
-		-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \
+	-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${SYMBOL}.html' \
-		-H 'Upgrade-Insecure-Requests: 1' \
+	-H 'Upgrade-Insecure-Requests: 1' \
-		-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'`
+	-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' `
 
-	durl = strings.ReplaceAll(durl, `${code}`, code.Code)
+	// http://quotes.money.163.com/0601988.html
-	durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr)
+
+	screen := devices.Device{
+		Title:        "Laptop with MDPI screen",
+		Capabilities: []string{"touch", "mobile"},
+		UserAgent:    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
+		Screen: devices.Screen{
+			DevicePixelRatio: 1,
+			Horizontal: devices.ScreenSize{
+				Width:  1920,
+				Height: 1080,
+			},
+		},
+	}
+	port := 40000
+	log.Println("get port:", port)
+	rodlauncher := launcher.New().
+		Bin(`google-chrome`).
+		RemoteDebuggingPort(port).
+		Set("user-data-dir", fmt.Sprintf("/tmp/%s_rod", "money-money")).
+		Delete("headless")
+	//debug url
+	launchers := rodlauncher.MustLaunch()
+	fmt.Printf("debug url: %s\n", launchers)
+	//连接浏览器
+	browser := rod.New().ControlURL(launchers).MustConnect()
+	page := browser.DefaultDevice(screen).MustPage(fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE))
+
+	p := page.Timeout(time.Second * 5)
+	ele, err := p.ElementsX("//a[@href='/trade/lsjysj_600096.html#01b07']/@href")
+	if err != nil {
+		panic(err)
+	}
+
+	time.Sleep(time.Second * 3)
+
+	if iter := ele.First(); iter != nil {
+
+		urlpath, err := iter.HTML()
+		if err != nil {
+			panic(err)
+		}
+		log.Println(page.MustInfo().URL)
+		page.MustNavigate("http://quotes.money.163.com" + urlpath)
+
+		ahref := page.MustElementX("//a[@id='downloadData']")
+
+		ahref.Click(proto.InputMouseButtonLeft)
+		time.Sleep(time.Second)
+		e := page.MustElementX("//a[@class='blue_btn submit']")
+		e.Click(proto.InputMouseButtonLeft)
+
+	}
+
+	durl = strings.ReplaceAll(durl, `${SYMBOL}`, code.SYMBOL)
+	durl = strings.ReplaceAll(durl, `${CODE}`, code.CODE)
 
 	resp, err := gcurl.Parse(durl).Temporary().Execute()
 	if err != nil {
@@ -192,7 +288,7 @@ func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
 	if err != nil {
 
 	}
-	log.Println(code.Code, r)
+	log.Println(code.CODE, r)
 	time.Sleep(time.Second * 2)
 }
 
@@ -215,3 +311,15 @@ func GbkToUtf8(s []byte) ([]byte, error) {
 	}
 	return d, nil
 }
+
+func GetPort() int {
+	l, _ := net.Listen("tcp", ":0") // listen on localhost
+	port := l.Addr().(*net.TCPAddr).Port
+	err := l.Close()
+	if err != nil {
+		panic(err)
+	}
+	// ip := l.Addr().(*net.TCPAddr).IP
+	return port
+
+}