diff --git a/go.mod b/go.mod index 7b152db..8e02f77 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require github.com/tidwall/gjson v1.14.1 require ( + github.com/go-rod/rod v0.107.2 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/klauspost/compress v1.13.6 // indirect @@ -13,6 +14,9 @@ require ( github.com/xdg-go/scram v1.0.2 // indirect github.com/xdg-go/stringprep v1.0.2 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/ysmood/goob v0.4.0 // indirect + github.com/ysmood/gson v0.7.1 // indirect + github.com/ysmood/leakless v0.7.0 // indirect go.mongodb.org/mongo-driver v1.9.1 // indirect golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect diff --git a/go.sum b/go.sum index 3705a2d..167a093 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/elazarl/goproxy/ext v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/go-rod/rod v0.107.2 h1:T2nSPKBrHuM/nhGPS/rl9Uwz0dI7T2M9xYF5owqxjMQ= +github.com/go-rod/rod v0.107.2/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -40,6 +42,14 @@ github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyh github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/got v0.29.5/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q= +github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw= +github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c= go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/main.go b/main.go index f2e92c7..b78b61f 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,11 @@ import ( "context" "encoding/csv" "encoding/gob" + "encoding/json" + "fmt" "io/ioutil" "log" + "net" "os" "regexp" "strconv" @@ -14,6 +17,10 @@ import ( "time" "github.com/474420502/gcurl" + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/devices" + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" "github.com/tidwall/gjson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -41,9 +48,33 @@ type Stock struct { Code int `json:"股票数字代码" bson:"股票数字代码"` } -type StockCode struct { - CodeStr string // 代地区码 - Code string // 不带地区码 +type StockBase struct { + // CodeStr string // 代地区码 + // Code string // 不带地区码 + + CODE string `json:"CODE"` + FIVE_MINUTE float64 `json:"FIVE_MINUTE"` + HIGH float64 `json:"HIGH"` + HS float64 `json:"HS"` + LB float64 `json:"LB"` + LOW float64 `json:"LOW"` + MCAP float64 `json:"MCAP"` + MFSUM float64 `json:"MFSUM"` + NAME string `json:"NAME"` + OPEN float64 `json:"OPEN"` + PE float64 `json:"PE"` + PERCENT float64 `json:"PERCENT"` + PRICE float64 `json:"PRICE"` + SNAME string `json:"SNAME"` + SYMBOL string `json:"SYMBOL"` + TCAP float64 `json:"TCAP"` + TURNOVER float64 `json:"TURNOVER"` + UPDOWN float64 `json:"UPDOWN"` + VOLUME float64 `json:"VOLUME"` + WB float64 `json:"WB"` + YESTCLOSE float64 `json:"YESTCLOSE"` + ZF float64 `json:"ZF"` + NO float64 `json:"NO"` } func main() { @@ -52,8 +83,8 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - // client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017")) - client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017")) + client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017")) + // client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017")) if err != nil { panic(err) } @@ -71,7 +102,7 @@ func main() { tp := gcurl.Parse(murl).Temporary() page := tp.QueryParam(`page=\d+`) var stockCodesFile = "./stock_codes.gob" - var stockCodes []StockCode + var stockCodes []*StockBase f, err := os.Open(stockCodesFile) if err == nil { err = gob.NewDecoder(f).Decode(&stockCodes) @@ -87,11 +118,20 @@ func main() { if err != nil { panic(err) } + jr := gjson.ParseBytes(resp.Content()) pagecount = jr.Get("pagecount").Int() - for _, ljr := range jr.Get("list").Array() { - stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str}) + + for _, s := range jr.Get("list").Array() { + var stockCode StockBase + err = json.Unmarshal([]byte(s.String()), &stockCode) + if err != nil { + panic(err) + } + + stockCodes = append(stockCodes, &stockCode) } + // log.Println(jr.String()) } f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664) @@ -112,23 +152,79 @@ func main() { } for _, code := range stockCodes { - DownloadDataFromCode(client, &code) + if code.MCAP >= 50000000000 { + DownloadDataFromCode(client, code) + } } } -func DownloadDataFromCode(client *mongo.Client, code *StockCode) { +func DownloadDataFromCode(client *mongo.Client, code *StockBase) { // 300731 - durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ - -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ - -H 'Connection: keep-alive' \ - -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \ - -H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'` + durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ + -H 'Connection: keep-alive' \ + -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; s_n_f_l_n3=90474b666b6678eb1655828892281; pgr_n_f_l_n3=90474b666b6678eb16558271774556486; vinfo_n_f_l_n3=90474b666b6678eb.1.6.1655737842842.1655828160869.1655828914287' \ + -H 'Referer: http://quotes.money.163.com/trade/lsjysj_${SYMBOL}.html' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' ` - durl = strings.ReplaceAll(durl, `${code}`, code.Code) - durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr) + // http://quotes.money.163.com/0601988.html + + screen := devices.Device{ + Title: "Laptop with MDPI screen", + Capabilities: []string{"touch", "mobile"}, + UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + Screen: devices.Screen{ + DevicePixelRatio: 1, + Horizontal: devices.ScreenSize{ + Width: 1920, + Height: 1080, + }, + }, + } + port := 40000 + log.Println("get port:", port) + rodlauncher := launcher.New(). + Bin(`google-chrome`). + RemoteDebuggingPort(port). + Set("user-data-dir", fmt.Sprintf("/tmp/%s_rod", "money-money")). + Delete("headless") + //debug url + launchers := rodlauncher.MustLaunch() + fmt.Printf("debug url: %s\n", launchers) + //连接浏览器 + browser := rod.New().ControlURL(launchers).MustConnect() + page := browser.DefaultDevice(screen).MustPage(fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE)) + + p := page.Timeout(time.Second * 5) + ele, err := p.ElementsX("//a[@href='/trade/lsjysj_600096.html#01b07']/@href") + if err != nil { + panic(err) + } + + time.Sleep(time.Second * 3) + + if iter := ele.First(); iter != nil { + + urlpath, err := iter.HTML() + if err != nil { + panic(err) + } + log.Println(page.MustInfo().URL) + page.MustNavigate("http://quotes.money.163.com" + urlpath) + + ahref := page.MustElementX("//a[@id='downloadData']") + + ahref.Click(proto.InputMouseButtonLeft) + time.Sleep(time.Second) + e := page.MustElementX("//a[@class='blue_btn submit']") + e.Click(proto.InputMouseButtonLeft) + + } + + durl = strings.ReplaceAll(durl, `${SYMBOL}`, code.SYMBOL) + durl = strings.ReplaceAll(durl, `${CODE}`, code.CODE) resp, err := gcurl.Parse(durl).Temporary().Execute() if err != nil { @@ -192,7 +288,7 @@ func DownloadDataFromCode(client *mongo.Client, code *StockCode) { if err != nil { } - log.Println(code.Code, r) + log.Println(code.CODE, r) time.Sleep(time.Second * 2) } @@ -215,3 +311,15 @@ func GbkToUtf8(s []byte) ([]byte, error) { } return d, nil } + +func GetPort() int { + l, _ := net.Listen("tcp", ":0") // listen on localhost + port := l.Addr().(*net.TCPAddr).Port + err := l.Close() + if err != nil { + panic(err) + } + // ip := l.Addr().(*net.TCPAddr).IP + return port + +} diff --git a/stock_codes.gob b/stock_codes.gob index 727f08a..0b4d0fd 100644 Binary files a/stock_codes.gob and b/stock_codes.gob differ