新版本
This commit is contained in:
eson
parent
eb99d0eb0d
commit
9dafc9d6d9
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
*.json
|
||||
122
main.go
122
main.go
|
|
@ -4,13 +4,17 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/gob"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
|
|
@ -37,46 +41,94 @@ type Stock struct {
|
|||
Code int `json:"股票数字代码" bson:"股票数字代码"`
|
||||
}
|
||||
|
||||
type StockCode struct {
|
||||
CodeStr string // 代地区码
|
||||
Code string // 不带地区码
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
log.SetFlags(log.Llongfile | log.LstdFlags)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
|
||||
// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \
|
||||
-H 'Accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
|
||||
-H 'Referer: http://quotes.money.163.com/old/' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
|
||||
-H 'X-Requested-With: XMLHttpRequest' \
|
||||
--compressed \
|
||||
--insecure`
|
||||
|
||||
// murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=24&type=query' \
|
||||
// -H 'Accept: application/json, text/javascript, */*; q=0.01' \
|
||||
// -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
// -H 'Connection: keep-alive' \
|
||||
// -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
|
||||
// -H 'Referer: http://quotes.money.163.com/old/' \
|
||||
// -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
|
||||
// -H 'X-Requested-With: XMLHttpRequest' \
|
||||
// --compressed \
|
||||
// --insecure`
|
||||
tp := gcurl.Parse(murl).Temporary()
|
||||
page := tp.QueryParam(`page=\d+`)
|
||||
var stockCodesFile = "./stock_codes.gob"
|
||||
var stockCodes []StockCode
|
||||
f, err := os.Open(stockCodesFile)
|
||||
if err == nil {
|
||||
err = gob.NewDecoder(f).Decode(&stockCodes)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
} else {
|
||||
var i int64 = 0
|
||||
var pagecount int64 = 100
|
||||
for ; i < pagecount; i++ {
|
||||
page.IntSet(i)
|
||||
resp, err := tp.Execute()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
jr := gjson.ParseBytes(resp.Content())
|
||||
pagecount = jr.Get("pagecount").Int()
|
||||
for _, ljr := range jr.Get("list").Array() {
|
||||
stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str})
|
||||
}
|
||||
// log.Println(jr.String())
|
||||
}
|
||||
f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = gob.NewEncoder(f).Encode(&stockCodes)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// tp := gcurl.Parse(murl).Temporary()
|
||||
// page := tp.QueryParam(`page=\d+`)
|
||||
if f != nil {
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// var i int64 = 0
|
||||
// for ; i < 1000; i++ {
|
||||
// page.IntSet(i)
|
||||
// resp, err := tp.Execute()
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
for _, code := range stockCodes {
|
||||
DownloadDataFromCode(client, &code)
|
||||
}
|
||||
}
|
||||
|
||||
// log.Println(string(resp.Content()))
|
||||
// }
|
||||
func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
|
||||
// 300731
|
||||
durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
|
||||
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
|
||||
-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \
|
||||
-H 'Upgrade-Insecure-Requests: 1' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'`
|
||||
|
||||
durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=1300731&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
|
||||
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
|
||||
-H 'Referer: http://quotes.money.163.com/trade/lsjysj_300731.html' \
|
||||
-H 'Upgrade-Insecure-Requests: 1' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
|
||||
--compressed \
|
||||
--insecure`
|
||||
durl = strings.ReplaceAll(durl, `${code}`, code.Code)
|
||||
durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr)
|
||||
|
||||
resp, err := gcurl.Parse(durl).Temporary().Execute()
|
||||
if err != nil {
|
||||
|
|
@ -93,7 +145,7 @@ func main() {
|
|||
for _, field := range alls[0] {
|
||||
v, _ := GbkToUtf8([]byte(field))
|
||||
jfield = append(jfield, string(v))
|
||||
log.Printf("%#v", string(v))
|
||||
// log.Printf("%#v", string(v))
|
||||
}
|
||||
|
||||
re, _ := regexp.Compile(`\d+`)
|
||||
|
|
@ -140,8 +192,8 @@ func main() {
|
|||
if err != nil {
|
||||
|
||||
}
|
||||
log.Println(r)
|
||||
|
||||
log.Println(code.Code, r)
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
|
||||
func ToFloat(s string) float64 {
|
||||
|
|
|
|||
BIN
my.json.7z
Normal file
BIN
my.json.7z
Normal file
Binary file not shown.
BIN
stock_codes.gob
Normal file
BIN
stock_codes.gob
Normal file
Binary file not shown.
Loading…
Reference in New Issue
Block a user