From eb99d0eb0d2d941a538c49e3ea8e0fd7b757f7e2 Mon Sep 17 00:00:00 2001 From: eson <474420502@qq.com> Date: Tue, 21 Jun 2022 02:17:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=9D=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 28 +++++++++ go.sum | 71 ++++++++++++++++++++++ main.go | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++ main_test.go | 7 +++ 4 files changed, 271 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 main_test.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7b152db --- /dev/null +++ b/go.mod @@ -0,0 +1,28 @@ +module moneymoney + +go 1.18 + +require github.com/tidwall/gjson v1.14.1 + +require ( + github.com/go-stack/stack v1.8.0 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.0.2 // indirect + github.com/xdg-go/stringprep v1.0.2 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.mongodb.org/mongo-driver v1.9.1 // indirect + golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect +) + +require ( + github.com/474420502/gcurl v0.5.1 // indirect + github.com/474420502/requests v1.11.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/net v0.0.0-20211105192438-b53810dc28af // indirect + golang.org/x/text v0.3.7 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3705a2d --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +github.com/474420502/gcurl v0.5.1 h1:Rpl0qdffIwgzOivc4BAn8biOpoGIFhOBAFe8vogswIk= +github.com/474420502/gcurl v0.5.1/go.mod h1:bYc/86ynsx28WNFd6MDt/8gy0xRvuqhspmEqEm7Sybo= +github.com/474420502/requests v1.11.1 h1:BLI95klIGRjTLeQ7U7CJLkkniVsL2tK0PLJnUTSTyfY= +github.com/474420502/requests v1.11.1/go.mod h1:epMXb90Z7hmiOu+hMLFj8eFbkqOXWThNDYHuh2ThTlE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/elazarl/goproxy/ext v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c= +go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20211105192438-b53810dc28af h1:SMeNJG/vclJ5wyBBd4xupMsSJIHTd1coW9g7q6KOjmY= +golang.org/x/net v0.0.0-20211105192438-b53810dc28af/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..a1df3ec --- /dev/null +++ b/main.go @@ -0,0 +1,165 @@ +package moneymoney + +import ( + "bytes" + "context" + "encoding/csv" + "io/ioutil" + "log" + "regexp" + "strconv" + "time" + + "github.com/474420502/gcurl" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" +) + +type Stock struct { + Date string `json:"日期" bson:"日期"` + CodeStr string `json:"股票代码" bson:"股票代码"` + Name string `json:"名称" bson:"名称"` + ClosingPrice float64 `json:"收盘价" bson:"收盘价"` + MaxPrice float64 `json:"最高价" bson:"最高价"` + MinPrice float64 `json:"最低价" bson:"最低价"` + OpeningPrice float64 `json:"开盘价" bson:"开盘价"` + PreviousClosingPrice float64 `json:"前收盘" bson:"前收盘"` + UpsDowns float64 `json:"涨跌额" bson:"涨跌额"` + UpsDownsRatio float64 `json:"涨跌幅" bson:"涨跌幅"` + TurnoverRate float64 `json:"换手率" bson:"换手率"` + Volume float64 `json:"成交量" bson:"成交量"` + Turnover float64 `json:"成交金额" bson:"成交金额"` + MarketValue float64 `json:"总市值" bson:"总市值"` + CirculatingMarketValue float64 `json:"流通市值" bson:"流通市值"` + + Code int `json:"股票数字代码" bson:"股票数字代码"` +} + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017")) + + // murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=24&type=query' \ + // -H 'Accept: application/json, text/javascript, */*; q=0.01' \ + // -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ + // -H 'Connection: keep-alive' \ + // -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \ + // -H 'Referer: http://quotes.money.163.com/old/' \ + // -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \ + // -H 'X-Requested-With: XMLHttpRequest' \ + // --compressed \ + // --insecure` + + // tp := gcurl.Parse(murl).Temporary() + // page := tp.QueryParam(`page=\d+`) + + // var i int64 = 0 + // for ; i < 1000; i++ { + // page.IntSet(i) + // resp, err := tp.Execute() + // if err != nil { + // panic(err) + // } + + // log.Println(string(resp.Content())) + // } + + durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=1300731&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ + -H 'Connection: keep-alive' \ + -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \ + -H 'Referer: http://quotes.money.163.com/trade/lsjysj_300731.html' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \ + --compressed \ + --insecure` + + resp, err := gcurl.Parse(durl).Temporary().Execute() + if err != nil { + panic(err) + } + + reader := csv.NewReader(bytes.NewBuffer(resp.Content())) + alls, err := reader.ReadAll() + if err != nil { + panic(err) + } + + var jfield []string + for _, field := range alls[0] { + v, _ := GbkToUtf8([]byte(field)) + jfield = append(jfield, string(v)) + log.Printf("%#v", string(v)) + } + + re, _ := regexp.Compile(`\d+`) + + var stocks []mongo.WriteModel + for _, line := range alls[1:] { + + var fields []string + for _, field := range line { + v, _ := GbkToUtf8([]byte(field)) + fields = append(fields, string(v)) + + } + + code, err := strconv.Atoi(re.FindString(fields[1])) + if err != nil { + panic(err) + } + + s := &Stock{ + Date: fields[0], + CodeStr: fields[1], + Name: fields[2], + ClosingPrice: ToFloat(fields[3]), + MaxPrice: ToFloat(fields[4]), + MinPrice: ToFloat(fields[5]), + OpeningPrice: ToFloat(fields[6]), + PreviousClosingPrice: ToFloat(fields[7]), + UpsDowns: ToFloat(fields[8]), + UpsDownsRatio: ToFloat(fields[9]), + TurnoverRate: ToFloat(fields[10]), + Volume: ToFloat(fields[11]), + Turnover: ToFloat(fields[12]), + MarketValue: ToFloat(fields[13]), + CirculatingMarketValue: ToFloat(fields[14]), + + Code: code, + } + stocks = append(stocks, &mongo.InsertOneModel{Document: s}) + } + + cstock := client.Database("money").Collection("stock") + r, err := cstock.BulkWrite(context.TODO(), stocks) + if err != nil { + + } + log.Println(r) + +} + +func ToFloat(s string) float64 { + if s == "None" { + return 0 + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + panic(err) + } + return v +} + +func GbkToUtf8(s []byte) ([]byte, error) { + reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder()) + d, e := ioutil.ReadAll(reader) + if e != nil { + return nil, e + } + return d, nil +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..b8ef46e --- /dev/null +++ b/main_test.go @@ -0,0 +1,7 @@ +package moneymoney + +import "testing" + +func TestCase1(t *testing.T) { + main() +}