diff --git a/api.go b/api.go new file mode 100644 index 0000000..ee05756 --- /dev/null +++ b/api.go @@ -0,0 +1,28 @@ +package moneymoney + +import ( + "github.com/474420502/gcurl" +) + +func GetFromAPI() { + tp := gcurl.Parse(`curl 'http://quotes.money.163.com/service/chddata.html?code=0601398&start=20140720&end=20150508'`).Temporary() + pcode := tp.QueryParam("code") + start := tp.QueryParam("start") + end := tp.QueryParam("end") + + for _, code := range GetStocks() { + + if code.MCAP >= 20000000000 { + pcode.StringSet(code.CODE) + start.StringSet("20170101") + end.StringSet("20220622") + + resp, err := tp.Execute() + if err != nil { + panic(err) + } + SaveFromCSV(resp.Content()) + } + } + +} diff --git a/main.go b/main.go index d8520fa..ddcf2ad 100644 --- a/main.go +++ b/main.go @@ -29,23 +29,23 @@ import ( ) type Stock struct { - Date string `json:"Date" bson:"Date"` // 日期 - CodeStr string `json:"CodeStr" bson:"CodeStr"` // 股票代码 - Name string `json:"Name" bson:"Name"` // 名称 - ClosingPrice float64 `json:"ClosingPrice" bson:"ClosingPrice"` // 收盘价 - MaxPrice float64 `json:"MaxPrice" bson:"MaxPrice"` // 最高价 - MinPrice float64 `json:"MinPrice" bson:"MinPrice"` // 最低价 - OpeningPrice float64 `json:"OpeningPrice" bson:"OpeningPrice"` // 开盘价 - PreviousClosingPrice float64 `json:"PreviousClosingPrice" bson:"PreviousClosingPrice"` // 前收盘 - UpsDowns float64 `json:"UpsDowns" bson:"UpsDowns"` // 涨跌额 - UpsDownsRatio float64 `json:"UpsDownsRatio" bson:"UpsDownsRatio"` // 涨跌幅 - TurnoverRate float64 `json:"TurnoverRate" bson:"TurnoverRate"` // 换手率 - Volume float64 `json:"Volume" bson:"Volume"` // 成交量 - Turnover float64 `json:"Turnover" bson:"Turnover"` // 成交金额 - MarketValue float64 `json:"MarketValue" bson:"MarketValue"` // 总市值 - CirculatingMarketValue float64 `json:"CirculatingMarketValue" bson:"CirculatingMarketValue"` // 流通市值 + Date string `json:"日期" bson:"日期"` + CodeStr string `json:"股票代码" bson:"股票代码"` + Name string `json:"名称" bson:"名称"` + ClosingPrice float64 `json:"收盘价" bson:"收盘价"` + MaxPrice float64 `json:"最高价" bson:"最高价"` + MinPrice float64 `json:"最低价" bson:"最低价"` + OpeningPrice float64 `json:"开盘价" bson:"开盘价"` + PreviousClosingPrice float64 `json:"前收盘" bson:"前收盘"` + UpsDowns float64 `json:"涨跌额" bson:"涨跌额"` + UpsDownsRatio float64 `json:"涨跌幅" bson:"涨跌幅"` + TurnoverRate float64 `json:"换手率" bson:"换手率"` + Volume float64 `json:"成交量" bson:"成交量"` + Turnover float64 `json:"成交金额" bson:"成交金额"` + MarketValue float64 `json:"总市值" bson:"总市值"` + CirculatingMarketValue float64 `json:"流通市值" bson:"流通市值"` - Code string `json:"Code" bson:"Code"` // 股票数字代码 + Code int64 `json:"股票数字代码" bson:"股票数字代码"` } type StockBase struct { @@ -117,49 +117,52 @@ func GetDefaultPage() *rod.Page { return DefaultPage } -func main() { +var client *mongo.Client +var err error +func init() { log.SetFlags(log.Llongfile | log.LstdFlags) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + client, err = mongo.Connect(context.TODO(), options.Client().ApplyURI("mongodb://root:6601502@localhost:27017")) + if err != nil { + panic(err) + } +} + +func main() { + + for _, code := range GetStocks() { + + if code.MCAP >= 50000000000 { + DownloadDataFromCode(code) + } + + } +} + +func GetStocks() []*StockBase { + // client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017")) - client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017")) + + cur, err := client.Database("money").Collection("stock").Distinct(context.TODO(), "股票数字代码", bson.M{}) if err != nil { panic(err) } - cur, err := client.Database("money").Collection("stock").Aggregate(context.TODO(), bson.A{ - bson.M{"$group": bson.M{"_id": "$股票数字代码"}}, - }) - if err != nil { - panic(err) - } + var skipMap map[int64]bool = make(map[int64]bool) - var skipMap map[string]bool = make(map[string]bool) - - for cur.Next(context.TODO()) { - var doc bson.M - err = cur.Decode(&doc) + for _, idoc := range cur { + var scode = idoc.(int64) + // err = cur.Decode(&doc) if err == nil { - skipMap[strconv.Itoa(int(doc["_id"].(int32)))] = true + + skipMap[scode] = true } else { log.Panic(err) } } - murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \ - -H 'Accept: application/json, text/javascript, */*; q=0.01' \ - -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ - -H 'Connection: keep-alive' \ - -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \ - -H 'Referer: http://quotes.money.163.com/old/' \ - -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \ - -H 'X-Requested-With: XMLHttpRequest'` - - tp := gcurl.Parse(murl).Temporary() - page := tp.QueryParam(`page=\d+`) var stockCodesFile = "./stock_codes.gob" var stockCodes []*StockBase f, err := os.Open(stockCodesFile) @@ -169,38 +172,34 @@ func main() { panic(err) } } else { - var i int64 = 0 - var pagecount int64 = 100 - for ; i < pagecount; i++ { - page.IntSet(i) - resp, err := tp.Execute() + + murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?count=5000'` + tp := gcurl.Parse(murl).Temporary() + resp, err := tp.Execute() + if err != nil { + panic(err) + } + jr := gjson.ParseBytes(resp.Content()) + + for _, s := range jr.Get("list").Array() { + var stockCode StockBase + err = json.Unmarshal([]byte(s.String()), &stockCode) if err != nil { panic(err) } - jr := gjson.ParseBytes(resp.Content()) - pagecount = jr.Get("pagecount").Int() - - for _, s := range jr.Get("list").Array() { - var stockCode StockBase - err = json.Unmarshal([]byte(s.String()), &stockCode) - if err != nil { - panic(err) - } - - stockCodes = append(stockCodes, &stockCode) - } - - // log.Println(jr.String()) - } - f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664) - if err != nil { - panic(err) - } - err = gob.NewEncoder(f).Encode(&stockCodes) - if err != nil { - panic(err) + stockCodes = append(stockCodes, &stockCode) } + + // log.Println(jr.String()) + } + f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664) + if err != nil { + panic(err) + } + err = gob.NewEncoder(f).Encode(&stockCodes) + if err != nil { + panic(err) } if f != nil { @@ -210,18 +209,30 @@ func main() { } } + re, _ := regexp.Compile(`\d+`) + log.Println("stocks", len(stockCodes)) + var result []*StockBase for _, code := range stockCodes { - if _, ok := skipMap[code.SYMBOL]; ok { + scode, err := strconv.ParseInt(re.FindString(code.SYMBOL), 10, 64) + if err != nil { + panic(err) + } + if _, ok := skipMap[scode]; ok { continue } - if code.MCAP >= 50000000000 { - DownloadDataFromCode(client, code) - } + result = append(result, code) + + // if code.MCAP >= 50000000000 { + // DownloadDataFromCode(client, code) + // } + } + + return result } -func DownloadDataFromCode(client *mongo.Client, code *StockBase) { +func DownloadDataFromCode(code *StockBase) { // 300731 // durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \ // -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ @@ -234,8 +245,6 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) { // http://quotes.money.163.com/0601988.html - cstock := client.Database("money").Collection("stock") - page := GetDefaultPage() stockurl := fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE) log.Println(stockurl) @@ -298,7 +307,7 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) { re, _ := regexp.Compile(`\d+`) - // var stocks []mongo.WriteModel + var stocks []mongo.WriteModel for _, line := range alls[1:] { var fields []string @@ -308,9 +317,12 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) { } - code := re.FindString(fields[1]) + code, err := strconv.ParseInt(re.FindString(fields[1]), 10, 64) + if err != nil { + panic(err) + } - s := Stock{ + s := &Stock{ Date: fields[0], CodeStr: fields[1], Name: fields[2], @@ -329,15 +341,80 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) { Code: code, } - - cstock.InsertOne(context.TODO(), mongo.InsertOneModel{Document: s}) + stocks = append(stocks, &mongo.InsertOneModel{Document: s}) } - log.Println(code.SYMBOL) + cstock := client.Database("money").Collection("stock") + r, err := cstock.BulkWrite(context.TODO(), stocks) + if err != nil { + log.Println(err) + } + log.Println(code.SYMBOL, r) time.Sleep(time.Second * 1) } } +func SaveFromCSV(downloaddata []byte) { + reader := csv.NewReader(bytes.NewBuffer(downloaddata)) + alls, err := reader.ReadAll() + if err != nil { + panic(err) + } + + var jfield []string + for _, field := range alls[0] { + v, _ := GbkToUtf8([]byte(field)) + jfield = append(jfield, string(v)) + // log.Printf("%#v", string(v)) + } + + re, _ := regexp.Compile(`\d+`) + + var stocks []mongo.WriteModel + for _, line := range alls[1:] { + + var fields []string + for _, field := range line { + v, _ := GbkToUtf8([]byte(field)) + fields = append(fields, string(v)) + + } + + code, err := strconv.ParseInt(re.FindString(fields[1]), 10, 64) + if err != nil { + panic(err) + } + + s := &Stock{ + Date: fields[0], + CodeStr: fields[1], + Name: fields[2], + ClosingPrice: ToFloat(fields[3]), + MaxPrice: ToFloat(fields[4]), + MinPrice: ToFloat(fields[5]), + OpeningPrice: ToFloat(fields[6]), + PreviousClosingPrice: ToFloat(fields[7]), + UpsDowns: ToFloat(fields[8]), + UpsDownsRatio: ToFloat(fields[9]), + TurnoverRate: ToFloat(fields[10]), + Volume: ToFloat(fields[11]), + Turnover: ToFloat(fields[12]), + MarketValue: ToFloat(fields[13]), + CirculatingMarketValue: ToFloat(fields[14]), + + Code: code, + } + stocks = append(stocks, &mongo.InsertOneModel{Document: s}) + } + + cstock := client.Database("money").Collection("stock") + r, err := cstock.BulkWrite(context.TODO(), stocks) + if err != nil { + log.Println(err) + } + log.Println(r) +} + func ToFloat(s string) float64 { if s == "None" { return 0 diff --git a/main_test.go b/main_test.go index b8ef46e..97092d0 100644 --- a/main_test.go +++ b/main_test.go @@ -2,6 +2,10 @@ package moneymoney import "testing" +func TestCase2(t *testing.T) { + GetFromAPI() +} + func TestCase1(t *testing.T) { main() } diff --git a/stock_codes.gob b/stock_codes.gob index 0b4d0fd..3462249 100644 Binary files a/stock_codes.gob and b/stock_codes.gob differ