最新完美算法

api.go

@@ -0,0 +1,28 @@
+package moneymoney
+
+import (
+	"github.com/474420502/gcurl"
+)
+
+func GetFromAPI() {
+	tp := gcurl.Parse(`curl 'http://quotes.money.163.com/service/chddata.html?code=0601398&start=20140720&end=20150508'`).Temporary()
+	pcode := tp.QueryParam("code")
+	start := tp.QueryParam("start")
+	end := tp.QueryParam("end")
+
+	for _, code := range GetStocks() {
+
+		if code.MCAP >= 20000000000 {
+			pcode.StringSet(code.CODE)
+			start.StringSet("20170101")
+			end.StringSet("20220622")
+
+			resp, err := tp.Execute()
+			if err != nil {
+				panic(err)
+			}
+			SaveFromCSV(resp.Content())
+		}
+	}
+
+}

main.go

@@ -29,23 +29,23 @@ import (
 )
 
 type Stock struct {
-	Date                   string  `json:"Date" bson:"Date"`                                     // 日期
+	Date                   string  `json:"日期" bson:"日期"`
-	CodeStr                string  `json:"CodeStr" bson:"CodeStr"`                               // 股票代码
+	CodeStr                string  `json:"股票代码" bson:"股票代码"`
-	Name                   string  `json:"Name" bson:"Name"`                                     // 名称
+	Name                   string  `json:"名称" bson:"名称"`
-	ClosingPrice           float64 `json:"ClosingPrice" bson:"ClosingPrice"`                     // 收盘价
+	ClosingPrice           float64 `json:"收盘价" bson:"收盘价"`
-	MaxPrice               float64 `json:"MaxPrice" bson:"MaxPrice"`                             // 最高价
+	MaxPrice               float64 `json:"最高价" bson:"最高价"`
-	MinPrice               float64 `json:"MinPrice" bson:"MinPrice"`                             // 最低价
+	MinPrice               float64 `json:"最低价" bson:"最低价"`
-	OpeningPrice           float64 `json:"OpeningPrice" bson:"OpeningPrice"`                     // 开盘价
+	OpeningPrice           float64 `json:"开盘价" bson:"开盘价"`
-	PreviousClosingPrice   float64 `json:"PreviousClosingPrice" bson:"PreviousClosingPrice"`     // 前收盘
+	PreviousClosingPrice   float64 `json:"前收盘" bson:"前收盘"`
-	UpsDowns               float64 `json:"UpsDowns" bson:"UpsDowns"`                             // 涨跌额
+	UpsDowns               float64 `json:"涨跌额" bson:"涨跌额"`
-	UpsDownsRatio          float64 `json:"UpsDownsRatio" bson:"UpsDownsRatio"`                   // 涨跌幅
+	UpsDownsRatio          float64 `json:"涨跌幅" bson:"涨跌幅"`
-	TurnoverRate           float64 `json:"TurnoverRate" bson:"TurnoverRate"`                     // 换手率
+	TurnoverRate           float64 `json:"换手率" bson:"换手率"`
-	Volume                 float64 `json:"Volume" bson:"Volume"`                                 // 成交量
+	Volume                 float64 `json:"成交量" bson:"成交量"`
-	Turnover               float64 `json:"Turnover" bson:"Turnover"`                             // 成交金额
+	Turnover               float64 `json:"成交金额" bson:"成交金额"`
-	MarketValue            float64 `json:"MarketValue" bson:"MarketValue"`                       // 总市值
+	MarketValue            float64 `json:"总市值" bson:"总市值"`
-	CirculatingMarketValue float64 `json:"CirculatingMarketValue" bson:"CirculatingMarketValue"` // 流通市值
+	CirculatingMarketValue float64 `json:"流通市值" bson:"流通市值"`
 
-	Code string `json:"Code" bson:"Code"` // 股票数字代码
+	Code int64 `json:"股票数字代码" bson:"股票数字代码"`
 }
 
 type StockBase struct {
@@ -117,49 +117,52 @@ func GetDefaultPage() *rod.Page {
 	return DefaultPage
 }
 
-func main() {
+var client *mongo.Client
+var err error
 
+func init() {
 	log.SetFlags(log.Llongfile | log.LstdFlags)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	client, err = mongo.Connect(context.TODO(), options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
-	defer cancel()
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+
+	for _, code := range GetStocks() {
+
+		if code.MCAP >= 50000000000 {
+			DownloadDataFromCode(code)
+		}
+
+	}
+}
+
+func GetStocks() []*StockBase {
+
 	// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
-	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
+
+	cur, err := client.Database("money").Collection("stock").Distinct(context.TODO(), "股票数字代码", bson.M{})
 	if err != nil {
 		panic(err)
 	}
 
-	cur, err := client.Database("money").Collection("stock").Aggregate(context.TODO(), bson.A{
+	var skipMap map[int64]bool = make(map[int64]bool)
-		bson.M{"$group": bson.M{"_id": "$股票数字代码"}},
-	})
-	if err != nil {
-		panic(err)
-	}
 
-	var skipMap map[string]bool = make(map[string]bool)
+	for _, idoc := range cur {
-
+		var scode = idoc.(int64)
-	for cur.Next(context.TODO()) {
+		// err = cur.Decode(&doc)
-		var doc bson.M
-		err = cur.Decode(&doc)
 		if err == nil {
-			skipMap[strconv.Itoa(int(doc["_id"].(int32)))] = true
+
+			skipMap[scode] = true
 		} else {
 			log.Panic(err)
 		}
 
 	}
 
-	murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \
-	  -H 'Accept: application/json, text/javascript, */*; q=0.01' \
-	  -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
-	  -H 'Connection: keep-alive' \
-	  -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
-	  -H 'Referer: http://quotes.money.163.com/old/' \
-	  -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
-	  -H 'X-Requested-With: XMLHttpRequest'`
-
-	tp := gcurl.Parse(murl).Temporary()
-	page := tp.QueryParam(`page=\d+`)
 	var stockCodesFile = "./stock_codes.gob"
 	var stockCodes []*StockBase
 	f, err := os.Open(stockCodesFile)
@@ -169,38 +172,34 @@ func main() {
 			panic(err)
 		}
 	} else {
-		var i int64 = 0
+
-		var pagecount int64 = 100
+		murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?count=5000'`
-		for ; i < pagecount; i++ {
+		tp := gcurl.Parse(murl).Temporary()
-			page.IntSet(i)
+		resp, err := tp.Execute()
-			resp, err := tp.Execute()
+		if err != nil {
+			panic(err)
+		}
+		jr := gjson.ParseBytes(resp.Content())
+
+		for _, s := range jr.Get("list").Array() {
+			var stockCode StockBase
+			err = json.Unmarshal([]byte(s.String()), &stockCode)
 			if err != nil {
 				panic(err)
 			}
 
-			jr := gjson.ParseBytes(resp.Content())
+			stockCodes = append(stockCodes, &stockCode)
-			pagecount = jr.Get("pagecount").Int()
-
-			for _, s := range jr.Get("list").Array() {
-				var stockCode StockBase
-				err = json.Unmarshal([]byte(s.String()), &stockCode)
-				if err != nil {
-					panic(err)
-				}
-
-				stockCodes = append(stockCodes, &stockCode)
-			}
-
-			// log.Println(jr.String())
-		}
-		f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
-		if err != nil {
-			panic(err)
-		}
-		err = gob.NewEncoder(f).Encode(&stockCodes)
-		if err != nil {
-			panic(err)
 		}
+
+		// log.Println(jr.String())
+	}
+	f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
+	if err != nil {
+		panic(err)
+	}
+	err = gob.NewEncoder(f).Encode(&stockCodes)
+	if err != nil {
+		panic(err)
 	}
 
 	if f != nil {
@@ -210,18 +209,30 @@ func main() {
 		}
 	}
 
+	re, _ := regexp.Compile(`\d+`)
+	log.Println("stocks", len(stockCodes))
+	var result []*StockBase
 	for _, code := range stockCodes {
-		if _, ok := skipMap[code.SYMBOL]; ok {
+		scode, err := strconv.ParseInt(re.FindString(code.SYMBOL), 10, 64)
+		if err != nil {
+			panic(err)
+		}
+		if _, ok := skipMap[scode]; ok {
 			continue
 		}
 
-		if code.MCAP >= 50000000000 {
+		result = append(result, code)
-			DownloadDataFromCode(client, code)
+
-		}
+		// if code.MCAP >= 50000000000 {
+		// 	DownloadDataFromCode(client, code)
+		// }
+
 	}
+
+	return result
 }
 
-func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
+func DownloadDataFromCode(code *StockBase) {
 	// 300731
 	// durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
 	// -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
@@ -234,8 +245,6 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
 
 	// http://quotes.money.163.com/0601988.html
 
-	cstock := client.Database("money").Collection("stock")
-
 	page := GetDefaultPage()
 	stockurl := fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE)
 	log.Println(stockurl)
@@ -298,7 +307,7 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
 
 		re, _ := regexp.Compile(`\d+`)
 
-		// var stocks []mongo.WriteModel
+		var stocks []mongo.WriteModel
 		for _, line := range alls[1:] {
 
 			var fields []string
@@ -308,9 +317,12 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
 
 			}
 
-			code := re.FindString(fields[1])
+			code, err := strconv.ParseInt(re.FindString(fields[1]), 10, 64)
+			if err != nil {
+				panic(err)
+			}
 
-			s := Stock{
+			s := &Stock{
 				Date:                   fields[0],
 				CodeStr:                fields[1],
 				Name:                   fields[2],
@@ -329,15 +341,80 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
 
 				Code: code,
 			}
-
+			stocks = append(stocks, &mongo.InsertOneModel{Document: s})
-			cstock.InsertOne(context.TODO(), mongo.InsertOneModel{Document: s})
 		}
 
-		log.Println(code.SYMBOL)
+		cstock := client.Database("money").Collection("stock")
+		r, err := cstock.BulkWrite(context.TODO(), stocks)
+		if err != nil {
+			log.Println(err)
+		}
+		log.Println(code.SYMBOL, r)
 		time.Sleep(time.Second * 1)
 	}
 }
 
+func SaveFromCSV(downloaddata []byte) {
+	reader := csv.NewReader(bytes.NewBuffer(downloaddata))
+	alls, err := reader.ReadAll()
+	if err != nil {
+		panic(err)
+	}
+
+	var jfield []string
+	for _, field := range alls[0] {
+		v, _ := GbkToUtf8([]byte(field))
+		jfield = append(jfield, string(v))
+		// log.Printf("%#v", string(v))
+	}
+
+	re, _ := regexp.Compile(`\d+`)
+
+	var stocks []mongo.WriteModel
+	for _, line := range alls[1:] {
+
+		var fields []string
+		for _, field := range line {
+			v, _ := GbkToUtf8([]byte(field))
+			fields = append(fields, string(v))
+
+		}
+
+		code, err := strconv.ParseInt(re.FindString(fields[1]), 10, 64)
+		if err != nil {
+			panic(err)
+		}
+
+		s := &Stock{
+			Date:                   fields[0],
+			CodeStr:                fields[1],
+			Name:                   fields[2],
+			ClosingPrice:           ToFloat(fields[3]),
+			MaxPrice:               ToFloat(fields[4]),
+			MinPrice:               ToFloat(fields[5]),
+			OpeningPrice:           ToFloat(fields[6]),
+			PreviousClosingPrice:   ToFloat(fields[7]),
+			UpsDowns:               ToFloat(fields[8]),
+			UpsDownsRatio:          ToFloat(fields[9]),
+			TurnoverRate:           ToFloat(fields[10]),
+			Volume:                 ToFloat(fields[11]),
+			Turnover:               ToFloat(fields[12]),
+			MarketValue:            ToFloat(fields[13]),
+			CirculatingMarketValue: ToFloat(fields[14]),
+
+			Code: code,
+		}
+		stocks = append(stocks, &mongo.InsertOneModel{Document: s})
+	}
+
+	cstock := client.Database("money").Collection("stock")
+	r, err := cstock.BulkWrite(context.TODO(), stocks)
+	if err != nil {
+		log.Println(err)
+	}
+	log.Println(r)
+}
+
 func ToFloat(s string) float64 {
 	if s == "None" {
 		return 0

main_test.go

@@ -2,6 +2,10 @@ package moneymoney
 
 import "testing"
 
+func TestCase2(t *testing.T) {
+	GetFromAPI()
+}
+
 func TestCase1(t *testing.T) {
 	main()
 }