money-money/main.go

package moneymoney

import (
	"bytes"
	"context"
	"encoding/csv"
	"io/ioutil"
	"log"
	"regexp"
	"strconv"
	"time"

	"github.com/474420502/gcurl"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
)

type Stock struct {
	Date                   string  `json:"日期" bson:"日期"`
	CodeStr                string  `json:"股票代码" bson:"股票代码"`
	Name                   string  `json:"名称" bson:"名称"`
	ClosingPrice           float64 `json:"收盘价" bson:"收盘价"`
	MaxPrice               float64 `json:"最高价" bson:"最高价"`
	MinPrice               float64 `json:"最低价" bson:"最低价"`
	OpeningPrice           float64 `json:"开盘价" bson:"开盘价"`
	PreviousClosingPrice   float64 `json:"前收盘" bson:"前收盘"`
	UpsDowns               float64 `json:"涨跌额" bson:"涨跌额"`
	UpsDownsRatio          float64 `json:"涨跌幅" bson:"涨跌幅"`
	TurnoverRate           float64 `json:"换手率" bson:"换手率"`
	Volume                 float64 `json:"成交量" bson:"成交量"`
	Turnover               float64 `json:"成交金额" bson:"成交金额"`
	MarketValue            float64 `json:"总市值" bson:"总市值"`
	CirculatingMarketValue float64 `json:"流通市值" bson:"流通市值"`

	Code int `json:"股票数字代码" bson:"股票数字代码"`
}

func main() {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))

	// 	murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=24&type=query' \
	//   -H 'Accept: application/json, text/javascript, */*; q=0.01' \
	//   -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
	//   -H 'Connection: keep-alive' \
	//   -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
	//   -H 'Referer: http://quotes.money.163.com/old/' \
	//   -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
	//   -H 'X-Requested-With: XMLHttpRequest' \
	//   --compressed \
	//   --insecure`

	// 	tp := gcurl.Parse(murl).Temporary()
	// 	page := tp.QueryParam(`page=\d+`)

	// 	var i int64 = 0
	// 	for ; i < 1000; i++ {
	// 		page.IntSet(i)
	// 		resp, err := tp.Execute()
	// 		if err != nil {
	// 			panic(err)
	// 		}

	// 		log.Println(string(resp.Content()))
	// 	}

	durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=1300731&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
	-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
	-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
	-H 'Connection: keep-alive' \
	-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
	-H 'Referer: http://quotes.money.163.com/trade/lsjysj_300731.html' \
	-H 'Upgrade-Insecure-Requests: 1' \
	-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
	--compressed \
	--insecure`

	resp, err := gcurl.Parse(durl).Temporary().Execute()
	if err != nil {
		panic(err)
	}

	reader := csv.NewReader(bytes.NewBuffer(resp.Content()))
	alls, err := reader.ReadAll()
	if err != nil {
		panic(err)
	}

	var jfield []string
	for _, field := range alls[0] {
		v, _ := GbkToUtf8([]byte(field))
		jfield = append(jfield, string(v))
		log.Printf("%#v", string(v))
	}

	re, _ := regexp.Compile(`\d+`)

	var stocks []mongo.WriteModel
	for _, line := range alls[1:] {

		var fields []string
		for _, field := range line {
			v, _ := GbkToUtf8([]byte(field))
			fields = append(fields, string(v))

		}

		code, err := strconv.Atoi(re.FindString(fields[1]))
		if err != nil {
			panic(err)
		}

		s := &Stock{
			Date:                   fields[0],
			CodeStr:                fields[1],
			Name:                   fields[2],
			ClosingPrice:           ToFloat(fields[3]),
			MaxPrice:               ToFloat(fields[4]),
			MinPrice:               ToFloat(fields[5]),
			OpeningPrice:           ToFloat(fields[6]),
			PreviousClosingPrice:   ToFloat(fields[7]),
			UpsDowns:               ToFloat(fields[8]),
			UpsDownsRatio:          ToFloat(fields[9]),
			TurnoverRate:           ToFloat(fields[10]),
			Volume:                 ToFloat(fields[11]),
			Turnover:               ToFloat(fields[12]),
			MarketValue:            ToFloat(fields[13]),
			CirculatingMarketValue: ToFloat(fields[14]),

			Code: code,
		}
		stocks = append(stocks, &mongo.InsertOneModel{Document: s})
	}

	cstock := client.Database("money").Collection("stock")
	r, err := cstock.BulkWrite(context.TODO(), stocks)
	if err != nil {

	}
	log.Println(r)

}

func ToFloat(s string) float64 {
	if s == "None" {
		return 0
	}
	v, err := strconv.ParseFloat(s, 64)
	if err != nil {
		panic(err)
	}
	return v
}

func GbkToUtf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}