money-money/main.go

package moneymoney

import (
	"bytes"
	"context"
	"encoding/csv"
	"encoding/gob"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/474420502/gcurl"
	"github.com/tidwall/gjson"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
)

type Stock struct {
	Date                   string  `json:"日期" bson:"日期"`
	CodeStr                string  `json:"股票代码" bson:"股票代码"`
	Name                   string  `json:"名称" bson:"名称"`
	ClosingPrice           float64 `json:"收盘价" bson:"收盘价"`
	MaxPrice               float64 `json:"最高价" bson:"最高价"`
	MinPrice               float64 `json:"最低价" bson:"最低价"`
	OpeningPrice           float64 `json:"开盘价" bson:"开盘价"`
	PreviousClosingPrice   float64 `json:"前收盘" bson:"前收盘"`
	UpsDowns               float64 `json:"涨跌额" bson:"涨跌额"`
	UpsDownsRatio          float64 `json:"涨跌幅" bson:"涨跌幅"`
	TurnoverRate           float64 `json:"换手率" bson:"换手率"`
	Volume                 float64 `json:"成交量" bson:"成交量"`
	Turnover               float64 `json:"成交金额" bson:"成交金额"`
	MarketValue            float64 `json:"总市值" bson:"总市值"`
	CirculatingMarketValue float64 `json:"流通市值" bson:"流通市值"`

	Code int `json:"股票数字代码" bson:"股票数字代码"`
}

type StockCode struct {
	CodeStr string // 代地区码
	Code    string // 不带地区码
}

func main() {

	log.SetFlags(log.Llongfile | log.LstdFlags)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
	if err != nil {
		panic(err)
	}
	murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \
	  -H 'Accept: application/json, text/javascript, */*; q=0.01' \
	  -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
	  -H 'Connection: keep-alive' \
	  -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
	  -H 'Referer: http://quotes.money.163.com/old/' \
	  -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
	  -H 'X-Requested-With: XMLHttpRequest' \
	  --compressed \
	  --insecure`

	tp := gcurl.Parse(murl).Temporary()
	page := tp.QueryParam(`page=\d+`)
	var stockCodesFile = "./stock_codes.gob"
	var stockCodes []StockCode
	f, err := os.Open(stockCodesFile)
	if err == nil {
		err = gob.NewDecoder(f).Decode(&stockCodes)
		if err != nil {
			panic(err)
		}
	} else {
		var i int64 = 0
		var pagecount int64 = 100
		for ; i < pagecount; i++ {
			page.IntSet(i)
			resp, err := tp.Execute()
			if err != nil {
				panic(err)
			}
			jr := gjson.ParseBytes(resp.Content())
			pagecount = jr.Get("pagecount").Int()
			for _, ljr := range jr.Get("list").Array() {
				stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str})
			}
			// log.Println(jr.String())
		}
		f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
		if err != nil {
			panic(err)
		}
		err = gob.NewEncoder(f).Encode(&stockCodes)
		if err != nil {
			panic(err)
		}
	}

	if f != nil {
		err = f.Close()
		if err != nil {
			panic(err)
		}
	}

	for _, code := range stockCodes {
		DownloadDataFromCode(client, &code)
	}
}

func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
	// 300731
	durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
		-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
		-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
		-H 'Connection: keep-alive' \
		-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
		-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \
		-H 'Upgrade-Insecure-Requests: 1' \
		-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'`

	durl = strings.ReplaceAll(durl, `${code}`, code.Code)
	durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr)

	resp, err := gcurl.Parse(durl).Temporary().Execute()
	if err != nil {
		panic(err)
	}

	reader := csv.NewReader(bytes.NewBuffer(resp.Content()))
	alls, err := reader.ReadAll()
	if err != nil {
		panic(err)
	}

	var jfield []string
	for _, field := range alls[0] {
		v, _ := GbkToUtf8([]byte(field))
		jfield = append(jfield, string(v))
		// log.Printf("%#v", string(v))
	}

	re, _ := regexp.Compile(`\d+`)

	var stocks []mongo.WriteModel
	for _, line := range alls[1:] {

		var fields []string
		for _, field := range line {
			v, _ := GbkToUtf8([]byte(field))
			fields = append(fields, string(v))

		}

		code, err := strconv.Atoi(re.FindString(fields[1]))
		if err != nil {
			panic(err)
		}

		s := &Stock{
			Date:                   fields[0],
			CodeStr:                fields[1],
			Name:                   fields[2],
			ClosingPrice:           ToFloat(fields[3]),
			MaxPrice:               ToFloat(fields[4]),
			MinPrice:               ToFloat(fields[5]),
			OpeningPrice:           ToFloat(fields[6]),
			PreviousClosingPrice:   ToFloat(fields[7]),
			UpsDowns:               ToFloat(fields[8]),
			UpsDownsRatio:          ToFloat(fields[9]),
			TurnoverRate:           ToFloat(fields[10]),
			Volume:                 ToFloat(fields[11]),
			Turnover:               ToFloat(fields[12]),
			MarketValue:            ToFloat(fields[13]),
			CirculatingMarketValue: ToFloat(fields[14]),

			Code: code,
		}
		stocks = append(stocks, &mongo.InsertOneModel{Document: s})
	}

	cstock := client.Database("money").Collection("stock")
	r, err := cstock.BulkWrite(context.TODO(), stocks)
	if err != nil {

	}
	log.Println(code.Code, r)
	time.Sleep(time.Second * 2)
}

func ToFloat(s string) float64 {
	if s == "None" {
		return 0
	}
	v, err := strconv.ParseFloat(s, 64)
	if err != nil {
		panic(err)
	}
	return v
}

func GbkToUtf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}