新版本

2022-06-21 13:38:38 +08:00 · 2022-06-21 13:38:38 +08:00 · 9dafc9d6d9
commit 9dafc9d6d9
parent eb99d0eb0d
4 changed files with 88 additions and 35 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1 @@
+*.json
--- a/main.go
+++ b/main.go
@ -4,13 +4,17 @@ import (
 	"bytes"
 	"context"
 	"encoding/csv"
+	"encoding/gob"
 	"io/ioutil"
 	"log"
+	"os"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"

 	"github.com/474420502/gcurl"
+	"github.com/tidwall/gjson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"golang.org/x/text/encoding/simplifiedchinese"
@ -37,46 +41,94 @@ type Stock struct {
 	Code int `json:"股票数字代码" bson:"股票数字代码"`
 }

+type StockCode struct {
+	CodeStr string // 代地区码
+	Code    string // 不带地区码
+}
+
 func main() {
+
+	log.SetFlags(log.Llongfile | log.LstdFlags)
+
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
+	// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
+	if err != nil {
+		panic(err)
+	}
+	murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \
+	  -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+	  -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
+	  -H 'Connection: keep-alive' \
+	  -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
+	  -H 'Referer: http://quotes.money.163.com/old/' \
+	  -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
+	  -H 'X-Requested-With: XMLHttpRequest' \
+	  --compressed \
+	  --insecure`

-	// 	murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=24&type=query' \
-	//   -H 'Accept: application/json, text/javascript, */*; q=0.01' \
-	//   -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
-	//   -H 'Connection: keep-alive' \
-	//   -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
-	//   -H 'Referer: http://quotes.money.163.com/old/' \
-	//   -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
-	//   -H 'X-Requested-With: XMLHttpRequest' \
-	//   --compressed \
-	//   --insecure`
+	tp := gcurl.Parse(murl).Temporary()
+	page := tp.QueryParam(`page=\d+`)
+	var stockCodesFile = "./stock_codes.gob"
+	var stockCodes []StockCode
+	f, err := os.Open(stockCodesFile)
+	if err == nil {
+		err = gob.NewDecoder(f).Decode(&stockCodes)
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		var i int64 = 0
+		var pagecount int64 = 100
+		for ; i < pagecount; i++ {
+			page.IntSet(i)
+			resp, err := tp.Execute()
+			if err != nil {
+				panic(err)
+			}
+			jr := gjson.ParseBytes(resp.Content())
+			pagecount = jr.Get("pagecount").Int()
+			for _, ljr := range jr.Get("list").Array() {
+				stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str})
+			}
+			// log.Println(jr.String())
+		}
+		f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
+		if err != nil {
+			panic(err)
+		}
+		err = gob.NewEncoder(f).Encode(&stockCodes)
+		if err != nil {
+			panic(err)
+		}
+	}

-	// 	tp := gcurl.Parse(murl).Temporary()
-	// 	page := tp.QueryParam(`page=\d+`)
+	if f != nil {
+		err = f.Close()
+		if err != nil {
+			panic(err)
+		}
+	}

-	// 	var i int64 = 0
-	// 	for ; i < 1000; i++ {
-	// 		page.IntSet(i)
-	// 		resp, err := tp.Execute()
-	// 		if err != nil {
-	// 			panic(err)
-	// 		}
+	for _, code := range stockCodes {
+		DownloadDataFromCode(client, &code)
+	}
+}

-	// 		log.Println(string(resp.Content()))
-	// 	}
-
-	durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=1300731&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
+func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
+	// 300731
+	durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
 		-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
 		-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
 		-H 'Connection: keep-alive' \
-	-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
-	-H 'Referer: http://quotes.money.163.com/trade/lsjysj_300731.html' \
+		-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
+		-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \
 		-H 'Upgrade-Insecure-Requests: 1' \
-	-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
-	--compressed \
-	--insecure`
+		-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'`
+
+	durl = strings.ReplaceAll(durl, `${code}`, code.Code)
+	durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr)

 	resp, err := gcurl.Parse(durl).Temporary().Execute()
 	if err != nil {
@ -93,7 +145,7 @@ func main() {
 	for _, field := range alls[0] {
 		v, _ := GbkToUtf8([]byte(field))
 		jfield = append(jfield, string(v))
-		log.Printf("%#v", string(v))
+		// log.Printf("%#v", string(v))
 	}

 	re, _ := regexp.Compile(`\d+`)
@ -140,8 +192,8 @@ func main() {
 	if err != nil {

 	}
-	log.Println(r)
-
+	log.Println(code.Code, r)
+	time.Sleep(time.Second * 2)
 }

 func ToFloat(s string) float64 {
--- a/my.json.7z
+++ b/my.json.7z
--- a/stock_codes.gob
+++ b/stock_codes.gob