package main import ( "log" "strings" "testing" "github.com/474420502/hunter" ) type AreaCode struct { hunter.PreCurlUrl } func (acode *AreaCode) Execute(cxt *hunter.TaskContext) { content, err := cxt.Hunt() if err != nil { panic(err) } extractor := hunter.NewExtractor(content.Content()) xp, err := extractor.XPath("//div[@class='ip']") if err != nil { panic(err) } texts, errlist := xp.ForEachText("./h4") if len(errlist) != 0 { panic(err) } var provinces []*NameCode provincesdict := make(map[string]*NameCode) provincemap := make(map[string]string) for _, t := range texts { pcode := strings.Split(t, " ") provincemap[pcode[1]] = pcode[0] nc := &NameCode{Name: pcode[0], Code: pcode[1]} provincesdict[pcode[1]] = nc provinces = append(provinces, nc) } // log.Println(texts) nxp, errlist := xp.ForEach("./ul//li") if len(errlist) != 0 { panic(err) } texts, errlist = nxp.ForEachText("./h5") if len(errlist) != 0 { panic(err) } var temp []string // var areas []*NameCode areasdict := make(map[string]*NameCode) for _, t := range texts { pcode := strings.Split(t, " ") var name, code string var nval string switch pcode[0] { case "市辖区": name = provincemap[pcode[1][0:2]] + pcode[0] code = pcode[1] case "县": name = provincemap[pcode[1][0:2]] + pcode[0] + "区" code = pcode[1] case "省直辖县级行政区划": name = provincemap[pcode[1][0:2]] + pcode[0] code = pcode[1] default: name = pcode[0] code = pcode[1] } nval = name + " " + code temp = append(temp, nval) nc := &NameCode{Name: name, Code: code} areasdict[code] = nc pnc := provincesdict[code[0:2]] pnc.Child = append(pnc.Child, nc) // areas = append(areas, nc) } log.Println(temp) texts, errlist = nxp.ForEachText("./ul//li") if len(errlist) != 0 { panic(err) } log.Println(texts) for _, t := range texts { pcode := strings.Split(t, " ") var name, code string switch pcode[0] { case "市辖区": name = areasdict[pcode[1][0:4]].Name + pcode[0] code = pcode[1] default: name = pcode[0] code = pcode[1] } nc := &NameCode{Name: name, Code: code} anc := areasdict[nc.Code[0:4]] anc.Child = append(anc.Child, nc) } kl := &KeyList{} for _, p := range provinces { kl.AppendKey(p) } SaveData("./data/province.gob", kl) } func TestHunterCountry(t *testing.T) { // gob.Register(&NameCode{}) defer db.Close() curlBash := `curl 'http://www.ip33.com/area_code.html'` ht := hunter.NewHunter() // first params PreCurlUrl curl := hunter.PreCurlUrl(curlBash) ht.AddTask(&AreaCode{PreCurlUrl: curl}) ht.Execute() } func estLoadProvince(t *testing.T) { kl := &KeyList{} LoadGob("./data/province.gob", kl) t.Error(GetRandomKeyByList(kl.GetKeys()).(NameCode)) }