57 lines
1.1 KiB
Go
57 lines
1.1 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"github.com/474420502/focus/compare"
|
||
|
|
"github.com/474420502/focus/tree/heap"
|
||
|
|
|
||
|
|
"log"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"github.com/474420502/requests"
|
||
|
|
"github.com/lestrrat-go/libxml2"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestMain(t *testing.T) {
|
||
|
|
|
||
|
|
searchurl := "https://twitcasting.tv/rankingindex.php"
|
||
|
|
queuedict := make(map[string]bool)
|
||
|
|
queue := heap.New(compare.String)
|
||
|
|
queue.Put(searchurl)
|
||
|
|
queuedict[searchurl] = true
|
||
|
|
|
||
|
|
for surl, ok := queue.Pop(); ok; surl, ok = queue.Pop() {
|
||
|
|
|
||
|
|
ses := requests.NewSession()
|
||
|
|
resp, err := ses.Get(surl.(string)).Execute()
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
|
||
|
|
doc, err := libxml2.ParseHTML(resp.Content())
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
defer doc.Free()
|
||
|
|
result, err := doc.Find("//*[contains(@class, 'tag')]/@href")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
defer result.Free()
|
||
|
|
|
||
|
|
iter := result.NodeIter()
|
||
|
|
for iter.Next() {
|
||
|
|
|
||
|
|
log.Println(iter.Node().NodeValue())
|
||
|
|
wurl := "https://twitcasting.tv" + iter.Node().NodeValue()
|
||
|
|
if ok := queuedict[wurl]; !ok {
|
||
|
|
queue.Put(wurl)
|
||
|
|
queuedict[wurl] = true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
doc.Find("//div[@class='tw-search-result-row']")
|
||
|
|
|
||
|
|
log.Println("finish remain", queue.Size())
|
||
|
|
}
|
||
|
|
}
|