diff --git a/tasks/twitcasting/twitcasting_task1/main.go b/tasks/twitcasting/twitcasting_task1/main.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/tasks/twitcasting/twitcasting_task1/main.go @@ -0,0 +1 @@ +package main diff --git a/tasks/twitcasting/twitcasting_task1/main_test.go b/tasks/twitcasting/twitcasting_task1/main_test.go new file mode 100644 index 0000000..4003cf7 --- /dev/null +++ b/tasks/twitcasting/twitcasting_task1/main_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "github.com/474420502/focus/compare" + "github.com/474420502/focus/tree/heap" + + "log" + "testing" + + "github.com/474420502/requests" + "github.com/lestrrat-go/libxml2" +) + +func TestMain(t *testing.T) { + + searchurl := "https://twitcasting.tv/rankingindex.php" + queuedict := make(map[string]bool) + queue := heap.New(compare.String) + queue.Put(searchurl) + queuedict[searchurl] = true + + for surl, ok := queue.Pop(); ok; surl, ok = queue.Pop() { + + ses := requests.NewSession() + resp, err := ses.Get(surl.(string)).Execute() + if err != nil { + panic(err) + } + + doc, err := libxml2.ParseHTML(resp.Content()) + if err != nil { + panic(err) + } + defer doc.Free() + result, err := doc.Find("//*[contains(@class, 'tag')]/@href") + if err != nil { + panic(err) + } + defer result.Free() + + iter := result.NodeIter() + for iter.Next() { + + log.Println(iter.Node().NodeValue()) + wurl := "https://twitcasting.tv" + iter.Node().NodeValue() + if ok := queuedict[wurl]; !ok { + queue.Put(wurl) + queuedict[wurl] = true + } + } + + doc.Find("//div[@class='tw-search-result-row']") + + log.Println("finish remain", queue.Size()) + } +}