+
+
+
+
+
+
+
+
+
+
+
+
+
+ + Tag Search : + vocaloid
+ + + Live(0) / + User(28) + + + + + + +
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ んぁゆ (NaYu)
+
+
+ @NaYu_NotYou
🇮🇩🛫🇦🇺 || Main Account || Korean Related: @NaYu_ha || Pull me into more hells- || icon: @AyaminTwT
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Egao No ShouJou
+
+
+ @f:100000242133373
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Yuu'll Be Back
+
+
+ @f:100004051574775
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Thamanan Wittayachamnankul
+
+
+ @f:100000714321056
อ่าา ชอบเกรียนที่สุดเบยย อ๊ายยย ขอให้ได้เกรียน เกรียนในสิ่งที่ชอบ และไม่เดือดร้อนคนอื่น โอเคโน๊ะ อยากจะเกรียนเพลงใส่กันเชิญจ้า กร๊ากก ผมเกรียนได้ทุกคนอยู่แระ ..
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ totally katië 123
+
+
+ @katie_diva_xoxo
❤️❤️❤️zoella
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ Fujisaki Hitomi
+
+
+ @c:vlemvpe
Fujisaki Hitomi
+https://www.facebook.com/ciazfah2
+คนไทยค่า :D
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ カワズ先輩≠カエル先輩
+
+
+ @kawazu1816
愛したって、愛されたっていいじゃないか
+
+
+
+
+
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Xio Steph
+
+
+ @f:100007528892324
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Kama Jewell Greco
+
+
+ @f:489672901190991
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ 英 姫柳/かんじむずかしい
+
+
+ @Kiryu_Hanabusa
YouTuberになるために準備中です。 基本受けの体制なので話し掛けてもらうまでROMってることが多いです。気軽にお声掛けください！ codevein/競馬/麻雀/雀魂/FPS #姫柳の..
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ ムジャオ＠3/9アイコン変わりました
+
+
+ @mujamiku
ただのザコなボカロファンです。
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ miming(みみん)
+
+
+ @c:mimingdayo
韓国人です。日本語勉強はじめたのは10年くらい前からで
+留学はしたことありません。
+まぁ気ままにやります。
+ヾ(⌒(_*'ω'*)_
+
+ 
+
+
+
+
+
+
+
+
+
diff --git a/tasks/twitcasting/twitcasting_task1/main.go b/tasks/twitcasting/twitcasting_task1/main.go
index 06ab7d0..736ef31 100644
--- a/tasks/twitcasting/twitcasting_task1/main.go
+++ b/tasks/twitcasting/twitcasting_task1/main.go
@@ -1 +1,5 @@
package main
+
+func main() {
+ Execute()
+}
diff --git a/tasks/twitcasting/twitcasting_task1/main_test.go b/tasks/twitcasting/twitcasting_task1/main_test.go
index 7acc76b..18bfb6a 100644
--- a/tasks/twitcasting/twitcasting_task1/main_test.go
+++ b/tasks/twitcasting/twitcasting_task1/main_test.go
@@ -1,26 +1,9 @@
package main
import (
- "database/sql"
- "encoding/json"
- "intimate"
- "net/http"
"net/url"
- "os"
- "os/signal"
- "syscall"
- "time"
- "github.com/474420502/extractor"
- "github.com/474420502/focus/compare"
- "github.com/474420502/focus/tree/heap"
-
- "log"
"testing"
-
- _ "net/http/pprof"
-
- "github.com/474420502/requests"
)
func Test(t *testing.T) {
@@ -30,161 +13,13 @@ func Test(t *testing.T) {
t.Error(u.String())
}
-// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
-var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitcasting))
-
-// estore 解析存储连接实例
-var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+func TestUpdateTime(t *testing.T) {
+ // streamer := &intimate.Streamer{}
+ // streamer.Uid = 420153
+ // streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+ // estore.Update(streamer, "update_time", streamer.UpdateTime)
+}
func TestMain(t *testing.T) {
- f, _ := os.OpenFile("./log", os.O_TRUNC|os.O_CREATE|os.O_RDWR, os.ModePerm)
- log.SetFlags(log.Llongfile | log.Ltime)
- log.SetOutput(f)
- go func() {
- log.Println(http.ListenAndServe(":4040", nil))
- }()
-
- homeurl := "https://twitcasting.tv"
- searchurl := "https://twitcasting.tv/rankingindex.php"
- queuedict := make(map[string]bool)
- queue := heap.New(compare.String)
- queue.Put(searchurl)
- queuedict[searchurl] = true
- ses := requests.NewSession()
- ses.Config().SetTimeout(15)
-
- var surl interface{}
- var ok bool
- var debugsp *SearchProfile
- var content []byte
-
- defer func() {
- if ierr := recover(); ierr != nil {
- log.Println(surl, debugsp)
- f, _ := os.OpenFile("./error.html", os.O_TRUNC|os.O_CREATE|os.O_RDWR, os.ModePerm)
- f.Write(content)
- f.Close()
- log.Panic(ierr)
- }
- }()
-
- go func() {
- signalchan := make(chan os.Signal)
- signal.Notify(signalchan, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
- log.Println("accept stop command:", <-signalchan)
- f, _ := os.OpenFile("./error.html", os.O_TRUNC|os.O_CREATE|os.O_RDWR, os.ModePerm)
- f.Write(content)
- f.Close()
- os.Exit(1)
- }()
-
- for surl, ok = queue.Pop(); ok; surl, ok = queue.Pop() {
- u, err := url.Parse(surl.(string))
- if err != nil {
- log.Println(err)
- continue
- }
-
- resp, err := ses.Get(u.String()).Execute()
- if err != nil {
- log.Println(err)
- log.Println(u.String(), surl)
- continue
- // log.Panic(err)
- }
-
- content = resp.Content()
- etor := extractor.ExtractXml(resp.Content())
- result, err := etor.XPath("//p[@class='taglist']/a[contains(@class, 'tag')]/@href")
- if err != nil {
- panic(err)
- }
-
- iter := result.NodeIter()
- for iter.Next() {
-
- wurl := homeurl + iter.Node().NodeValue()
- if ok := queuedict[wurl]; !ok {
- log.Println(wurl)
- sl := &intimate.StreamerList{}
- sl.Platform = intimate.Ptwitcasting
- sl.Url = wurl
- sl.Operator = 0
- sl.UpdateInterval = 120
- sl.UpdateTime = time.Now()
-
- estore.InsertStreamerList(sl)
-
- queue.Put(wurl)
- queuedict[wurl] = true
- }
- }
-
- // doc.Find("//div[@class='tw-search-result-row']")
- xps, err := etor.XPaths("//div[@class='tw-search-result-row']")
- if err != nil {
- log.Println(surl, err)
- continue
- }
-
- log.Println("extract tag")
- var splist = xps.ForEachTag(SearchProfile{})
- log.Println("finish extract tag")
- for _, isp := range splist {
- sp := isp.(*SearchProfile)
- if sp.LiveUrl == "" {
- continue
- }
-
- sp.UserId = sp.LiveUrl[1:]
- for i := 0; i < len(sp.TagUrl); i++ {
- wurl := homeurl + sp.TagUrl[i]
- sp.TagUrl[i] = wurl
- if ok := queuedict[wurl]; !ok {
- sl := &intimate.StreamerList{}
- sl.Platform = intimate.Ptwitcasting
- sl.Url = wurl
- sl.Operator = 0
- sl.UpdateInterval = 120
- sl.UpdateTime = time.Now()
- estore.InsertStreamerList(sl)
-
- queue.Put(wurl)
- queuedict[wurl] = true
- }
- }
- // log.Println(sp.(SearchProfile))
- }
-
- log.Println("find user:", len(splist))
- for _, isp := range splist {
- sp := isp.(*SearchProfile)
- // log.Println(sp)
- streamer := &intimate.Streamer{}
- streamer.Platform = intimate.Ptwitcasting
- streamer.LiveUrl = sql.NullString{String: sp.LiveUrl, Valid: true}
- if btags, err := json.Marshal(sp.Tag); err != nil {
- log.Println(err)
- } else {
- streamer.Tags = btags
- }
- streamer.UpdateInterval = 120
- streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
- streamer.UserName = sql.NullString{String: sp.UserName, Valid: true}
- streamer.UserId = sp.UserId
- debugsp = sp
- estore.InsertStreamer(streamer)
- }
-
- log.Println("finish remain", queue.Size())
- }
-}
-
-type SearchProfile struct {
- UserName string `exp:".//span[@class='username']" method:"Text"`
- UserId string // `exp:".//span[@class='fullname']" method:"Text"`
- LiveUrl string `exp:".//div[@class='usertext']/a[@href]" method:"Attribute,href Value"`
- Tag []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"Text"`
- TagUrl []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"Attribute,href Value"`
}
diff --git a/tasks/twitcasting/twitcasting_task1/twitcasting.go b/tasks/twitcasting/twitcasting_task1/twitcasting.go
new file mode 100644
index 0000000..58e2cba
--- /dev/null
+++ b/tasks/twitcasting/twitcasting_task1/twitcasting.go
@@ -0,0 +1,140 @@
+package main
+
+import (
+ "database/sql"
+ "encoding/json"
+ "intimate"
+ "log"
+ "net/url"
+ "time"
+
+ "github.com/474420502/extractor"
+ "github.com/474420502/focus/compare"
+ "github.com/474420502/focus/tree/heap"
+ "github.com/474420502/requests"
+)
+
+// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitcasting))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+type SearchProfile struct {
+ UserName string `exp:".//span[@class='username']" method:"Text"`
+ UserId string // `exp:".//span[@class='fullname']" method:"Text"`
+ LiveUrl string `exp:".//div[@class='usertext']/a[@href]" method:"AttributeValue,href"`
+ Tag []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"Text"`
+ TagUrl []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"AttributeValue,href"`
+}
+
+func Execute() {
+ homeurl := "https://twitcasting.tv"
+ searchurl := "https://twitcasting.tv/rankingindex.php"
+ queuedict := make(map[string]bool)
+ queue := heap.New(compare.String)
+ queue.Put(searchurl)
+ queuedict[searchurl] = true
+ ses := requests.NewSession()
+ ses.Config().SetTimeout(15)
+
+ var surl interface{}
+ var ok bool
+
+ ps := intimate.NewPerfectShutdown()
+
+ for surl, ok = queue.Pop(); ok && !ps.IsClose(); surl, ok = queue.Pop() {
+ u, err := url.Parse(surl.(string))
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+
+ resp, err := ses.Get(u.String()).Execute()
+ if err != nil {
+ log.Println(err)
+ log.Println(u.String(), surl)
+ continue
+ // log.Panic(err)
+ }
+
+ etor := extractor.ExtractXml(resp.Content())
+ result, err := etor.XPaths("//p[@class='taglist']/a[contains(@class, 'tag')]/@href")
+ if err != nil {
+ panic(err)
+ }
+
+ for _, href := range result.GetTexts() {
+
+ wurl := homeurl + href
+ if ok := queuedict[wurl]; !ok {
+ log.Println(wurl)
+ sl := &intimate.StreamerList{}
+ sl.Platform = intimate.Ptwitcasting
+ sl.Url = wurl
+ sl.Operator = 0
+ sl.UpdateInterval = 120
+ sl.UpdateTime = time.Now()
+
+ estore.InsertStreamerList(sl)
+
+ queue.Put(wurl)
+ queuedict[wurl] = true
+ }
+ }
+
+ xps, err := etor.XPaths("//div[@class='tw-search-result-row']")
+ if err != nil {
+ log.Println(surl, err)
+ continue
+ }
+
+ var splist = xps.ForEachTag(SearchProfile{})
+ for _, isp := range splist {
+ sp := isp.(*SearchProfile)
+ if sp.LiveUrl == "" {
+ continue
+ }
+
+ sp.UserId = sp.LiveUrl[1:]
+ for i := 0; i < len(sp.TagUrl); i++ {
+ wurl := homeurl + sp.TagUrl[i]
+ sp.TagUrl[i] = wurl
+ if ok := queuedict[wurl]; !ok {
+ sl := &intimate.StreamerList{}
+ sl.Platform = intimate.Ptwitcasting
+ sl.Url = wurl
+ sl.Operator = 0
+ sl.UpdateInterval = 120
+ sl.UpdateTime = time.Now()
+ estore.InsertStreamerList(sl)
+
+ queue.Put(wurl)
+ queuedict[wurl] = true
+ }
+ }
+ // log.Println(sp.(SearchProfile))
+ }
+
+ log.Println("find user:", len(splist))
+ for _, isp := range splist {
+ sp := isp.(*SearchProfile)
+ // log.Println(sp)
+ streamer := &intimate.Streamer{}
+ streamer.Platform = intimate.Ptwitcasting
+ streamer.LiveUrl = sql.NullString{String: sp.LiveUrl, Valid: true}
+ if btags, err := json.Marshal(sp.Tag); err != nil {
+ log.Println(err)
+ } else {
+ streamer.Tags = btags
+ }
+ streamer.UpdateInterval = 120
+ streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+ streamer.UserName = sql.NullString{String: sp.UserName, Valid: true}
+ streamer.UserId = sp.UserId
+ estore.InsertStreamer(streamer)
+ }
+
+ log.Println("finish remain", queue.Size())
+ }
+}