nimo 插件优化

nimo 数据测试提取成功

crx/myblock.pem

@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDSG09DSvB03TOe
-eOmQwfiCIf0wa2WRB31ewxa6i/PRgEKeJSUvIsIuaECUer2ss+J3rwSS2lDpGuiw
-FnsVyZqKI/+Rcuc83YJGYg6OAzVMz6UL8YCWhXu3huTJ+V+a5iNereIC69ZERRJt
-nXlWqsq6HKya+6BP9sX9CI4GTHQrnWBysAxsswhdnnnRvu+GxglWafSIzuS6OizT
-1M1CmkZxNvDJhTSOR7SJlIYm2kM5/fIL53BdndF2IGAjfV1WV7AjwhTfun5cViEO
-i8niQUIMY4L0AiO9grFD1g1xIYkeuVBoLxOUBzPxJwQmb64gseb9Dvt0BKLRGoou
-SIOyE+KVAgMBAAECggEAI4b6J2kR0VUBEDwmVHO0K38HUstqNHSVgrNO0dLt8sAz
-I44o5DhGqPW4a9L4ZS5SrkWyKonPcic6buISRIwfPVoacjQBfVWAXJnil6lbtyYK
-ZMNcqLcgBRfCcpOgEq91DiKta6yIwekDFXVyCdFd78v+9ML1J+hUsLVkXJTLdP88
-PGamRWVd6vGy3QMRjyM29GLPgS+/6Vrp1cptSuYNqYhlszohmu8lBvzjH9jbPh9d
-GFrrd8Bs7IRCdtKZig/3fbln4JEyyOYE+gcT2jplPksB6mR/5DBIdkVbeuFwGB0+
-h1/PKlprNQt7+Ei0HhHnTib7lZP8WGo4HkSi7PsAGQKBgQD1Ptho0wJiI2+6gL1O
-iNsEJVKIQ2Sxdx3wI/qudphM99t6xKCpPyVI2Nd9PBf2jbZjGAaz+P/KQYxEqb6i
-PRcQ+i99wCQoRfnRvUbKA4goEpKwRXmvn+499dm6D5pEuumOXGQYCmaFXuLTRN/I
-BL6GNgLtoZAlLjUXaWtk8TszGQKBgQDbUf3p3HLpCjRvRDW/vA5xj+08t7xtF9uO
-NilGK79uOA4VnxE2w3ioYqQ7t3I8J/0rAzGKq3tylg4QX6UpQ4b2koRr2B3cqoAk
-dsRdNWAHwCNepz8hTLsZyuihzbNv2nHmoqhzjK/FcrBHx5NAM+T6OBpLzQBnbUzk
-3wIcqm223QKBgQDo/IRxyY0pGMtLXoT6ODACF0b6JzRhGG37tuKvngGAlbQQRP7w
-6wmL1F2cH1wQon7UU34CupqfVnhgvvZZgToJqfU2PTTcgeYc6Pl4b7SJhWOQTOCX
-BZQ7jvYCulHv27aIxaNd53uQVx2cYoFKr58lN+i+QtADUoujq0YYxshb+QKBgQDW
-ZOti7kZCeuBRGIu2V56C8uBFp5MBzf2polZsqx1iIFfcWPfZ4fGUIYFMgwKfvbOl
-lWSbmxB9LiSnaugoU0OezBG43rYqXV4Qxy0jtKagTPoGcFWtNrX7+7e3XD8Zi6Am
-hkFHW3MEAB5EvNq8Oz6OP8Os78SCVn2BimMlJJFF3QKBgQCF+aEAiBv+ivcmHUeP
-2eBq9nLltPFAfXJ/p31MMQ6Jgo36DBqUeoLeyq/WfIXvwqbVbP9fANZrKoTPbI97
-dilCHUoO33rafXJy6jtaggtpz14tt9soecTop0vM/rU7tGtfBe6NXg9LRl+oDJCU
-37I3a9Is+2CLyAUXWCk9mLfFsQ==
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDA6O6owySvzaJ8
+iv6aXRMcuk0lnNKquPNVHeraadSkBVhtaJqPAT7saOVnn9g4s/21OBV6wGmWdl5W
+5+8h+F4qXrlGO8N1Tl8mmBDetqg6T6RrI9vB2Th9hjlp2PCGCEg/lX+tx25qBiDG
+9uCB3pU4KMyfof1Z2jbo7aOfttgxZ1YdssenB//+31QjeZVw7wZACKaSzl6Z+WvY
+pastMZ95YgTQvUq1AP93ny8AG+/bKUlE2vZpDygUJAXT8R7PGVD6T0/hUjTZk9HM
+hrLShNjsBMxWWsrc7rdfW+R44pSuw5kHtIBrjRdrr8m1W0BCGTlGjq7Sk0OOZbsO
+sh7x/1kvAgMBAAECggEAQYFXOTs6g/AB0uSn2+brDIA+nh0IdCxlhGuqSgxRxniC
+NHSmtxSnN0PPUka291o3Noga0xnvQMZk9oG/c/uQjjaoFy9ICddY/BP69oVzzoko
+vbDpeIO4aPdhG44s7nZ1ghHTz0C/P4VhL+65AaCBbxZ7lWE+VHqC1smLHDrMYsmr
+PfOxk5LZwdNE/YmZuOj1zX3nQGX+wtK0EMslRbyYjg3EwXUa9d5WNnhyFxt5MD9m
+w9G1rJh5uDUIS60xHUB7ET4BKJMLj+pfy8Ahf3SDFfIzinjuvKg+z1St0nTSNqI7
+QP5KFNvfxqcrtYwf0wU6MGmjJ98FFiDoPY2QQfIlwQKBgQDj3eWDHA02Kr9DAde9
+CuR0y4LixFhVDp7mYBnYxod3GMkPCGhSfuRcA/Tu7JgpACK/zsLi9gbzPkUAS2Y/
+ly8lBFyCYlY96mRF0FgzOUCwP5Xjc9roDgLi1RVM3+Tnv1FcIa0CkoXPuWtmoVYS
+W0S/Y9JKOrV2CEJvirvM3hysHwKBgQDYuipzsV+i3qx4STKXa7F0F8jGp+jr4Pgc
+GqnMo5ysFePFFd4QZ1UHJ8HNmRlXsD+hAawzljVL8yCns8dZOE99xvSjgFzA/p/N
+QToOgCRy6YZ5kZQA1ldZOhoVS5xTJRf7eavLwnV/8VGWxFNXdlvzdZecC8YvwEbh
++gONR9Sw8QKBgQDiBwnSs9CnGl/+J6XIbzWOZRC/v+Oe+5HDEzCQRt241W0a7EwP
+CK+cVpBvL2D/ypKtkHrghwr93Za0ZBZYna62GU1qSAEkCYzNc40JwpDYQzp7FEDg
+ToWuW491YeG9v5kEuqLTGmSBWHugogiuzLm031+JIhM9sZO08Pi4jtoCIQKBgEKC
+XyTB+idxTlxbmjr9wadu5R6RyNkaYQm+5Pz3+MyWqx+AZnWCQIXBnagKo8xVOjGe
+4cmy/wgfxJWyhirLLRncRJcofU581W2X5sksagMZxzrnghP/sy6etgJzAbZCW+nA
+nQga1RQ+altqRGlC2VUNcUliE5/z0cznfM+Oi6cBAoGBAK68wjkperl8+ARQrqII
+Zg41PVrsFv6XmuLWv3bqnsx81BTbnHdqmPT4pScPLZvj+6c3iX8D8wzhzbgZtS0I
+h+r1yMEdzyRZMGtvdi/f/+1TuhOTr/ZeoqYHWEduo8iRYLQBN9gGEkkKBQ4d14zi
+8elaha7vrG/teCrFoeFcXIPr
 -----END PRIVATE KEY-----

extractor/nimo_extractor/nimo_extractor.go

@@ -0,0 +1,9 @@
+package main
+
+import "intimate"
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STNimo))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()

extractor/nimo_extractor/nimo_extractor_test.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"intimate"
+	"log"
+	"testing"
+	"time"
+
+	"github.com/474420502/extractor"
+	"github.com/tebeka/selenium"
+)
+
+func TestMain(t *testing.T) {
+	Execute()
+}
+
+type WaitFor struct {
+	WebDriver selenium.WebDriver
+}
+
+func NewWaitFor(wd selenium.WebDriver) *WaitFor {
+	return &WaitFor{WebDriver: wd}
+}
+
+func (wf *WaitFor) Default(xpath string, do func(elements ...selenium.WebElement) bool) error {
+	return wf.WebDriver.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) {
+		elements, err := wd.FindElements(selenium.ByXPATH, xpath)
+		if err != nil {
+			log.Println(err)
+			return false, err
+		}
+
+		if len(elements) > 0 {
+			if do == nil {
+				return true, nil
+			}
+			if do(elements...) {
+				return true, nil
+			}
+		}
+		return false, nil
+
+	}, time.Second*15)
+}
+
+type LiveInfo struct {
+	Followers int64   `exp:"//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2']" mth:"r:ExtractNumber"`
+	Views     int64   `exp:"//div[contains(@class,'nimo-rm_audience')]//span[@class='text c2']" mth:"r:ExtractNumber"`
+	Giver     []int64 `exp:"//div[contains(@class,'rank-item-after3')]//span[contains(@class,'nimo-currency__count')]"`
+}
+
+func Execute() {
+	wd := intimate.GetChromeDriver(3031)
+	waitfor := NewWaitFor(wd)
+	ps := intimate.NewPerfectShutdown()
+
+	for !ps.IsClose() {
+		// streamer, err := estore.Pop(intimate.PNimo)
+		// if err != nil {
+		// 	log.Println(err)
+		// 	// estore.UpdateError(streamer, err)
+		// 	continue
+		// }
+		// log.Println(streamer)
+		var pagesource string
+		// wd.Get(streamer.LiveUrl.String)
+		wd.Get("https://www.nimo.tv/live/1253835677")
+
+		waitfor.Default("//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2' and text() != '']", nil)
+		waitfor.Default("//div[contains(@class,'rank-item-top3')]", nil)
+		element, err := wd.FindElement(selenium.ByXPATH, "//div[contains(@class,'rank-item-top3')]")
+		if err != nil {
+			log.Println(err)
+		} else {
+			err = element.MoveTo(50, 50)
+			element.Click()
+			if err != nil {
+				log.Println(err)
+			}
+		}
+
+		waitfor.Default("//div[contains(@class,'nimo-rm_audience')]//span[@class='text c2']", nil)
+		pagesource, _ = wd.PageSource()
+		etor := extractor.ExtractHtmlString(pagesource)
+		li := etor.GetObjectByTag(LiveInfo{}).(*LiveInfo)
+		log.Printf("%#v", li)
+	}
+}

extractor/twitch_extractor/tiwtch_extractor.go

@@ -29,7 +29,7 @@ func main() {
 	// var err error
 
 	for !ps.IsClose() {
-		streamer, err := estore.PopNoWait(intimate.Ptwitch, "tags is NULL", 15)
+		streamer, err := estore.Pop(intimate.Ptwitch)
 		if streamer == nil || err != nil {
 			if err != lasterr {
 				log.Println(err, lasterr)

store.go

@@ -228,6 +228,7 @@ func NewStoreExtractor() *StoreExtractor {
 	return &StoreExtractor{db: db}
 }
 
+// PopNoWait 弹出一个不用按时间间隔更新的主播信息, 主要用来测试.
 func (store *StoreExtractor) PopNoWait(platform Platform, condition string, operators ...int32) (*Streamer, error) {
 	tx, err := store.db.Begin()
 	if err != nil {

supervisor_conf/nimo_task1.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=false
+
+[program:nimo_task1]
+environment=DISPLAY=":99"
+directory = MYPATH/bin/nimo_task1/
+command= MYPATH/bin/nimo_task1/nimo_task1 
+# process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+# numprocs=1                                                   ;启动多个进程
+autorestart=true
+stderr_logfile=MYPATH/bin/nimo_task1/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

tasks/nimo/nimo_task1/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+	Execute()
+}

tasks/nimo/nimo_task1/nimo_task1.go

@@ -59,7 +59,7 @@ func Execute() {
 		log.Println(tp.GetURLRawPath(), "rooms:", len(rooms))
 		if len(rooms) == 0 {
 			param.IntSet(1)
-			time.Sleep(time.Minute * 10)
+			time.Sleep(time.Minute * 15)
 			continue
 		}
 

utils.go

@@ -87,9 +87,12 @@ func GetChromeDriver(port int) selenium.WebDriver {
 	caps := selenium.Capabilities{"browserName": "chrome"}
 
 	chromecaps := chrome.Capabilities{}
+
+	// chromecaps.AddExtension("/home/eson/test/myblock.crx")
 	for _, epath := range []string{"../../../crx/myblock.crx", "../../crx/myblock.crx"} {
 		_, err := os.Stat(epath)
 		if err == nil {
+
 			err := chromecaps.AddExtension(epath)
 			if err != nil {
 				panic(err)