110 lines
2.1 KiB
Go
110 lines
2.1 KiB
Go
package parser
|
|
|
|
import (
|
|
"encoding/json"
|
|
"log"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/tidwall/gjson"
|
|
)
|
|
|
|
func TestParser(t *testing.T) {
|
|
a := NewADParser(12)
|
|
data, err := json.Marshal(a)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
t.Error(string(data))
|
|
}
|
|
|
|
type Toutiao struct {
|
|
Parser
|
|
// Url string // "amqp://aso:Wtu(!Ft559W%>mHK~i@172.19.30.60:5672/test_adspider"
|
|
}
|
|
|
|
func (tt *Toutiao) GetSpiderID() int {
|
|
return 1000073
|
|
}
|
|
|
|
func (tt *Toutiao) ToDoParser(adstring string) (string, error) {
|
|
|
|
// adstring = strconv.QuoteToASCII(adstring)
|
|
// adstring = strings.Replace(adstring, "\\", "", -1)
|
|
// log.Println(adstring)
|
|
|
|
var adlist []string
|
|
if err := json.Unmarshal([]byte(adstring), &adlist); err != nil {
|
|
log.Println(err)
|
|
}
|
|
|
|
for _, data := range adlist {
|
|
var v interface{}
|
|
|
|
log.Println(gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).Exists())
|
|
|
|
data = strings.Trim(data, "\"")
|
|
unq, _ := strconv.Unquote("\"" + data + "\"")
|
|
log.Println(gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).Exists())
|
|
data = unq
|
|
|
|
result := gjson.Parse(data)
|
|
log.Println(result.String())
|
|
if result.Get("mixed").Exists() {
|
|
log.Println("this is mixed")
|
|
log.Println(result.Get("mixed.track_url"))
|
|
}
|
|
|
|
json.Unmarshal([]byte(data), &v)
|
|
// spew.Dump(v)
|
|
regexp.Compile(``)
|
|
}
|
|
|
|
adparser := NewADParser(tt.GetSpiderID())
|
|
data, err := adparser.ToJSON()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// log.Println(string(data))
|
|
return string(data), nil
|
|
}
|
|
|
|
func TestParserToutiao(t *testing.T) {
|
|
log.SetFlags(log.Llongfile)
|
|
|
|
tt := Toutiao{}
|
|
tt.ConfigLogDB("logdb.yaml")
|
|
tt.ConfigQueue("queue.yaml")
|
|
ADParserServer(&tt)
|
|
|
|
t.Error("")
|
|
}
|
|
|
|
func TestMQ(t *testing.T) {
|
|
|
|
var l []interface{}
|
|
|
|
data := make(map[string]interface{})
|
|
data["fuck"] = "123"
|
|
|
|
l = append(l, data)
|
|
pjson, err := json.Marshal(&l)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
t.Error(string(pjson))
|
|
|
|
que := NewQueue("amqp://spider:spider@172.16.6.109:5672/test_adspider", "ad_process", "CN")
|
|
que.Push(pjson)
|
|
|
|
// msgs, _, err := ch.Get("ad_process:CN", true)
|
|
// if err != nil {
|
|
// panic(err)
|
|
// }
|
|
// log.Println(string(msgs.Body))
|
|
|
|
}
|