111 lines
1.9 KiB
Go
111 lines
1.9 KiB
Go
package parser
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"log"
|
|
"regexp"
|
|
"testing"
|
|
|
|
"github.com/tidwall/gjson"
|
|
)
|
|
|
|
func TestParser(t *testing.T) {
|
|
a := NewADParser(12)
|
|
data, err := json.Marshal(a)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
t.Error(string(data))
|
|
}
|
|
|
|
type Toutiao struct {
|
|
Parser
|
|
// Url string // "amqp://aso:Wtu(!Ft559W%>mHK~i@172.19.30.60:5672/test_adspider"
|
|
}
|
|
|
|
func (tt *Toutiao) GetSpiderID() int {
|
|
return 1000073
|
|
}
|
|
|
|
func (tt *Toutiao) ToDoParser(adstring string) (presult string, err error) {
|
|
|
|
defer func() {
|
|
if ierr := recover(); ierr != nil {
|
|
log.Println(presult, ierr)
|
|
err = ierr.(error)
|
|
presult = ""
|
|
}
|
|
}()
|
|
|
|
var adlist []string
|
|
if err = json.Unmarshal([]byte(adstring), &adlist); err != nil {
|
|
log.Println(err)
|
|
}
|
|
|
|
for _, data := range adlist {
|
|
|
|
if IsJSON(data) {
|
|
return "", errors.New("json invalid")
|
|
}
|
|
|
|
result := gjson.Parse(data)
|
|
// log.Println(result.String())
|
|
|
|
for i, rkey := range []string{"app", "mixed", "image_recom"} {
|
|
root := result.Get("mixed")
|
|
if root.Exists() {
|
|
|
|
}
|
|
}
|
|
|
|
// spew.Dump(v)
|
|
regexp.Compile(``)
|
|
}
|
|
|
|
adparser := NewADParser(tt.GetSpiderID())
|
|
data, err := adparser.ToJSON()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// log.Println(string(data))
|
|
return string(data), nil
|
|
}
|
|
|
|
func TestParserToutiao(t *testing.T) {
|
|
log.SetFlags(log.Llongfile)
|
|
|
|
tt := Toutiao{}
|
|
tt.ConfigLogDB("logdb.yaml")
|
|
tt.ConfigQueue("queue.yaml")
|
|
ADParserServer(&tt)
|
|
|
|
t.Error("")
|
|
}
|
|
|
|
func TestMQ(t *testing.T) {
|
|
|
|
var l []interface{}
|
|
|
|
data := make(map[string]interface{})
|
|
data["fuck"] = "123"
|
|
|
|
l = append(l, data)
|
|
pjson, err := json.Marshal(&l)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
t.Error(string(pjson))
|
|
|
|
que := NewQueue("amqp://spider:spider@172.16.6.109:5672/test_adspider", "ad_process", "CN")
|
|
que.Push(pjson)
|
|
|
|
// msgs, _, err := ch.Get("ad_process:CN", true)
|
|
// if err != nil {
|
|
// panic(err)
|
|
// }
|
|
// log.Println(string(msgs.Body))
|
|
|
|
}
|