ad_parser/parser_test.go

package parser

import (
	"encoding/json"
	"log"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"github.com/tidwall/gjson"
)

func TestParser(t *testing.T) {
	a := NewADParser(12)
	data, err := json.Marshal(a)
	if err != nil {
		t.Error(err)
	}
	t.Error(string(data))
}

type Toutiao struct {
	Parser
	// Url string // "amqp://aso:Wtu(!Ft559W%>mHK~i@172.19.30.60:5672/test_adspider"
}

func (tt *Toutiao) GetSpiderID() int {
	return 1000073
}

func (tt *Toutiao) ToDoParser(adstring string) (presult string, err error) {

	defer func() {
		if err = recover().(error); err != nil {
			log.Println(err)
		}
	}()

	var adlist []string
	if err = json.Unmarshal([]byte(adstring), &adlist); err != nil {
		log.Println(err)
	}

	for _, data := range adlist {
		log.Println("test json -----------------------")
		var v interface{}

		// log.Println(gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).Get("app").Exists())

		data = strings.Trim(data, "\"")
		unq, _ := strconv.Unquote("\"" + data + "\"")
		log.Println(gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).Get("app").Exists())
		data = unq
		log.Println(gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).Get("app").Exists())

		result := gjson.Parse(data)
		// log.Println(result.String())
		if result.Get("mixed").Exists() {
			log.Println("this is mixed")
			log.Println(result.Get("mixed.track_url").Exists())
		}

		json.Unmarshal([]byte(data), &v)
		// spew.Dump(v)
		regexp.Compile(``)
	}

	adparser := NewADParser(tt.GetSpiderID())
	data, err := adparser.ToJSON()
	if err != nil {
		return "", err
	}
	// log.Println(string(data))
	return string(data), nil
}

func TestParserToutiao(t *testing.T) {
	log.SetFlags(log.Llongfile)

	tt := Toutiao{}
	tt.ConfigLogDB("logdb.yaml")
	tt.ConfigQueue("queue.yaml")
	ADParserServer(&tt)

	t.Error("")
}

func TestMQ(t *testing.T) {

	var l []interface{}

	data := make(map[string]interface{})
	data["fuck"] = "123"

	l = append(l, data)
	pjson, err := json.Marshal(&l)
	if err != nil {
		panic(err)
	}

	t.Error(string(pjson))

	que := NewQueue("amqp://spider:spider@172.16.6.109:5672/test_adspider", "ad_process", "CN")
	que.Push(pjson)

	// msgs, _, err := ch.Get("ad_process:CN", true)
	// if err != nil {
	// 	panic(err)
	// }
	// log.Println(string(msgs.Body))

}