Compare commits
No commits in common. "master" and "feature/add-twitcasting" have entirely different histories.
master
...
feature/ad
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,4 +1,3 @@
|
|||
bin
|
||||
*.log
|
||||
log
|
||||
screenlog.*
|
||||
|
|
|
|||
578
autostore.go
578
autostore.go
|
|
@ -1,578 +0,0 @@
|
|||
package intimate
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StoreExtractorDB 全局的Extractor DB 库链接
|
||||
var StoreExtractorDB *Store
|
||||
|
||||
// TStreamer 全局的Streamer. 在config init 完成初始化
|
||||
var TStreamer *Table
|
||||
|
||||
// TClog 全局的Clog
|
||||
var TClog *Table
|
||||
|
||||
// TStreamerList 全局的streamer list 这个表存的url. 进去可以找到主播的列表. 便于动态更新
|
||||
var TStreamerList *Table
|
||||
|
||||
/*Store 结构体. 必须使用tag. field 数据库字段标签 uid 唯一id字段标签必须存在
|
||||
*/
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// Table 表
|
||||
type Table struct {
|
||||
store *Store
|
||||
name string
|
||||
setting interface{}
|
||||
|
||||
updatesql string
|
||||
selectsql string
|
||||
insertsql string
|
||||
duplicatesql string
|
||||
}
|
||||
|
||||
func NewStore(uri string) *Store {
|
||||
db, err := sql.Open("mysql", uri)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
s := &Store{db: db}
|
||||
return s
|
||||
}
|
||||
|
||||
// Table 选择表.
|
||||
func (store *Store) Table(name string) *Table {
|
||||
table := &Table{store: store}
|
||||
table.name = name
|
||||
|
||||
table.insertsql = `INSERT INTO ` + table.name + `(%s) values(%s)`
|
||||
table.duplicatesql = `INSERT INTO ` + table.name + `(%s) values(%s) ON DUPLICATE KEY UPDATE %s`
|
||||
table.updatesql = `UPDATE ` + table.name + ` SET %s WHERE %s = ?`
|
||||
table.selectsql = `SELECT %s FROM ` + table.name + ` WHERE %s `
|
||||
return table
|
||||
}
|
||||
|
||||
// Queue mysql 队列结构
|
||||
type Queue struct {
|
||||
table *Table
|
||||
obj reflect.Type
|
||||
fieldIndex []int
|
||||
selected string
|
||||
|
||||
cond CondWhere
|
||||
|
||||
uidname string
|
||||
uididx int
|
||||
}
|
||||
|
||||
type CondWhere struct {
|
||||
Condition string
|
||||
CondArgs []interface{}
|
||||
}
|
||||
|
||||
// OperatorType 字典Operator 标志位的类型
|
||||
type OperatorType string
|
||||
|
||||
const (
|
||||
// OpOK 正常
|
||||
OpOK OperatorType = "0"
|
||||
// OpWAIT 等待处理
|
||||
OpWAIT OperatorType = "1000"
|
||||
// OpERROR 错误处理
|
||||
OpERROR OperatorType = "10000"
|
||||
)
|
||||
|
||||
// ConditionDefault 默认的条件
|
||||
func ConditionDefault(platform Platform) CondWhere {
|
||||
return CondWhere{
|
||||
Condition: "platform = ? and operator = 0 and TIMESTAMPDIFF(MINUTE , update_time, CURRENT_TIMESTAMP()) >= update_interval",
|
||||
CondArgs: []interface{}{string(platform)},
|
||||
}
|
||||
}
|
||||
|
||||
// Queue 根据Table生成一个队列. 处理结构. 每次弹出一个 obj 是要处理的结构体 自定义的whereCondition条件
|
||||
func (t *Table) Queue(obj interface{}, whereCondition CondWhere) *Queue {
|
||||
q := &Queue{}
|
||||
q.cond = whereCondition
|
||||
q.obj = reflect.TypeOf(obj)
|
||||
q.table = t
|
||||
q.fieldIndex = []int{} // select 需要配对字段变量的对应index位置
|
||||
|
||||
for i := 0; i < q.obj.NumField(); i++ {
|
||||
field := q.obj.Field(i)
|
||||
if fname, ok := field.Tag.Lookup("field"); ok {
|
||||
q.selected += fname + ","
|
||||
if _, ok := field.Tag.Lookup("uid"); ok {
|
||||
q.uididx = i
|
||||
q.uidname = fname
|
||||
}
|
||||
q.fieldIndex = append(q.fieldIndex, i)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
q.selected = q.selected[:len(q.selected)-1]
|
||||
return q
|
||||
}
|
||||
|
||||
// Pop 队列弹出一个数据(任务). 参考队列处理 不支持嵌套.
|
||||
func (queue *Queue) Pop() (result interface{}, err error) {
|
||||
|
||||
db := queue.table.store.db
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
cerr := tx.Commit()
|
||||
if cerr != nil {
|
||||
log.Println(cerr)
|
||||
log.Println(tx.Rollback())
|
||||
}
|
||||
}()
|
||||
|
||||
selectsql := `SELECT ` + queue.selected + ` FROM ` + queue.table.name + ` WHERE ` + queue.cond.Condition + " limit 1 for update"
|
||||
rows, err := tx.Query(selectsql, queue.cond.CondArgs...)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("table: %s queue is empty. %s", queue.table.name, err.Error())
|
||||
}
|
||||
|
||||
var fields = make([]interface{}, len(queue.fieldIndex))
|
||||
for i := range fields {
|
||||
var iv interface{}
|
||||
fields[i] = &iv
|
||||
}
|
||||
|
||||
if rows.Next() {
|
||||
err = rows.Scan(fields...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
columntypes, err := rows.ColumnTypes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = tx.Exec("UPDATE "+queue.table.name+" SET operator = "+string(OpWAIT)+" WHERE "+queue.uidname+" = ?", fields[queue.uididx])
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
obj := reflect.New(queue.obj).Elem()
|
||||
for i, idx := range queue.fieldIndex {
|
||||
field := obj.Field(idx)
|
||||
convert(*fields[i].(*interface{}), field, columntypes[i])
|
||||
}
|
||||
|
||||
return obj.Addr().Interface(), err
|
||||
}
|
||||
|
||||
// Insert nil 不插入. 不支持嵌套. 必须是Ptr类型
|
||||
func (t *Table) Insert(obj interface{}) error {
|
||||
ov := reflect.ValueOf(obj).Elem()
|
||||
ot := reflect.TypeOf(obj)
|
||||
|
||||
fieldsql := ""
|
||||
argssql := ""
|
||||
|
||||
var args []interface{}
|
||||
for i := 0; i < ov.NumField(); i++ {
|
||||
field := ov.Field(i)
|
||||
ftype := ot.Elem().Field(i)
|
||||
|
||||
if fname, ok := ftype.Tag.Lookup("field"); ok {
|
||||
if flag, ok := ftype.Tag.Lookup("uid"); ok {
|
||||
if flag == "auto" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
k := ftype.Type.Kind()
|
||||
if k == reflect.Ptr || k == reflect.Interface {
|
||||
if !field.IsNil() {
|
||||
felem := field.Elem()
|
||||
args = append(args, felem.Interface())
|
||||
fieldsql += fname + ","
|
||||
argssql += "?,"
|
||||
}
|
||||
} else {
|
||||
args = append(args, field.Interface())
|
||||
fieldsql += fname + ","
|
||||
argssql += "?,"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ssql := fmt.Sprintf(t.insertsql, fieldsql[:len(fieldsql)-1], argssql[:len(argssql)-1])
|
||||
_, err := t.store.db.Exec(ssql, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
// DUpdate ON DUPLICATE KEY UPDATE struct. Field对应的tag field字段
|
||||
type DUpdate struct {
|
||||
Field string // selected 字段
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
// InsertOrUpdate nil 不插入. 不支持嵌套. 必须是Ptr类型
|
||||
func (t *Table) InsertOrUpdate(obj interface{}, updates ...DUpdate) error {
|
||||
ov := reflect.ValueOf(obj).Elem()
|
||||
ot := reflect.TypeOf(obj)
|
||||
|
||||
fieldsql := ""
|
||||
argssql := ""
|
||||
|
||||
var SourceUpdate []*DUpdate
|
||||
var OtherUpdate []*DUpdate
|
||||
for _, u := range updates {
|
||||
if u.Value == nil {
|
||||
SourceUpdate = append(SourceUpdate, &u)
|
||||
} else {
|
||||
OtherUpdate = append(OtherUpdate, &u)
|
||||
}
|
||||
}
|
||||
|
||||
var args []interface{}
|
||||
for i := 0; i < ov.NumField(); i++ {
|
||||
field := ov.Field(i)
|
||||
ftype := ot.Elem().Field(i)
|
||||
|
||||
if fname, ok := ftype.Tag.Lookup("field"); ok {
|
||||
// if flag, ok := ftype.Tag.Lookup("uid"); ok {
|
||||
// if flag == "auto" {
|
||||
// continue
|
||||
// }
|
||||
// }
|
||||
|
||||
k := ftype.Type.Kind()
|
||||
if k == reflect.Ptr || k == reflect.Interface {
|
||||
if !field.IsNil() {
|
||||
felem := field.Elem()
|
||||
args = append(args, felem.Interface())
|
||||
fieldsql += fname + ","
|
||||
argssql += "?,"
|
||||
}
|
||||
} else {
|
||||
args = append(args, field.Interface())
|
||||
fieldsql += fname + ","
|
||||
argssql += "?,"
|
||||
}
|
||||
|
||||
for _, u := range SourceUpdate {
|
||||
if u.Field == fname {
|
||||
u.Value = args[len(args)-1]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var duplicateSet string = ""
|
||||
for _, u := range SourceUpdate {
|
||||
duplicateSet += u.Field + " = ?,"
|
||||
args = append(args, u.Value)
|
||||
}
|
||||
|
||||
for _, u := range OtherUpdate {
|
||||
duplicateSet += u.Field + " = ?,"
|
||||
args = append(args, u.Value)
|
||||
}
|
||||
|
||||
ssql := fmt.Sprintf(t.duplicatesql, fieldsql[:len(fieldsql)-1], argssql[:len(argssql)-1], duplicateSet[:len(duplicateSet)-1])
|
||||
_, err := t.store.db.Exec(ssql, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
// InsertRetAutoID nil 不插入. 不支持嵌套. 并返回auto uid
|
||||
func (t *Table) InsertRetAutoID(obj interface{}) (int64, error) {
|
||||
ov := reflect.ValueOf(obj).Elem()
|
||||
ot := reflect.TypeOf(obj)
|
||||
|
||||
fieldsql := ""
|
||||
argssql := ""
|
||||
|
||||
var args []interface{}
|
||||
for i := 0; i < ov.NumField(); i++ {
|
||||
field := ov.Field(i)
|
||||
ftype := ot.Elem().Field(i)
|
||||
|
||||
if fname, ok := ftype.Tag.Lookup("field"); ok {
|
||||
if flag, ok := ftype.Tag.Lookup("uid"); ok {
|
||||
if flag == "auto" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
k := ftype.Type.Kind()
|
||||
if k == reflect.Ptr || k == reflect.Interface {
|
||||
if !field.IsNil() {
|
||||
felem := field.Elem()
|
||||
args = append(args, felem.Interface())
|
||||
fieldsql += fname + ","
|
||||
argssql += "?,"
|
||||
}
|
||||
} else {
|
||||
args = append(args, field.Interface())
|
||||
fieldsql += fname + ","
|
||||
argssql += "?,"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ssql := fmt.Sprintf(t.insertsql, fieldsql[:len(fieldsql)-1], argssql[:len(argssql)-1])
|
||||
result, err := t.store.db.Exec(ssql, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.LastInsertId()
|
||||
}
|
||||
|
||||
// Update 结构体更新
|
||||
func (t *Table) Update(obj interface{}) error {
|
||||
|
||||
ov := reflect.ValueOf(obj).Elem()
|
||||
ot := reflect.TypeOf(obj)
|
||||
|
||||
fieldsql := ""
|
||||
var uidname string
|
||||
var uidvalue interface{}
|
||||
|
||||
var args []interface{}
|
||||
for i := 0; i < ov.NumField(); i++ {
|
||||
field := ov.Field(i)
|
||||
ftype := ot.Elem().Field(i)
|
||||
|
||||
if fname, ok := ftype.Tag.Lookup("field"); ok {
|
||||
if _, ok := ftype.Tag.Lookup("uid"); ok {
|
||||
if uidvalue != nil {
|
||||
panic(fmt.Errorf("uid must unique, %s and %s", uidname, fname))
|
||||
}
|
||||
uidname = fname
|
||||
uidvalue = field.Interface()
|
||||
continue
|
||||
}
|
||||
|
||||
k := ftype.Type.Kind()
|
||||
if k == reflect.Ptr || k == reflect.Interface {
|
||||
if !field.IsNil() {
|
||||
felem := field.Elem()
|
||||
args = append(args, felem.Interface())
|
||||
fieldsql += fname + " = ?,"
|
||||
}
|
||||
} else {
|
||||
args = append(args, field.Interface())
|
||||
fieldsql += fname + " = ?,"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if uidvalue == nil {
|
||||
panic(fmt.Errorf("update must contain `uid` tag"))
|
||||
}
|
||||
|
||||
usql := fmt.Sprintf(t.updatesql, fieldsql[:len(fieldsql)-1], uidname)
|
||||
args = append(args, uidvalue)
|
||||
_, err := t.store.db.Exec(usql, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateError 更新错误数据
|
||||
func (t *Table) UpdateError(obj interface{}, err error) {
|
||||
|
||||
ov := reflect.ValueOf(obj).Elem()
|
||||
ot := reflect.TypeOf(obj)
|
||||
|
||||
var uidname string
|
||||
var uidvalue interface{}
|
||||
|
||||
for i := 0; i < ov.NumField(); i++ {
|
||||
field := ov.Field(i)
|
||||
ftype := ot.Elem().Field(i)
|
||||
|
||||
if fname, ok := ftype.Tag.Lookup("field"); ok {
|
||||
if _, ok := ftype.Tag.Lookup("uid"); ok {
|
||||
if uidvalue != nil {
|
||||
panic(fmt.Errorf("uid must unique, %s and %s", uidname, fname))
|
||||
}
|
||||
uidname = fname
|
||||
uidvalue = field.Interface()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, dberr := t.store.db.Exec("update "+t.name+" set operator = ?, error_msg = ? where "+uidname+" = ?", 10000, sql.NullString{String: err.Error(), Valid: true}, uidvalue)
|
||||
if dberr != nil {
|
||||
// email tell owner to deal with
|
||||
panic(dberr)
|
||||
}
|
||||
}
|
||||
|
||||
func assign(field reflect.Value, src interface{}) (bool, error) {
|
||||
|
||||
switch field.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
s := asString(src)
|
||||
i64, err := strconv.ParseInt(s, 10, field.Type().Bits())
|
||||
if err != nil {
|
||||
err = strconvErr(err)
|
||||
return false, fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, field.Kind(), err)
|
||||
}
|
||||
field.SetInt(i64)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
|
||||
s := asString(src)
|
||||
u64, err := strconv.ParseUint(s, 10, field.Type().Bits())
|
||||
if err != nil {
|
||||
err = strconvErr(err)
|
||||
return false, fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, field.Kind(), err)
|
||||
}
|
||||
field.SetUint(u64)
|
||||
case reflect.Float32, reflect.Float64:
|
||||
s := asString(src)
|
||||
f64, err := strconv.ParseFloat(s, field.Type().Bits())
|
||||
if err != nil {
|
||||
err = strconvErr(err)
|
||||
return false, fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, field.Kind(), err)
|
||||
}
|
||||
field.SetFloat(f64)
|
||||
case reflect.String:
|
||||
field.SetString(string(src.([]byte)))
|
||||
case reflect.Interface:
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func convert(src interface{}, field reflect.Value, columntype *sql.ColumnType) error {
|
||||
|
||||
// log.Println("type:", field.Type(), ",kind:", field.Kind(), ",field:", field, "scanType:", columntype.ScanType(), "databaseType:", columntype.DatabaseTypeName())
|
||||
if field.Kind() == reflect.Ptr {
|
||||
fn := field.Type().Elem() // New 一个 field.Type().Elem() . 然后判断 columntype 转化 成 NullString Time
|
||||
field.Set(reflect.New(fn))
|
||||
field = field.Elem()
|
||||
// log.Println("type:", fn.Name(), ",kind:", field.Kind(), ",fieldtype:", field.Type())
|
||||
}
|
||||
|
||||
// log.Println(field.Kind(), field, reflect.TypeOf(field).Elem().Name(), columntype.ScanType().Kind())
|
||||
|
||||
if src == nil {
|
||||
return fmt.Errorf("converting NULL to %s is unsupported", field.Kind())
|
||||
}
|
||||
|
||||
switch columntype.DatabaseTypeName() {
|
||||
case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT":
|
||||
isdefault, err := assign(field, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isdefault {
|
||||
s := asString(src)
|
||||
i64, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
err = strconvErr(err)
|
||||
return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, field.Kind(), err)
|
||||
}
|
||||
// reflect.New(reflect.TypeOf(i64))
|
||||
field.Set(reflect.ValueOf(i64))
|
||||
}
|
||||
|
||||
case "FLOAT", "DOUBLE", "DECIMAL":
|
||||
isdefault, err := assign(field, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isdefault {
|
||||
s := asString(src)
|
||||
f64, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
err = strconvErr(err)
|
||||
return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, field.Kind(), err)
|
||||
}
|
||||
field.Set(reflect.ValueOf(f64))
|
||||
}
|
||||
|
||||
case "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB", "JSON":
|
||||
isdefault, err := assign(field, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isdefault {
|
||||
field.Set(reflect.ValueOf(src.([]byte)))
|
||||
}
|
||||
case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT":
|
||||
isdefault, err := assign(field, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isdefault {
|
||||
field.Set(reflect.ValueOf(string(src.([]byte))))
|
||||
}
|
||||
|
||||
case "BIT":
|
||||
var bits []byte = make([]byte, 8)
|
||||
copy(bits, src.([]byte))
|
||||
|
||||
switch field.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
field.SetInt(int64(binary.LittleEndian.Uint64(bits)))
|
||||
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
field.SetUint(binary.LittleEndian.Uint64(bits))
|
||||
|
||||
case reflect.Interface:
|
||||
field.Set(reflect.ValueOf(binary.LittleEndian.Uint64(bits)))
|
||||
|
||||
}
|
||||
|
||||
case "YEAR", "TIME", "DATE", "DATETIME", "TIMESTAMP":
|
||||
s := src.(time.Time)
|
||||
switch field.Interface().(type) {
|
||||
case time.Time:
|
||||
field.Set(reflect.ValueOf(src))
|
||||
case string:
|
||||
field.SetString(s.Format(time.RFC3339Nano))
|
||||
case []byte:
|
||||
field.SetBytes([]byte(s.Format(time.RFC3339Nano)))
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// log.Println(fv, columntype.ScanType().Kind())
|
||||
|
||||
if iscan, ok := field.Addr().Interface().(sql.Scanner); ok {
|
||||
err := iscan.Scan(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
package intimate
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAutoStore(t *testing.T) {
|
||||
uri := "root:@tcp(127.0.0.1:4000)/test?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_unicode_ci"
|
||||
store := NewStore(uri)
|
||||
|
||||
// queue := store.Table("streamer").Queue(TSreamer{}, CondWhere{Condition: "operator = 0"})
|
||||
// re, _ := queue.Pop()
|
||||
|
||||
// pstreamer := re.(*TSreamer)
|
||||
// m := make(map[string]interface{})
|
||||
// json.Unmarshal(pstreamer.Iface.([]byte), &m)
|
||||
// spew.Println(re.(*TSreamer), m)
|
||||
|
||||
streamer := &TSreamer{}
|
||||
|
||||
streamer.Uid = 1
|
||||
streamer.UserID = &sql.NullString{String: "xixi", Valid: true}
|
||||
streamer.Name = "streamer"
|
||||
streamer.Operator = 0
|
||||
streamer.Bit = 0b11
|
||||
// streamer.Ext = &sql.NullString{String: "ext", Valid: true}
|
||||
|
||||
tag := make(map[string]interface{})
|
||||
tag["json"] = true
|
||||
tag["name"] = "test"
|
||||
btag, err := json.Marshal(tag)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
streamer.Iface = btag
|
||||
now := time.Now()
|
||||
streamer.UpdateTime = &now
|
||||
|
||||
err = store.Table("streamer").InsertOrUpdate(streamer, DUpdate{Field: "userid"})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
type TSreamer struct {
|
||||
Uid int `field:"uid" uid:"auto"`
|
||||
Name interface{} `field:"name"`
|
||||
UserID *sql.NullString `field:"userid"`
|
||||
Ext *sql.NullString `field:"ext"`
|
||||
Iface interface{} `field:"tag"`
|
||||
Bit uint64 `field:"bit"`
|
||||
Operator int `field:"operator"`
|
||||
UpdateTime *time.Time `field:"update_time"`
|
||||
}
|
||||
10
build.sh
10
build.sh
|
|
@ -7,17 +7,13 @@ src=`pwd`
|
|||
for path in `ls -d $source_tasks`
|
||||
do
|
||||
echo $path
|
||||
projectname=${path##*/}
|
||||
projectworkspace=$src/bin/$projectname
|
||||
cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname
|
||||
cd $path && go build
|
||||
cd $src
|
||||
done
|
||||
|
||||
for path in `ls -d $extractor_tasks`
|
||||
do
|
||||
echo $path
|
||||
projectname=${path##*/}
|
||||
projectworkspace=$src/bin/$projectname
|
||||
cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname
|
||||
cd $path && go build
|
||||
cd $src
|
||||
done
|
||||
done
|
||||
|
|
|
|||
12
config.go
12
config.go
|
|
@ -18,18 +18,6 @@ func init() {
|
|||
// storeOpenrec = NewStore()
|
||||
|
||||
log.SetFlags(log.Llongfile | log.Ltime)
|
||||
|
||||
// StoreExtractorDB 全局的Extractor DB 库链接
|
||||
StoreExtractorDB = NewStore(InitConfig.Database.ExtractorURI)
|
||||
|
||||
// TStreamer 全局的Streamer
|
||||
TStreamer = StoreExtractorDB.Table("streamer")
|
||||
|
||||
// TClog 全局的Clog
|
||||
TClog = StoreExtractorDB.Table("collect_log")
|
||||
|
||||
// TStreamerList 全局的streamer list 这个表存的url. 进去可以找到主播的列表. 便于动态更新
|
||||
TStreamerList = StoreExtractorDB.Table("streamer_list")
|
||||
}
|
||||
|
||||
// Config 配置
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
database:
|
||||
source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_unicode_ci"
|
||||
extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_unicode_ci"
|
||||
source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true&loc=Local"
|
||||
extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true&loc=Local"
|
||||
376
convert.go
376
convert.go
|
|
@ -1,376 +0,0 @@
|
|||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Type conversions for Scan.
|
||||
|
||||
package intimate
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
var errNilPtr = errors.New("destination pointer is nil") // embedded in descriptive error
|
||||
|
||||
// convertAssignRows copies to dest the value in src, converting it if possible.
|
||||
// An error is returned if the copy would result in loss of information.
|
||||
// dest should be a pointer type. If rows is passed in, the rows will
|
||||
// be used as the parent for any cursor values converted from a
|
||||
// driver.Rows to a *Rows.
|
||||
// func convertAssignRows(dest, src interface{}, rows *sql.Rows) error {
|
||||
// // Common cases, without reflect.
|
||||
// switch s := src.(type) {
|
||||
// case string:
|
||||
// switch d := dest.(type) {
|
||||
// case *string:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = s
|
||||
// return nil
|
||||
// case *[]byte:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = []byte(s)
|
||||
// return nil
|
||||
// case *sql.RawBytes:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = append((*d)[:0], s...)
|
||||
// return nil
|
||||
// }
|
||||
// case []byte:
|
||||
// switch d := dest.(type) {
|
||||
// case *string:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = string(s)
|
||||
// return nil
|
||||
// case *interface{}:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = cloneBytes(s)
|
||||
// return nil
|
||||
// case *[]byte:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = cloneBytes(s)
|
||||
// return nil
|
||||
// case *sql.RawBytes:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = s
|
||||
// return nil
|
||||
// }
|
||||
// case time.Time:
|
||||
// switch d := dest.(type) {
|
||||
// case *time.Time:
|
||||
// *d = s
|
||||
// return nil
|
||||
// case *string:
|
||||
// *d = s.Format(time.RFC3339Nano)
|
||||
// return nil
|
||||
// case *[]byte:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = []byte(s.Format(time.RFC3339Nano))
|
||||
// return nil
|
||||
// case *sql.RawBytes:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = s.AppendFormat((*d)[:0], time.RFC3339Nano)
|
||||
// return nil
|
||||
// }
|
||||
// case decimalDecompose:
|
||||
// switch d := dest.(type) {
|
||||
// case decimalCompose:
|
||||
// return d.Compose(s.Decompose(nil))
|
||||
// }
|
||||
// case nil:
|
||||
// switch d := dest.(type) {
|
||||
// case *interface{}:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = nil
|
||||
// return nil
|
||||
// case *[]byte:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = nil
|
||||
// return nil
|
||||
// case *sql.RawBytes:
|
||||
// if d == nil {
|
||||
// return errNilPtr
|
||||
// }
|
||||
// *d = nil
|
||||
// return nil
|
||||
// }
|
||||
// // The driver is returning a cursor the client may iterate over.
|
||||
// }
|
||||
|
||||
// var sv reflect.Value
|
||||
|
||||
// switch d := dest.(type) {
|
||||
// case *string:
|
||||
// sv = reflect.ValueOf(src)
|
||||
// switch sv.Kind() {
|
||||
// case reflect.Bool,
|
||||
// reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
|
||||
// reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
|
||||
// reflect.Float32, reflect.Float64:
|
||||
// *d = asString(src)
|
||||
// return nil
|
||||
// }
|
||||
// case *[]byte:
|
||||
// sv = reflect.ValueOf(src)
|
||||
// if b, ok := asBytes(nil, sv); ok {
|
||||
// *d = b
|
||||
// return nil
|
||||
// }
|
||||
// case *sql.RawBytes:
|
||||
// sv = reflect.ValueOf(src)
|
||||
// if b, ok := asBytes([]byte(*d)[:0], sv); ok {
|
||||
// *d = sql.RawBytes(b)
|
||||
// return nil
|
||||
// }
|
||||
// case *bool:
|
||||
// bv, err := driver.Bool.ConvertValue(src)
|
||||
// if err == nil {
|
||||
// *d = bv.(bool)
|
||||
// }
|
||||
// return err
|
||||
// case *interface{}:
|
||||
// *d = src
|
||||
// return nil
|
||||
// }
|
||||
|
||||
// if scanner, ok := dest.(sql.Scanner); ok {
|
||||
// return scanner.Scan(src)
|
||||
// }
|
||||
|
||||
// dpv := reflect.ValueOf(dest)
|
||||
// if dpv.Kind() != reflect.Ptr {
|
||||
// return errors.New("destination not a pointer")
|
||||
// }
|
||||
// if dpv.IsNil() {
|
||||
// return errNilPtr
|
||||
// }
|
||||
|
||||
// if !sv.IsValid() {
|
||||
// sv = reflect.ValueOf(src)
|
||||
// }
|
||||
|
||||
// dv := reflect.Indirect(dpv)
|
||||
// if sv.IsValid() && sv.Type().AssignableTo(dv.Type()) {
|
||||
// switch b := src.(type) {
|
||||
// case []byte:
|
||||
// dv.Set(reflect.ValueOf(cloneBytes(b)))
|
||||
// default:
|
||||
// dv.Set(sv)
|
||||
// }
|
||||
// return nil
|
||||
// }
|
||||
|
||||
// if dv.Kind() == sv.Kind() && sv.Type().ConvertibleTo(dv.Type()) {
|
||||
// dv.Set(sv.Convert(dv.Type()))
|
||||
// return nil
|
||||
// }
|
||||
|
||||
// // The following conversions use a string value as an intermediate representation
|
||||
// // to convert between various numeric types.
|
||||
// //
|
||||
// // This also allows scanning into user defined types such as "type Int int64".
|
||||
// // For symmetry, also check for string destination types.
|
||||
// switch dv.Kind() {
|
||||
// case reflect.Ptr:
|
||||
// if src == nil {
|
||||
// dv.Set(reflect.Zero(dv.Type()))
|
||||
// return nil
|
||||
// }
|
||||
// dv.Set(reflect.New(dv.Type().Elem()))
|
||||
// return convertAssignRows(dv.Interface(), src, rows)
|
||||
// case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
// if src == nil {
|
||||
// return fmt.Errorf("converting NULL to %s is unsupported", dv.Kind())
|
||||
// }
|
||||
// s := asString(src)
|
||||
// i64, err := strconv.ParseInt(s, 10, dv.Type().Bits())
|
||||
// if err != nil {
|
||||
// err = strconvErr(err)
|
||||
// return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err)
|
||||
// }
|
||||
// dv.SetInt(i64)
|
||||
// return nil
|
||||
// case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
// if src == nil {
|
||||
// return fmt.Errorf("converting NULL to %s is unsupported", dv.Kind())
|
||||
// }
|
||||
// s := asString(src)
|
||||
// u64, err := strconv.ParseUint(s, 10, dv.Type().Bits())
|
||||
// if err != nil {
|
||||
// err = strconvErr(err)
|
||||
// return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err)
|
||||
// }
|
||||
// dv.SetUint(u64)
|
||||
// return nil
|
||||
// case reflect.Float32, reflect.Float64:
|
||||
// if src == nil {
|
||||
// return fmt.Errorf("converting NULL to %s is unsupported", dv.Kind())
|
||||
// }
|
||||
// s := asString(src)
|
||||
// f64, err := strconv.ParseFloat(s, dv.Type().Bits())
|
||||
// if err != nil {
|
||||
// err = strconvErr(err)
|
||||
// return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err)
|
||||
// }
|
||||
// dv.SetFloat(f64)
|
||||
// return nil
|
||||
// case reflect.String:
|
||||
// if src == nil {
|
||||
// return fmt.Errorf("converting NULL to %s is unsupported", dv.Kind())
|
||||
// }
|
||||
// switch v := src.(type) {
|
||||
// case string:
|
||||
// dv.SetString(v)
|
||||
// return nil
|
||||
// case []byte:
|
||||
// dv.SetString(string(v))
|
||||
// return nil
|
||||
// }
|
||||
// }
|
||||
|
||||
// return fmt.Errorf("unsupported Scan, storing driver.Value type %T into type %T", src, dest)
|
||||
// }
|
||||
|
||||
func strconvErr(err error) error {
|
||||
if ne, ok := err.(*strconv.NumError); ok {
|
||||
return ne.Err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func cloneBytes(b []byte) []byte {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
c := make([]byte, len(b))
|
||||
copy(c, b)
|
||||
return c
|
||||
}
|
||||
|
||||
func asString(src interface{}) string {
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
return v
|
||||
case []byte:
|
||||
return string(v)
|
||||
}
|
||||
rv := reflect.ValueOf(src)
|
||||
switch rv.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
return strconv.FormatInt(rv.Int(), 10)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
return strconv.FormatUint(rv.Uint(), 10)
|
||||
case reflect.Float64:
|
||||
return strconv.FormatFloat(rv.Float(), 'g', -1, 64)
|
||||
case reflect.Float32:
|
||||
return strconv.FormatFloat(rv.Float(), 'g', -1, 32)
|
||||
case reflect.Bool:
|
||||
return strconv.FormatBool(rv.Bool())
|
||||
}
|
||||
return fmt.Sprintf("%v", src)
|
||||
}
|
||||
|
||||
func asBytes(buf []byte, rv reflect.Value) (b []byte, ok bool) {
|
||||
switch rv.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
return strconv.AppendInt(buf, rv.Int(), 10), true
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
return strconv.AppendUint(buf, rv.Uint(), 10), true
|
||||
case reflect.Float32:
|
||||
return strconv.AppendFloat(buf, rv.Float(), 'g', -1, 32), true
|
||||
case reflect.Float64:
|
||||
return strconv.AppendFloat(buf, rv.Float(), 'g', -1, 64), true
|
||||
case reflect.Bool:
|
||||
return strconv.AppendBool(buf, rv.Bool()), true
|
||||
case reflect.String:
|
||||
s := rv.String()
|
||||
return append(buf, s...), true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
|
||||
|
||||
// callValuerValue returns vr.Value(), with one exception:
|
||||
// If vr.Value is an auto-generated method on a pointer type and the
|
||||
// pointer is nil, it would panic at runtime in the panicwrap
|
||||
// method. Treat it like nil instead.
|
||||
// Issue 8415.
|
||||
//
|
||||
// This is so people can implement driver.Value on value types and
|
||||
// still use nil pointers to those types to mean nil/NULL, just like
|
||||
// string/*string.
|
||||
//
|
||||
// This function is mirrored in the database/sql/driver package.
|
||||
func callValuerValue(vr driver.Valuer) (v driver.Value, err error) {
|
||||
if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr &&
|
||||
rv.IsNil() &&
|
||||
rv.Type().Elem().Implements(valuerReflectType) {
|
||||
return nil, nil
|
||||
}
|
||||
return vr.Value()
|
||||
}
|
||||
|
||||
// decimal composes or decomposes a decimal value to and from individual parts.
|
||||
// There are four parts: a boolean negative flag, a form byte with three possible states
|
||||
// (finite=0, infinite=1, NaN=2), a base-2 big-endian integer
|
||||
// coefficient (also known as a significand) as a []byte, and an int32 exponent.
|
||||
// These are composed into a final value as "decimal = (neg) (form=finite) coefficient * 10 ^ exponent".
|
||||
// A zero length coefficient is a zero value.
|
||||
// The big-endian integer coefficient stores the most significant byte first (at coefficient[0]).
|
||||
// If the form is not finite the coefficient and exponent should be ignored.
|
||||
// The negative parameter may be set to true for any form, although implementations are not required
|
||||
// to respect the negative parameter in the non-finite form.
|
||||
//
|
||||
// Implementations may choose to set the negative parameter to true on a zero or NaN value,
|
||||
// but implementations that do not differentiate between negative and positive
|
||||
// zero or NaN values should ignore the negative parameter without error.
|
||||
// If an implementation does not support Infinity it may be converted into a NaN without error.
|
||||
// If a value is set that is larger than what is supported by an implementation,
|
||||
// an error must be returned.
|
||||
// Implementations must return an error if a NaN or Infinity is attempted to be set while neither
|
||||
// are supported.
|
||||
//
|
||||
// NOTE(kardianos): This is an experimental interface. See https://golang.org/issue/30870
|
||||
type decimal interface {
|
||||
decimalDecompose
|
||||
decimalCompose
|
||||
}
|
||||
|
||||
type decimalDecompose interface {
|
||||
// Decompose returns the internal decimal state in parts.
|
||||
// If the provided buf has sufficient capacity, buf may be returned as the coefficient with
|
||||
// the value set and length set as appropriate.
|
||||
Decompose(buf []byte) (form byte, negative bool, coefficient []byte, exponent int32)
|
||||
}
|
||||
|
||||
type decimalCompose interface {
|
||||
// Compose sets the internal decimal value from parts. If the value cannot be
|
||||
// represented then an error should be returned.
|
||||
Compose(form byte, negative bool, coefficient []byte, exponent int32) error
|
||||
}
|
||||
BIN
crx/myblock.crx
BIN
crx/myblock.crx
Binary file not shown.
|
|
@ -1,28 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDA6O6owySvzaJ8
|
||||
iv6aXRMcuk0lnNKquPNVHeraadSkBVhtaJqPAT7saOVnn9g4s/21OBV6wGmWdl5W
|
||||
5+8h+F4qXrlGO8N1Tl8mmBDetqg6T6RrI9vB2Th9hjlp2PCGCEg/lX+tx25qBiDG
|
||||
9uCB3pU4KMyfof1Z2jbo7aOfttgxZ1YdssenB//+31QjeZVw7wZACKaSzl6Z+WvY
|
||||
pastMZ95YgTQvUq1AP93ny8AG+/bKUlE2vZpDygUJAXT8R7PGVD6T0/hUjTZk9HM
|
||||
hrLShNjsBMxWWsrc7rdfW+R44pSuw5kHtIBrjRdrr8m1W0BCGTlGjq7Sk0OOZbsO
|
||||
sh7x/1kvAgMBAAECggEAQYFXOTs6g/AB0uSn2+brDIA+nh0IdCxlhGuqSgxRxniC
|
||||
NHSmtxSnN0PPUka291o3Noga0xnvQMZk9oG/c/uQjjaoFy9ICddY/BP69oVzzoko
|
||||
vbDpeIO4aPdhG44s7nZ1ghHTz0C/P4VhL+65AaCBbxZ7lWE+VHqC1smLHDrMYsmr
|
||||
PfOxk5LZwdNE/YmZuOj1zX3nQGX+wtK0EMslRbyYjg3EwXUa9d5WNnhyFxt5MD9m
|
||||
w9G1rJh5uDUIS60xHUB7ET4BKJMLj+pfy8Ahf3SDFfIzinjuvKg+z1St0nTSNqI7
|
||||
QP5KFNvfxqcrtYwf0wU6MGmjJ98FFiDoPY2QQfIlwQKBgQDj3eWDHA02Kr9DAde9
|
||||
CuR0y4LixFhVDp7mYBnYxod3GMkPCGhSfuRcA/Tu7JgpACK/zsLi9gbzPkUAS2Y/
|
||||
ly8lBFyCYlY96mRF0FgzOUCwP5Xjc9roDgLi1RVM3+Tnv1FcIa0CkoXPuWtmoVYS
|
||||
W0S/Y9JKOrV2CEJvirvM3hysHwKBgQDYuipzsV+i3qx4STKXa7F0F8jGp+jr4Pgc
|
||||
GqnMo5ysFePFFd4QZ1UHJ8HNmRlXsD+hAawzljVL8yCns8dZOE99xvSjgFzA/p/N
|
||||
QToOgCRy6YZ5kZQA1ldZOhoVS5xTJRf7eavLwnV/8VGWxFNXdlvzdZecC8YvwEbh
|
||||
+gONR9Sw8QKBgQDiBwnSs9CnGl/+J6XIbzWOZRC/v+Oe+5HDEzCQRt241W0a7EwP
|
||||
CK+cVpBvL2D/ypKtkHrghwr93Za0ZBZYna62GU1qSAEkCYzNc40JwpDYQzp7FEDg
|
||||
ToWuW491YeG9v5kEuqLTGmSBWHugogiuzLm031+JIhM9sZO08Pi4jtoCIQKBgEKC
|
||||
XyTB+idxTlxbmjr9wadu5R6RyNkaYQm+5Pz3+MyWqx+AZnWCQIXBnagKo8xVOjGe
|
||||
4cmy/wgfxJWyhirLLRncRJcofU581W2X5sksagMZxzrnghP/sy6etgJzAbZCW+nA
|
||||
nQga1RQ+altqRGlC2VUNcUliE5/z0cznfM+Oi6cBAoGBAK68wjkperl8+ARQrqII
|
||||
Zg41PVrsFv6XmuLWv3bqnsx81BTbnHdqmPT4pScPLZvj+6c3iX8D8wzhzbgZtS0I
|
||||
h+r1yMEdzyRZMGtvdi/f/+1TuhOTr/ZeoqYHWEduo8iRYLQBN9gGEkkKBQ4d14zi
|
||||
8elaha7vrG/teCrFoeFcXIPr
|
||||
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDSG09DSvB03TOe
|
||||
eOmQwfiCIf0wa2WRB31ewxa6i/PRgEKeJSUvIsIuaECUer2ss+J3rwSS2lDpGuiw
|
||||
FnsVyZqKI/+Rcuc83YJGYg6OAzVMz6UL8YCWhXu3huTJ+V+a5iNereIC69ZERRJt
|
||||
nXlWqsq6HKya+6BP9sX9CI4GTHQrnWBysAxsswhdnnnRvu+GxglWafSIzuS6OizT
|
||||
1M1CmkZxNvDJhTSOR7SJlIYm2kM5/fIL53BdndF2IGAjfV1WV7AjwhTfun5cViEO
|
||||
i8niQUIMY4L0AiO9grFD1g1xIYkeuVBoLxOUBzPxJwQmb64gseb9Dvt0BKLRGoou
|
||||
SIOyE+KVAgMBAAECggEAI4b6J2kR0VUBEDwmVHO0K38HUstqNHSVgrNO0dLt8sAz
|
||||
I44o5DhGqPW4a9L4ZS5SrkWyKonPcic6buISRIwfPVoacjQBfVWAXJnil6lbtyYK
|
||||
ZMNcqLcgBRfCcpOgEq91DiKta6yIwekDFXVyCdFd78v+9ML1J+hUsLVkXJTLdP88
|
||||
PGamRWVd6vGy3QMRjyM29GLPgS+/6Vrp1cptSuYNqYhlszohmu8lBvzjH9jbPh9d
|
||||
GFrrd8Bs7IRCdtKZig/3fbln4JEyyOYE+gcT2jplPksB6mR/5DBIdkVbeuFwGB0+
|
||||
h1/PKlprNQt7+Ei0HhHnTib7lZP8WGo4HkSi7PsAGQKBgQD1Ptho0wJiI2+6gL1O
|
||||
iNsEJVKIQ2Sxdx3wI/qudphM99t6xKCpPyVI2Nd9PBf2jbZjGAaz+P/KQYxEqb6i
|
||||
PRcQ+i99wCQoRfnRvUbKA4goEpKwRXmvn+499dm6D5pEuumOXGQYCmaFXuLTRN/I
|
||||
BL6GNgLtoZAlLjUXaWtk8TszGQKBgQDbUf3p3HLpCjRvRDW/vA5xj+08t7xtF9uO
|
||||
NilGK79uOA4VnxE2w3ioYqQ7t3I8J/0rAzGKq3tylg4QX6UpQ4b2koRr2B3cqoAk
|
||||
dsRdNWAHwCNepz8hTLsZyuihzbNv2nHmoqhzjK/FcrBHx5NAM+T6OBpLzQBnbUzk
|
||||
3wIcqm223QKBgQDo/IRxyY0pGMtLXoT6ODACF0b6JzRhGG37tuKvngGAlbQQRP7w
|
||||
6wmL1F2cH1wQon7UU34CupqfVnhgvvZZgToJqfU2PTTcgeYc6Pl4b7SJhWOQTOCX
|
||||
BZQ7jvYCulHv27aIxaNd53uQVx2cYoFKr58lN+i+QtADUoujq0YYxshb+QKBgQDW
|
||||
ZOti7kZCeuBRGIu2V56C8uBFp5MBzf2polZsqx1iIFfcWPfZ4fGUIYFMgwKfvbOl
|
||||
lWSbmxB9LiSnaugoU0OezBG43rYqXV4Qxy0jtKagTPoGcFWtNrX7+7e3XD8Zi6Am
|
||||
hkFHW3MEAB5EvNq8Oz6OP8Os78SCVn2BimMlJJFF3QKBgQCF+aEAiBv+ivcmHUeP
|
||||
2eBq9nLltPFAfXJ/p31MMQ6Jgo36DBqUeoLeyq/WfIXvwqbVbP9fANZrKoTPbI97
|
||||
dilCHUoO33rafXJy6jtaggtpz14tt9soecTop0vM/rU7tGtfBe6NXg9LRl+oDJCU
|
||||
37I3a9Is+2CLyAUXWCk9mLfFsQ==
|
||||
-----END PRIVATE KEY-----
|
||||
|
|
|
|||
|
|
@ -1,159 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
sessionstr := `
|
||||
-H 'authority: www.mirrativ.com'
|
||||
-H 'accept: application/json'
|
||||
-H 'x-timezone: Asia/Shanghai'
|
||||
-H 'x-csrf-token: F3Ojd6RBtApP6YAZzVn-9jWN1of159VxAqOQL1Zn'
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
|
||||
-H 'content-type: application/json'
|
||||
-H 'sec-fetch-site: same-origin'
|
||||
-H 'sec-fetch-mode: cors'
|
||||
-H 'sec-fetch-dest: empty'
|
||||
-H 'referer: https://www.mirrativ.com/live/O5Ia4iX9c5CeZj7DFtg52Q'
|
||||
-H 'accept-language: zh-CN,zh;q=0.9,ja;q=0.8'
|
||||
-H 'cookie: f=A2D75F0E-D218-11EA-A042-452BF6D21CE8; _ga=GA1.2.689947597.1596081392; mr_id=kxb65LddGMZf5C28jkR_tGCZD_ZFOAepD5gfXO7eNjfPMB8EKYvU1Vg_Y29V1lsa; _gid=GA1.2.2116692650.1600139685; lang=ja'`
|
||||
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
gprofile := gcurl.Parse(`curl 'https://www.mirrativ.com/api/user/profile?user_id=103383701'` + sessionstr)
|
||||
tpProfile := gprofile.CreateTemporary(nil)
|
||||
tpProfileUserID := tpProfile.QueryParam("user_id")
|
||||
|
||||
g := gcurl.Parse(`culr 'https://www.mirrativ.com/api/live/live?live_id=O5Ia4iX9c5CeZj7DFtg52Q'` + sessionstr)
|
||||
tpLive := g.CreateTemporary(nil)
|
||||
tpLiveID := tpLive.QueryParam("live_id")
|
||||
|
||||
var lasterr error
|
||||
queue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.PMirrativ))
|
||||
for !ps.IsClose() {
|
||||
istreamer, err := queue.Pop()
|
||||
if err != nil {
|
||||
if lasterr != err {
|
||||
lasterr = err
|
||||
log.Println(err)
|
||||
}
|
||||
ps.Wait(time.Second * 5)
|
||||
continue
|
||||
}
|
||||
|
||||
now := &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
streamer := istreamer.(*intimate.Streamer)
|
||||
streamer.UpdateTime = now
|
||||
userid := *streamer.UserId
|
||||
log.Println(userid)
|
||||
|
||||
tpProfileUserID.StringSet(userid)
|
||||
resp, err := tpProfile.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
clog := &intimate.CollectLog{}
|
||||
clog.Platform = intimate.PMirrativ
|
||||
clog.UpdateTime = now
|
||||
clog.UserId = userid
|
||||
clog.StreamerUid = streamer.Uid
|
||||
|
||||
profilejson := gjson.ParseBytes(resp.Content())
|
||||
if result := profilejson.Get("follower_num"); result.Exists() {
|
||||
clog.Followers = &sql.NullInt64{Int64: result.Int(), Valid: true}
|
||||
}
|
||||
|
||||
if result := profilejson.Get("onlive.live_id"); result.Exists() {
|
||||
liveID := result.String()
|
||||
tpLiveID.StringSet(liveID)
|
||||
resp, err = tpLive.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
livejson := gjson.ParseBytes(resp.Content())
|
||||
if result := livejson.Get("total_viewer_num"); result.Exists() {
|
||||
clog.Views = &sql.NullInt64{Int64: result.Int(), Valid: true}
|
||||
}
|
||||
|
||||
if result := livejson.Get("title"); result.Exists() {
|
||||
clog.LiveTitle = &sql.NullString{String: result.String(), Valid: true}
|
||||
}
|
||||
|
||||
if result := livejson.Get("started_at"); result.Exists() {
|
||||
clog.LiveStartTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true}
|
||||
}
|
||||
|
||||
if result := livejson.Get("heartbeated_at"); result.Exists() {
|
||||
clog.LiveEndTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true}
|
||||
}
|
||||
|
||||
if result := livejson.Get("app_id"); result.Exists() {
|
||||
streamer.Channel = &sql.NullString{String: result.String(), Valid: true}
|
||||
}
|
||||
|
||||
if result := livejson.Get("timeline.#.app.short_title"); result.Exists() {
|
||||
for _, tl := range result.Array() {
|
||||
var tags []string = []string{tl.String()}
|
||||
jtags, _ := json.Marshal(tags)
|
||||
streamer.Tags = jtags
|
||||
clog.Tags = jtags
|
||||
break
|
||||
}
|
||||
} else {
|
||||
log.Println(string(resp.Content()))
|
||||
return
|
||||
}
|
||||
|
||||
if result := livejson.Get("gift_ranking_url"); result.Exists() {
|
||||
// streamer.Channel = &sql.NullString{String: result.String(), Valid: true}
|
||||
gifturl := "curl '" + result.String() + "&type=monthly&cursor='" + sessionstr
|
||||
ggift := gcurl.Parse(gifturl)
|
||||
tp := ggift.CreateTemporary(nil)
|
||||
tp.SetURLRawPath("/api/gift/ranking")
|
||||
pcursor := tp.QueryParam("cursor")
|
||||
var gratuity int64 = 0
|
||||
|
||||
for {
|
||||
giftdata, err := tp.Execute()
|
||||
giftjson := gjson.ParseBytes(giftdata.Content())
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
for _, rpoint := range giftjson.Get("ranking.#.point").Array() {
|
||||
gratuity += rpoint.Int()
|
||||
}
|
||||
}
|
||||
ncursor := giftjson.Get("next_cursor").String()
|
||||
if ncursor == "" {
|
||||
break
|
||||
}
|
||||
pcursor.StringSet(ncursor)
|
||||
}
|
||||
// https://www.mirrativ.com/gift/ranking?live_id=O5Ia4iX9c5CeZj7DFtg52Q&obfuscated_user_id=PgIBEgc6jVc
|
||||
clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true}
|
||||
}
|
||||
|
||||
cid, err := intimate.TClog.InsertRetAutoID(clog)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
streamer.LatestLogUid = cid
|
||||
}
|
||||
|
||||
intimate.TStreamer.Update(streamer)
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDo(t *testing.T) {
|
||||
main()
|
||||
}
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"intimate"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/extractor"
|
||||
"github.com/tebeka/selenium"
|
||||
)
|
||||
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
// var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STNimo))
|
||||
|
||||
// // estore 解析存储连接实例
|
||||
// var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
func main() {
|
||||
Execute()
|
||||
}
|
||||
|
||||
type LiveInfo struct {
|
||||
Followers int64 `exp:"//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2']" mth:"r:ExtractNumber"`
|
||||
Views int64 `exp:"//div[contains(@class,'nimo-rm_audience')]//span[@class='text c2']" mth:"r:ExtractNumber"`
|
||||
Channel string `exp:"//div[contains(@class,'nimo-rm_type')]//span"`
|
||||
Gratuity []int64 `exp:"//div[contains(@class,'rank-item-after3')]//span[contains(@class,'nimo-currency__count')]"`
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
adriver := intimate.GetChromeDriver()
|
||||
count := 0
|
||||
countlimit := 200
|
||||
wd := adriver.Webdriver
|
||||
waitfor := intimate.NewWaitFor(wd)
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
|
||||
queue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.PNimo))
|
||||
|
||||
for !ps.IsClose() {
|
||||
istreamer, err := queue.Pop()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
intimate.TStreamer.UpdateError(istreamer, err)
|
||||
continue
|
||||
}
|
||||
|
||||
streamer := istreamer.(*intimate.Streamer)
|
||||
|
||||
wd.Get(streamer.LiveUrl.String)
|
||||
// wd.Get("https://www.nimo.tv/live/1253835677")
|
||||
|
||||
waitfor.Default("//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2' and text() != '']", nil)
|
||||
waitfor.WaitWithTimeout("//div[contains(@class,'rank-item-top3')]", 7*time.Second, nil)
|
||||
element, err := wd.FindElement(selenium.ByXPATH, "//div[contains(@class,'rank-item-top3')]")
|
||||
if err != nil {
|
||||
log.Println(streamer.Uid, err)
|
||||
} else {
|
||||
err = element.MoveTo(50, 50)
|
||||
element.Click()
|
||||
if err != nil {
|
||||
log.Println(streamer.Uid, err)
|
||||
}
|
||||
}
|
||||
waitfor.Default("//div[contains(@class,'nimo-rm_audience')]//span[@class='text c2']", nil)
|
||||
|
||||
var pagesource string
|
||||
pagesource, _ = wd.PageSource()
|
||||
|
||||
etor := extractor.ExtractHtmlString(pagesource)
|
||||
li := etor.GetObjectByTag(LiveInfo{}).(*LiveInfo)
|
||||
// log.Printf("%#v", li)
|
||||
|
||||
utime := sql.NullTime{Time: time.Now(), Valid: true}
|
||||
|
||||
clog := &intimate.CollectLog{}
|
||||
clog.Platform = intimate.PNimo
|
||||
clog.Followers = &sql.NullInt64{Int64: li.Followers, Valid: true}
|
||||
clog.Views = &sql.NullInt64{Int64: li.Views, Valid: true}
|
||||
clog.UpdateTime = &utime
|
||||
clog.StreamerUid = streamer.Uid
|
||||
|
||||
var sum int64 = 0
|
||||
for _, v := range li.Gratuity {
|
||||
sum += v
|
||||
}
|
||||
clog.Gratuity = &sql.NullInt64{Int64: sum, Valid: true}
|
||||
|
||||
cuid, err := intimate.TClog.InsertRetAutoID(clog)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
streamer.Channel = &sql.NullString{String: li.Channel, Valid: true}
|
||||
streamer.LatestLogUid = cuid
|
||||
streamer.UpdateTime = &utime
|
||||
streamer.Operator = 0
|
||||
|
||||
switch {
|
||||
case li.Followers <= 1000:
|
||||
streamer.UpdateInterval = 720
|
||||
case li.Followers <= 10000:
|
||||
streamer.UpdateInterval = 360
|
||||
case li.Followers <= 100000:
|
||||
streamer.UpdateInterval = 180
|
||||
case li.Followers <= 1000000:
|
||||
streamer.UpdateInterval = 90
|
||||
default:
|
||||
streamer.UpdateInterval = 60
|
||||
}
|
||||
|
||||
// estore.Update(streamer, "update_interval", streamer.UpdateInterval, "operator", streamer.Operator, "channel", streamer.Channel, "latest_log_uid", streamer.LatestLogUid, "update_time", streamer.UpdateTime)
|
||||
err = intimate.TStreamer.Update(streamer)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
count++
|
||||
if count >= countlimit {
|
||||
count = 0
|
||||
adriver.Close()
|
||||
adriver = intimate.GetChromeDriver()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
Execute()
|
||||
}
|
||||
|
|
@ -15,5 +15,6 @@ import (
|
|||
*/
|
||||
|
||||
func main() {
|
||||
Execute()
|
||||
oe := &OpenrecExtractor{}
|
||||
oe.Execute()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,260 +3,236 @@ package main
|
|||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/extractor"
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/474420502/requests"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
//UserInfo 提取信息的结构体
|
||||
type UserInfo struct {
|
||||
UserName string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"`
|
||||
Followers int64 `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"`
|
||||
Views int64 `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"`
|
||||
var estore = intimate.NewStoreExtractor()
|
||||
var sstore = intimate.NewStoreSource(string(intimate.STOpenrec))
|
||||
|
||||
// OpenrecExtractor 提取方法
|
||||
type OpenrecExtractor struct {
|
||||
user *intimate.ExtractorSource
|
||||
userLive *intimate.ExtractorSource
|
||||
supporters *intimate.ExtractorSource
|
||||
}
|
||||
|
||||
//UserLive 提取信息的结构体
|
||||
type UserLive struct {
|
||||
Title string `exp:"//h1[contains(@class,'MovieTitle__Title')]"`
|
||||
LiveStartTime string `exp:"//meta[@itemprop='uploadDate']/@content"`
|
||||
LiveEndTime string `exp:"//meta[@itemprop='duration']/@content"`
|
||||
Tags []string `exp:"//div[contains(@class,'MovieMetaContent__TagContainer')]//a[@role ='button']"`
|
||||
}
|
||||
func (oe *OpenrecExtractor) Execute() {
|
||||
|
||||
// Execute 执行
|
||||
func Execute() {
|
||||
var loop int32 = 1
|
||||
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
ses := requests.NewSession()
|
||||
squeue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.Popenrec))
|
||||
go func() {
|
||||
signalchan := make(chan os.Signal)
|
||||
signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
|
||||
log.Println("accept stop command:", <-signalchan)
|
||||
atomic.StoreInt32(&loop, 0)
|
||||
}()
|
||||
|
||||
var lasterr error = nil
|
||||
for !ps.IsClose() {
|
||||
istreamer, err := squeue.Pop()
|
||||
for atomic.LoadInt32(&loop) > 0 {
|
||||
var err error
|
||||
|
||||
// streamer, err := estore.Pop(intimate.Popenrec) //队列里弹出一个streamer行. 进行解析
|
||||
|
||||
if istreamer == nil || err != nil {
|
||||
source, err := sstore.Pop(intimate.TOpenrecUser, 0)
|
||||
if err != nil {
|
||||
if err != lasterr {
|
||||
log.Println(err, lasterr)
|
||||
lasterr = err
|
||||
}
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
streamer := istreamer.(*intimate.Streamer)
|
||||
|
||||
userId := *streamer.UserId
|
||||
|
||||
var updateUrl map[string]string
|
||||
err = json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl) // 反序列化update_url, 里面存了需要采集的url
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
// Check Userid
|
||||
|
||||
userUrl := updateUrl["user"]
|
||||
log.Println(userUrl)
|
||||
tp := ses.Get(userUrl) // 获取user url页面数据
|
||||
resp, err := tp.Execute()
|
||||
streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
intimate.TStreamer.UpdateError(streamer, err)
|
||||
time.Sleep(time.Second * 5)
|
||||
continue
|
||||
}
|
||||
|
||||
cookies := ses.GetCookies(tp.GetParsedURL())
|
||||
sdata := source.Ext.([]byte)
|
||||
datamap := gjson.ParseBytes(sdata).Map()
|
||||
|
||||
scurl := updateUrl["supporters"] //获取打赏者的数据
|
||||
curl := gcurl.Parse(scurl)
|
||||
supportersSession := curl.CreateSession()
|
||||
source.Operator = int32(intimate.OperatorError)
|
||||
userId := datamap["var_user_id"].String()
|
||||
|
||||
temporary := curl.CreateTemporary(supportersSession)
|
||||
supportersSession.SetCookies(temporary.GetParsedURL(), cookies)
|
||||
var supporters []string
|
||||
for { // supporters 数据需要登录信息. 下面为赋值 supporters链接获取的uid token random码
|
||||
streamer := &intimate.Streamer{}
|
||||
streamer.UserId = userId
|
||||
streamer.Platform = intimate.Popenrec
|
||||
|
||||
supportersQuery := temporary.GetQuery()
|
||||
htmlUser := datamap["html_user"]
|
||||
oe.user = intimate.NewExtractorSource(&htmlUser)
|
||||
oe.user.CreateExtractor()
|
||||
|
||||
for _, cookie := range cookies {
|
||||
if cookie.Name == "uuid" {
|
||||
supportersQuery.Set("Uuid", cookie.Value)
|
||||
continue
|
||||
}
|
||||
htmlLive := datamap["html_live"]
|
||||
oe.userLive = intimate.NewExtractorSource(&htmlLive)
|
||||
oe.userLive.CreateExtractor()
|
||||
|
||||
if cookie.Name == "token" {
|
||||
supportersQuery.Set("Token", cookie.Value)
|
||||
continue
|
||||
}
|
||||
jsonSupporters := datamap["json_supporters"]
|
||||
oe.supporters = intimate.NewExtractorSource(&jsonSupporters)
|
||||
clog := &intimate.CollectLog{}
|
||||
|
||||
if cookie.Name == "random" {
|
||||
supportersQuery.Set("Random", cookie.Value)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// log.Println(anchorId)
|
||||
|
||||
supportersQuery.Set("identify_id", userId)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
oe.extractFollowers(clog)
|
||||
oe.extractUserName(streamer)
|
||||
oe.extractViewsAndLiveStreaming(clog)
|
||||
oe.extractGiversAndGratuity(clog)
|
||||
oe.extractLive(clog)
|
||||
oe.extractTags(clog)
|
||||
|
||||
resp, err := temporary.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
supporterjson := gjson.ParseBytes(resp.Content())
|
||||
supporterdata := supporterjson.Get("data") //解析supporters获取的json数据
|
||||
if supporterdata.Type == gjson.Null {
|
||||
break
|
||||
}
|
||||
supporters = append(supporters, string(resp.Content()))
|
||||
streamer.Uid = source.StreamerId.Int64
|
||||
streamer.UpdateTime = source.UpdateTime
|
||||
streamer.Tags = clog.Tags
|
||||
|
||||
temporary.QueryParam("page_number").IntAdd(1)
|
||||
}
|
||||
clog.Platform = string(intimate.Popenrec)
|
||||
clog.UserId = userId
|
||||
clog.UpdateTime = source.UpdateTime
|
||||
|
||||
// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
// ext := make(map[string]interface{})
|
||||
logUid := estore.InsertClog(clog)
|
||||
|
||||
jsonSupporters := supporters
|
||||
htmlUser := string(resp.Content())
|
||||
|
||||
liveUrl := updateUrl["live"]
|
||||
tp = ses.Get(liveUrl)
|
||||
resp, err = tp.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
intimate.TStreamer.UpdateError(streamer, err)
|
||||
continue
|
||||
}
|
||||
htmlLive := string(resp.Content())
|
||||
// ext["var_user_id"] = userId
|
||||
|
||||
// streamer.Platform = intimate.Popenrec
|
||||
streamer.UpdateInterval = 120
|
||||
streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
LiveUrl := "https://www.openrec.tv/live/" + userId
|
||||
streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true}
|
||||
streamer.LatestLogUid = logUid
|
||||
streamer.Operator = 0
|
||||
Extractor(streamer, userId, htmlUser, htmlLive, jsonSupporters)
|
||||
estore.UpdateStreamer(streamer)
|
||||
|
||||
source.Operator = int32(intimate.OperatorExtractorOK)
|
||||
sstore.UpdateOperator(source)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func Extractor(streamer *intimate.Streamer, userId string, htmlUser, htmlLive string, jsonSupporters []string) {
|
||||
|
||||
// sdata := source.Ext.([]byte)
|
||||
// datamap := gjson.ParseBytes(sdata).Map()
|
||||
|
||||
// userId := datamap["var_user_id"].String()
|
||||
|
||||
// streamer := &intimate.Streamer{}
|
||||
// streamer.UserId = &userId
|
||||
// streamer.Platform = intimate.Popenrec 不需要更新字段
|
||||
|
||||
// htmlUser := datamap["html_user"]
|
||||
|
||||
userEtor := extractor.ExtractHtmlString(htmlUser)
|
||||
ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo)
|
||||
|
||||
// htmlLive := datamap["html_live"]
|
||||
|
||||
liveEtor := extractor.ExtractHtmlString(htmlLive)
|
||||
ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive)
|
||||
|
||||
// jsonSupporters := datamap["json_supporters"]
|
||||
clog := &intimate.CollectLog{}
|
||||
|
||||
if ok1 {
|
||||
clog.Followers = &sql.NullInt64{Int64: ui.Followers, Valid: true}
|
||||
clog.Views = &sql.NullInt64{Int64: ui.Views, Valid: true}
|
||||
if ui.Views != 0 {
|
||||
clog.IsLiveStreaming = true
|
||||
}
|
||||
streamer.UserName = &sql.NullString{String: ui.UserName, Valid: true}
|
||||
|
||||
// giverjson := jsonSupporters
|
||||
var givers []interface{}
|
||||
var gratuity int64 = 0
|
||||
|
||||
for _, v := range jsonSupporters {
|
||||
giverSource := gjson.Parse(v)
|
||||
for _, item := range giverSource.Get("data.items").Array() {
|
||||
givers = append(givers, item.Map())
|
||||
gratuity += item.Get("total_yells").Int()
|
||||
}
|
||||
}
|
||||
|
||||
giversbytes, err := json.Marshal(givers)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
clog.ErrorMsg = &sql.NullString{String: err.Error(), Valid: true}
|
||||
} else {
|
||||
clog.Giver = giversbytes
|
||||
}
|
||||
|
||||
clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true}
|
||||
} else {
|
||||
log.Println("UserInfo may be not exists")
|
||||
intimate.TStreamer.UpdateError(streamer, errors.New("UserInfo may be not exists"))
|
||||
return
|
||||
}
|
||||
|
||||
//log.Println(ul)
|
||||
if ok2 {
|
||||
clog.LiveTitle = &sql.NullString{String: ul.Title, Valid: true}
|
||||
|
||||
startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
clog.LiveStartTime = &sql.NullTime{Time: startTime.Local(), Valid: true}
|
||||
duration, err := intimate.ParseDuration(ul.LiveEndTime)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
endTime := startTime.Add(duration)
|
||||
clog.LiveEndTime = &sql.NullTime{Time: endTime.Local(), Valid: true}
|
||||
}
|
||||
}
|
||||
|
||||
if tags, err := json.Marshal(ul.Tags); err == nil {
|
||||
clog.Tags = tags
|
||||
} else {
|
||||
log.Println("json error", ul.Tags, clog.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// streamer.Uid = source.StreamerId.Int64
|
||||
// streamer.UpdateTime = &source.UpdateTime
|
||||
if clog.Tags != nil {
|
||||
streamer.Tags = clog.Tags
|
||||
}
|
||||
clog.Platform = intimate.Popenrec
|
||||
clog.UserId = userId
|
||||
clog.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
clog.StreamerUid = streamer.Uid
|
||||
|
||||
logUid, err := intimate.TClog.InsertRetAutoID(clog)
|
||||
func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) {
|
||||
extractor := oe.user.GetExtractor()
|
||||
xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
if !xp.NodeIter().Next() {
|
||||
log.Println("不存在粉丝数")
|
||||
}
|
||||
|
||||
LiveUrl := "https://www.openrec.tv/live/" + userId
|
||||
|
||||
streamer.LiveUrl = &sql.NullString{String: LiveUrl, Valid: true}
|
||||
streamer.LatestLogUid = logUid
|
||||
// streamer.Operator = 0
|
||||
|
||||
// log.Println(*streamer.UserId)
|
||||
intimate.TStreamer.Update(streamer)
|
||||
|
||||
// source.Operator = int32(intimate.OperatorExtractorOK)
|
||||
// sstore.UpdateOperator(source)
|
||||
followers := strings.ReplaceAll(xp.String(), ",", "")
|
||||
followersInt, err := strconv.ParseInt(followers, 10, 64)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
clog.Set("Followers", sql.NullInt64{Int64: followersInt, Valid: true})
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractUserName(streamer intimate.ISet) {
|
||||
extractor := oe.user.GetExtractor()
|
||||
xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
if xp.NodeIter().Next() {
|
||||
userName := xp.String()
|
||||
streamer.Set("UserName", sql.NullString{String: userName, Valid: true})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractViewsAndLiveStreaming(clog intimate.ISet) {
|
||||
extractor := oe.user.GetExtractor()
|
||||
// c-contents
|
||||
xp, err := extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
if xp.NodeIter().Next() {
|
||||
views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String())
|
||||
views = strings.ReplaceAll(views, ",", "")
|
||||
viewsint, err := strconv.Atoi(views)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
clog.Set("Views", sql.NullInt64{Int64: int64(viewsint), Valid: true})
|
||||
clog.Set("IsLiveStreaming", true)
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) {
|
||||
// extractor := oe.user.GetExtractor()
|
||||
giverjson := oe.supporters.GetSource()
|
||||
var givers []interface{}
|
||||
var gratuity int64 = 0
|
||||
|
||||
for _, v := range giverjson.Array() {
|
||||
giverSource := gjson.Parse(v.String())
|
||||
for _, item := range giverSource.Get("data.items").Array() {
|
||||
givers = append(givers, item.Map())
|
||||
gratuity += item.Get("total_yells").Int()
|
||||
}
|
||||
}
|
||||
|
||||
giversbytes, err := json.Marshal(givers)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
clog.Set("ErrorMsg", sql.NullString{String: err.Error(), Valid: true})
|
||||
} else {
|
||||
clog.Set("Giver", giversbytes)
|
||||
}
|
||||
|
||||
clog.Set("Gratuity", sql.NullInt64{Int64: gratuity, Valid: true})
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) {
|
||||
extractor := oe.userLive.GetExtractor()
|
||||
mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})</h1>").FindStringSubmatch(oe.userLive.GetSource().Str)
|
||||
if len(mathes) == 2 {
|
||||
|
||||
clog.Set("LiveTitle", sql.NullString{String: mathes[1], Valid: true})
|
||||
|
||||
content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
iter := content.NodeIter()
|
||||
if iter.Next() {
|
||||
tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
// log.Println(iter.Node().NodeValue(), tm.Local())
|
||||
clog.Set("LiveStartTime", sql.NullTime{Time: tm.Local(), Valid: true})
|
||||
|
||||
duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
diter := duration.NodeIter()
|
||||
if diter.Next() {
|
||||
|
||||
dt, err := intimate.ParseDuration(diter.Node().NodeValue())
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
endtm := tm.Add(dt)
|
||||
clog.Set("LiveEndTime", sql.NullTime{Time: endtm.Local(), Valid: true})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) {
|
||||
var tags []string
|
||||
matheslist := regexp.MustCompile(`<[^>]+TagButton[^>]+>([^<]{1,100})<`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1)
|
||||
for _, m := range matheslist {
|
||||
tags = append(tags, m[1])
|
||||
}
|
||||
tagsBytes, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
clog.Set("Tags", tagsBytes)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
"github.com/lestrrat-go/libxml2"
|
||||
)
|
||||
|
||||
|
|
@ -90,6 +91,29 @@ func TestCase(t *testing.T) {
|
|||
t.Error(xr)
|
||||
}
|
||||
|
||||
func TestExtractor(t *testing.T) {
|
||||
Execute()
|
||||
func TestUserName(t *testing.T) {
|
||||
f, err := os.Open("test.html")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
data, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
extractor := hunter.NewExtractor(data)
|
||||
xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
if xp.NodeIter().Next() {
|
||||
userName := xp.String()
|
||||
t.Error(userName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractor(t *testing.T) {
|
||||
oe := &OpenrecExtractor{}
|
||||
oe.Execute()
|
||||
}
|
||||
|
|
|
|||
4
extractor/twitcasting_extractor/.gitignore
vendored
4
extractor/twitcasting_extractor/.gitignore
vendored
|
|
@ -1,4 +0,0 @@
|
|||
*.html
|
||||
log
|
||||
screenlog.*
|
||||
twitcasting_extractor
|
||||
|
|
@ -1,185 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/extractor"
|
||||
"github.com/474420502/requests"
|
||||
)
|
||||
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
|
||||
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
type LiveData struct {
|
||||
UserName string `exp:"//span[@class='tw-live-author__info-username']" method:"Text"`
|
||||
Follower int64 `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"r:ExtractNumber"`
|
||||
MaxViews int64 `exp:"//span[@id='max_viewer_count']/text()" method:"r:ExtractNumber"`
|
||||
LiveTitle string `exp:"//meta[@property='og:title']" method:"AttributeValue,content"`
|
||||
LiveStart string `exp:"//time[@data-kind='relative']" method:"AttributeValue,datetime"`
|
||||
LiveDuration string `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"`
|
||||
Tags []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag tag-info']" method:"Text"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
ses := requests.NewSession()
|
||||
streamerQueue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.Ptwitcasting))
|
||||
var lasterr error
|
||||
for !ps.IsClose() {
|
||||
|
||||
// streamer, err := estore.Pop(intimate.Ptwitcasting)
|
||||
isteamer, err := streamerQueue.Pop()
|
||||
if err != nil {
|
||||
if lasterr != err {
|
||||
lasterr = err
|
||||
log.Println(err)
|
||||
}
|
||||
time.Sleep(time.Minute)
|
||||
continue
|
||||
}
|
||||
|
||||
streamer := isteamer.(*intimate.Streamer)
|
||||
streamer.LiveUrl = &sql.NullString{String: "https://twitcasting.tv/" + *streamer.UserId, Valid: true}
|
||||
resp, err := ses.Get(streamer.LiveUrl.String).Execute()
|
||||
if err != nil {
|
||||
intimate.TStreamer.UpdateError(streamer, err)
|
||||
log.Println(err, *streamer.UserId)
|
||||
continue
|
||||
}
|
||||
var ldata *LiveData
|
||||
// f, _ := os.OpenFile("./twistcasting.html", os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm)
|
||||
// f.Write(resp.Content())
|
||||
etor := extractor.ExtractHtml(resp.Content())
|
||||
ildata := etor.GetObjectByTag(LiveData{})
|
||||
if ildata == nil {
|
||||
log.Println(streamer.LiveUrl.String)
|
||||
continue
|
||||
}
|
||||
ldata = ildata.(*LiveData)
|
||||
// ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews)
|
||||
coincount := 0
|
||||
|
||||
for i := 0; ; i++ {
|
||||
|
||||
giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i)
|
||||
resp, err = ses.Get(giverurl).Execute()
|
||||
if err != nil {
|
||||
intimate.TStreamer.UpdateError(streamer, err)
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
etor := extractor.ExtractHtml(resp.Content())
|
||||
xp, err := etor.XPaths("//td[@class='tw-memorial-table-recent-point']")
|
||||
if err != nil {
|
||||
intimate.TStreamer.UpdateError(streamer, err)
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
coins := xp.GetTexts()
|
||||
for _, cointxt := range coins {
|
||||
scointxt := strings.Split(cointxt, "/")
|
||||
if len(scointxt) == 2 {
|
||||
coin := strings.Trim(scointxt[1], " ")
|
||||
c, err := strconv.Atoi(coin)
|
||||
if err == nil {
|
||||
coincount += c
|
||||
}
|
||||
// log.Println(coin, coincount)
|
||||
} else {
|
||||
log.Println("coin error: ", cointxt)
|
||||
}
|
||||
}
|
||||
|
||||
if len(coins) < 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var tags []byte
|
||||
tags, err = json.Marshal(ldata.Tags)
|
||||
if err != nil {
|
||||
log.Println(err, streamer.UserId)
|
||||
}
|
||||
|
||||
streamer.Platform = intimate.Ptwitcasting
|
||||
streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
streamer.UserName = &sql.NullString{String: ldata.UserName, Valid: true}
|
||||
streamer.Operator = 0
|
||||
streamer.Tags = tags
|
||||
// streamer.UpdateInterval = 60
|
||||
clog := &intimate.CollectLog{}
|
||||
clog.UserId = *streamer.UserId
|
||||
clog.Gratuity = &sql.NullInt64{Int64: int64(coincount), Valid: true}
|
||||
clog.Platform = streamer.Platform
|
||||
clog.UpdateTime = streamer.UpdateTime
|
||||
clog.LiveTitle = &sql.NullString{String: ldata.LiveTitle, Valid: true}
|
||||
clog.Tags = tags
|
||||
|
||||
clog.Followers = &sql.NullInt64{Int64: int64(ldata.Follower), Valid: true}
|
||||
switch {
|
||||
case ldata.Follower <= 100:
|
||||
streamer.UpdateInterval = 720
|
||||
case ldata.Follower <= 1000:
|
||||
streamer.UpdateInterval = 320
|
||||
case ldata.Follower <= 10000:
|
||||
streamer.UpdateInterval = 240
|
||||
default:
|
||||
streamer.UpdateInterval = 120
|
||||
}
|
||||
|
||||
clog.Views = &sql.NullInt64{Int64: ldata.MaxViews, Valid: true}
|
||||
if ldata.LiveStart != "" {
|
||||
st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart)
|
||||
if err == nil {
|
||||
startTime := st
|
||||
clog.LiveStartTime = &sql.NullTime{Time: startTime, Valid: true}
|
||||
dt, err := strconv.Atoi(ldata.LiveDuration)
|
||||
|
||||
liveduration := time.Now().Sub(startTime)
|
||||
switch {
|
||||
case liveduration >= time.Hour*24*240:
|
||||
streamer.Operator = 5
|
||||
case liveduration >= time.Hour*24*60:
|
||||
streamer.UpdateInterval = 60 * 24 * 30
|
||||
case liveduration >= time.Hour*24*30:
|
||||
streamer.UpdateInterval = 60 * 24 * 15
|
||||
case liveduration >= time.Hour*24*15:
|
||||
streamer.UpdateInterval = 60 * 24 * 7
|
||||
case liveduration >= time.Hour*24*7:
|
||||
streamer.UpdateInterval = 60 * 24 * 3
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
endTime := startTime.Add((time.Duration)(dt) * time.Millisecond)
|
||||
clog.LiveEndTime = &sql.NullTime{Time: endTime, Valid: true}
|
||||
} else {
|
||||
log.Println(err, streamer.UserId)
|
||||
}
|
||||
} else {
|
||||
log.Println(err, streamer.UserId)
|
||||
}
|
||||
}
|
||||
|
||||
clog.StreamerUid = streamer.Uid
|
||||
uid, err := intimate.TClog.InsertRetAutoID(clog)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
streamer.LatestLogUid = uid
|
||||
intimate.TStreamer.Update(streamer)
|
||||
// estore.UpdateStreamer(streamer)
|
||||
log.Println(*streamer.UserId)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// type LiveData struct {
|
||||
// UserName string `exp:".//span[@class='tw-live-author__info-username']" method:"Text"`
|
||||
// Follower string `exp:".//span[@class='tw-user-nav-list-count']" method:"Text"`
|
||||
// }
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
main()
|
||||
}
|
||||
|
||||
func TestDateFormat(t *testing.T) {
|
||||
df := "Sat, 09 Sep 2017 18:19:17 +0900"
|
||||
|
||||
if _, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", df); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -3,7 +3,6 @@ package main
|
|||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"intimate"
|
||||
"log"
|
||||
"regexp"
|
||||
|
|
@ -12,317 +11,119 @@ import (
|
|||
"github.com/tebeka/selenium"
|
||||
)
|
||||
|
||||
// // sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
// var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
|
||||
|
||||
// // estore 解析存储连接实例
|
||||
// var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
func main() {
|
||||
|
||||
adriver := intimate.GetChromeDriver()
|
||||
|
||||
wd := intimate.GetChromeDriver(3030)
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
|
||||
slqueue := intimate.TStreamerList.Queue(intimate.StreamerList{}, intimate.ConditionDefault(intimate.Ptwitch))
|
||||
squeue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.Ptwitch))
|
||||
|
||||
var count = 0
|
||||
var countlimt = 200
|
||||
var recreate = time.Now()
|
||||
counter := intimate.NewCounter()
|
||||
counter.SetMaxLimit(200)
|
||||
counter.SetMaxToDo(func(olist ...interface{}) error {
|
||||
owd := olist[0].(*selenium.WebDriver)
|
||||
(*owd).Close()
|
||||
(*owd).Quit()
|
||||
*owd = intimate.GetChromeDriver(3030)
|
||||
return nil
|
||||
}, &wd)
|
||||
|
||||
var lasterr error = nil
|
||||
// var err error
|
||||
|
||||
for !ps.IsClose() {
|
||||
|
||||
wd := adriver.Webdriver
|
||||
// sourceChannel, err := sstore.Pop(intimate.TTwitchChannel)
|
||||
isl, err := slqueue.Pop()
|
||||
if err != nil {
|
||||
if lasterr != err {
|
||||
streamer, err := estore.Pop(intimate.Ptwitch, 0)
|
||||
if streamer == nil || err != nil {
|
||||
if err != lasterr {
|
||||
log.Println(err, lasterr)
|
||||
lasterr = err
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
istreamer, err := squeue.Pop()
|
||||
if err != nil {
|
||||
if lasterr != err {
|
||||
lasterr = err
|
||||
log.Println(err)
|
||||
ps.Wait(time.Minute)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
streamer := istreamer.(*intimate.Streamer)
|
||||
Extractor(wd, streamer)
|
||||
if err = intimate.TStreamer.Update(streamer); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
count++
|
||||
if count >= countlimt || time.Now().Sub(recreate) >= time.Minute*120 {
|
||||
count = 0
|
||||
adriver.Close()
|
||||
adriver = intimate.GetChromeDriver()
|
||||
recreate = time.Now()
|
||||
}
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
streamerlist := isl.(*intimate.StreamerList)
|
||||
var updateUrl map[string]string
|
||||
json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
|
||||
liveUrl := updateUrl["live"]
|
||||
log.Println(liveUrl)
|
||||
|
||||
weburl := streamerlist.Url + "?sort=VIEWER_COUNT"
|
||||
err = wd.Get(weburl)
|
||||
// err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
|
||||
err = wd.Get(liveUrl + "/about")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
// sstore.UpdateError(sourceChannel, err)
|
||||
intimate.TStreamerList.UpdateError(streamerlist, err)
|
||||
time.Sleep(time.Second * 10)
|
||||
estore.UpdateError(streamer, err)
|
||||
time.Sleep(time.Second * 5)
|
||||
continue
|
||||
}
|
||||
|
||||
wd.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) {
|
||||
_, err := wd.FindElement(selenium.ByXPATH, "(//div/p[@class=''])[last()]")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}, time.Second*10)
|
||||
streamer.LiveUrl = sql.NullString{String: liveUrl, Valid: true}
|
||||
clog := &intimate.CollectLog{}
|
||||
clog.UserId = streamer.UserId
|
||||
clog.Gratuity = sql.NullInt64{Int64: 0, Valid: false}
|
||||
|
||||
btn, err := wd.FindElement(selenium.ByXPATH, "//button[@data-a-target='browse-sort-menu']")
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
err = extractUserName(wd, streamer)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
btn.Click()
|
||||
|
||||
var elements []selenium.WebElement
|
||||
var liveurls = 0
|
||||
var delayerror = 2
|
||||
for i := 0; i < 200 && !ps.IsClose(); i++ {
|
||||
elements, err = wd.FindElements(selenium.ByXPATH, "(//div/p[@class=''])[last()]")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyDown(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyUp(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 2000)
|
||||
if len(elements) == liveurls {
|
||||
delayerror--
|
||||
if delayerror <= 0 {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
delayerror = 2
|
||||
}
|
||||
liveurls = len(elements)
|
||||
}
|
||||
articles, err := wd.FindElements(selenium.ByXPATH, "//article")
|
||||
err = extractFollowers(wd, clog)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
var streamers []*intimate.Streamer
|
||||
for _, article := range articles {
|
||||
|
||||
e, err := article.FindElement(selenium.ByXPATH, ".//a[@data-a-target='preview-card-title-link' and @href]")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
href, err := e.GetAttribute("href")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
btns, err := article.FindElements(selenium.ByXPATH, ".//div[@class='tw-full-width tw-inline-block']//button")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
var tags []string
|
||||
for _, btn := range btns {
|
||||
tag, err := btn.GetAttribute("data-a-target")
|
||||
if err == nil {
|
||||
tags = append(tags, tag)
|
||||
err = extractViews(wd, clog) // views + tags + gratuity
|
||||
if err != nil {
|
||||
// 不直播时提取礼物 gratuity
|
||||
wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
|
||||
channelchat, err := wd.FindElement(selenium.ByXPATH, `//a[@data-a-target="channel-home-tab-Chat"]`)
|
||||
btn, _ := web.FindElement(selenium.ByXPATH, `//button[@data-test-selector="expand-grabber"]`)
|
||||
if (err == nil && channelchat != nil) || btn != nil {
|
||||
if channelchat != nil {
|
||||
channelchat.Click()
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
extractGratuity(wd, clog)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
|
||||
streamer := &intimate.Streamer{}
|
||||
matches := regexp.MustCompile(`https://www.twitch.tv/(\w+)`).FindStringSubmatch(href)
|
||||
if len(matches) == 2 {
|
||||
mc := matches[1]
|
||||
streamer.UserId = &mc
|
||||
} else {
|
||||
log.Println(href)
|
||||
continue
|
||||
}
|
||||
|
||||
jtags, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
streamer.Tags = jtags
|
||||
}
|
||||
|
||||
streamer.Platform = intimate.Ptwitch
|
||||
streamer.LiveUrl = &sql.NullString{String: href, Valid: true}
|
||||
streamer.Operator = 0
|
||||
|
||||
streamers = append(streamers, streamer)
|
||||
|
||||
// if estore.InsertStreamer(streamer) {
|
||||
// // log.Println("streamer update tags", streamer.Uid, tags)
|
||||
// if streamer.Tags != nil {
|
||||
// estore.Update(streamer, "Tags", streamer.Tags)
|
||||
// }
|
||||
// }
|
||||
}, time.Second*4)
|
||||
}
|
||||
|
||||
for _, streamer := range streamers {
|
||||
Extractor(wd, streamer)
|
||||
streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
if err = intimate.TStreamer.InsertOrUpdate(streamer,
|
||||
intimate.DUpdate{Field: "tags"},
|
||||
intimate.DUpdate{Field: "update_time"},
|
||||
intimate.DUpdate{Field: "update_interval"},
|
||||
); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
streamer.Platform = intimate.Ptwitch
|
||||
clog.Platform = string(streamer.Platform)
|
||||
clog.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
|
||||
lastClogId := estore.InsertClog(clog)
|
||||
|
||||
streamer.Operator = 10
|
||||
streamer.LatestLogUid = lastClogId
|
||||
if clog.Tags != nil {
|
||||
streamer.Tags = clog.Tags
|
||||
}
|
||||
|
||||
log.Println("streamer find", len(articles))
|
||||
if len(articles) == 0 {
|
||||
intimate.TStreamerList.UpdateError(streamerlist, fmt.Errorf(""))
|
||||
}
|
||||
|
||||
count++
|
||||
if count >= countlimt || time.Now().Sub(recreate) >= time.Minute*120 {
|
||||
count = 0
|
||||
adriver.Close()
|
||||
adriver = intimate.GetChromeDriver()
|
||||
recreate = time.Now()
|
||||
switch fl := clog.Followers.Int64; {
|
||||
case fl > 100000:
|
||||
streamer.UpdateInterval = 120
|
||||
case fl > 10000:
|
||||
streamer.UpdateInterval = 240
|
||||
case fl > 1000:
|
||||
streamer.UpdateInterval = 360
|
||||
case fl > 100:
|
||||
streamer.UpdateInterval = 720
|
||||
case fl > 0:
|
||||
streamer.UpdateInterval = 1440
|
||||
}
|
||||
|
||||
streamer.UpdateTime = clog.UpdateTime
|
||||
estore.UpdateStreamer(streamer)
|
||||
counter.AddWithReset(1)
|
||||
}
|
||||
|
||||
adriver.Close()
|
||||
}
|
||||
|
||||
func Extractor(wd selenium.WebDriver, streamer *intimate.Streamer) {
|
||||
// streamer, err := estore.Pop(intimate.Ptwitch)
|
||||
// if streamer == nil || err != nil {
|
||||
// if err != lasterr {
|
||||
// log.Println(err, lasterr)
|
||||
// lasterr = err
|
||||
// }
|
||||
// time.Sleep(time.Second * 2)
|
||||
// continue
|
||||
// }
|
||||
|
||||
// var updateUrl map[string]string
|
||||
// json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
|
||||
liveUrl := "https://www.twitch.tv/" + (*streamer.UserId)
|
||||
// liveUrl = strings.Replace(liveUrl, "/watchparty", "", -1)
|
||||
log.Println(liveUrl)
|
||||
|
||||
// err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
|
||||
err := wd.Get(liveUrl + "/about")
|
||||
if err != nil {
|
||||
errstr := fmt.Errorf("%s: %s", err.Error(), liveUrl+"/about")
|
||||
log.Println(errstr)
|
||||
intimate.TStreamer.UpdateError(streamer, errstr)
|
||||
time.Sleep(time.Second * 5)
|
||||
return
|
||||
}
|
||||
|
||||
streamer.LiveUrl = &sql.NullString{String: liveUrl, Valid: true}
|
||||
clog := &intimate.CollectLog{}
|
||||
clog.UserId = *streamer.UserId
|
||||
clog.Gratuity = &sql.NullInt64{Int64: 0, Valid: false}
|
||||
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
err = extractUserName(wd, streamer)
|
||||
if err != nil {
|
||||
_, err = wd.FindElement(selenium.ByXPATH, "//a[@data-a-target='browse-channels-button']")
|
||||
if err == nil {
|
||||
log.Println(*streamer.UserId, "may be cancell")
|
||||
streamer.Operator = 5
|
||||
intimate.TStreamer.UpdateError(streamer, fmt.Errorf(*streamer.UserId, "may be cancell"))
|
||||
}
|
||||
return
|
||||
}
|
||||
err = extractFollowers(wd, clog)
|
||||
if err != nil {
|
||||
// log.Println(err)
|
||||
streamer.UpdateInterval += 30
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
err = extractViews(wd, clog) // views + tags + gratuity
|
||||
if err != nil {
|
||||
// 不直播时提取礼物 gratuity
|
||||
wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
|
||||
channelchat, err := wd.FindElement(selenium.ByXPATH, `//a[@data-a-target="channel-home-tab-Chat"]`)
|
||||
btn, _ := web.FindElement(selenium.ByXPATH, `//button[@data-test-selector="expand-grabber"]`)
|
||||
if (err == nil && channelchat != nil) || btn != nil {
|
||||
if channelchat != nil {
|
||||
channelchat.Click()
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
extractGratuity(wd, clog)
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
|
||||
}, time.Second*4)
|
||||
}
|
||||
|
||||
streamer.Platform = intimate.Ptwitch
|
||||
clog.Platform = streamer.Platform
|
||||
clog.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
// clog.StreamerUid = streamer.Uid
|
||||
lastClogId, err := intimate.TClog.InsertRetAutoID(clog)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
streamer.LatestLogUid = lastClogId
|
||||
if clog.Tags != nil {
|
||||
streamer.Tags = clog.Tags
|
||||
}
|
||||
|
||||
switch fl := clog.Followers.Int64; {
|
||||
case fl > 100000:
|
||||
streamer.UpdateInterval = 120
|
||||
case fl > 10000:
|
||||
streamer.UpdateInterval = 240 * 2
|
||||
case fl > 1000:
|
||||
streamer.UpdateInterval = 360 * 2
|
||||
case fl > 100:
|
||||
streamer.UpdateInterval = 720 * 2
|
||||
case fl > 0:
|
||||
streamer.UpdateInterval = 1440 * 4
|
||||
}
|
||||
|
||||
streamer.UpdateTime = clog.UpdateTime
|
||||
// intimate.TStreamer.InsertOrUpdate(streamer)
|
||||
// count++
|
||||
// if count >= countlimt {
|
||||
// count = 0
|
||||
// // wd.Quit()
|
||||
// wd = intimate.GetChromeDriver(3030)
|
||||
// }
|
||||
wd.Close()
|
||||
wd.Quit()
|
||||
}
|
||||
|
||||
func extractUserName(wd selenium.WebDriver, streamer *intimate.Streamer) error {
|
||||
|
|
@ -331,7 +132,7 @@ func extractUserName(wd selenium.WebDriver, streamer *intimate.Streamer) error {
|
|||
if err == nil {
|
||||
if ltxt, err := label.Text(); err == nil && ltxt != "" {
|
||||
// log.Println("label:", ltxt)
|
||||
streamer.UserName = &sql.NullString{String: ltxt, Valid: true}
|
||||
streamer.UserName = sql.NullString{String: ltxt, Valid: true}
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
|
@ -351,7 +152,7 @@ func extractFollowers(wd selenium.WebDriver, clog *intimate.CollectLog) error {
|
|||
}
|
||||
followers = regexp.MustCompile(`[\d,]+`).FindString(followers)
|
||||
fint, _ := intimate.ParseNumber(followers)
|
||||
clog.Followers = &sql.NullInt64{Int64: int64(fint), Valid: true}
|
||||
clog.Followers = sql.NullInt64{Int64: int64(fint), Valid: true}
|
||||
// log.Println("followers: ", followers, fint)
|
||||
return true, nil
|
||||
}, 4*time.Second)
|
||||
|
|
@ -364,7 +165,7 @@ func extractViews(wd selenium.WebDriver, clog *intimate.CollectLog) error {
|
|||
if txt, err := views.Text(); err == nil {
|
||||
|
||||
vint, _ := intimate.ParseNumber(txt)
|
||||
clog.Views = &sql.NullInt64{Int64: vint, Valid: true}
|
||||
clog.Views = sql.NullInt64{Int64: vint, Valid: true}
|
||||
// log.Println("views:", txt)
|
||||
views.Click()
|
||||
|
||||
|
|
@ -384,7 +185,7 @@ func extractTitle(wd selenium.WebDriver, clog *intimate.CollectLog) error {
|
|||
title, err := web.FindElement(selenium.ByXPATH, `//h2[@data-a-target='stream-title']`)
|
||||
if err == nil {
|
||||
if txt, err := title.Text(); err == nil {
|
||||
clog.LiveTitle = &sql.NullString{String: txt, Valid: true}
|
||||
clog.LiveTitle = sql.NullString{String: txt, Valid: true}
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
|
@ -436,7 +237,7 @@ func extractGratuity(wd selenium.WebDriver, clog *intimate.CollectLog) error {
|
|||
log.Println(err)
|
||||
}
|
||||
}
|
||||
clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true}
|
||||
clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,25 +3,29 @@ package intimate
|
|||
import (
|
||||
"database/sql"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
type GetSet struct {
|
||||
}
|
||||
|
||||
type StreamerList struct {
|
||||
UrlHash string `field:"urlhash" uid:"true"` //
|
||||
Platform string `field:"platform" ` //
|
||||
Url string `field:"url" ` //
|
||||
UrlHash []byte //
|
||||
Platform Platform //
|
||||
Url string //
|
||||
|
||||
Label *sql.NullString `field:"label" ` //
|
||||
Label sql.NullString //
|
||||
|
||||
Serialize interface{} `field:"serialize" `
|
||||
Serialize interface{}
|
||||
|
||||
UpdateInterval int32 `field:"update_interval" `
|
||||
UpdateTime *sql.NullTime `field:"update_time" ` //
|
||||
UpdateInterval int32
|
||||
UpdateTime time.Time //
|
||||
|
||||
ErrorMsg *sql.NullString `field:"error_msg" ` //
|
||||
Operator int32 `field:"operator" `
|
||||
ErrorMsg sql.NullString
|
||||
Operator int32
|
||||
|
||||
LastOperator int32
|
||||
}
|
||||
|
|
@ -37,26 +41,25 @@ func (sl *StreamerList) Set(field string, value interface{}) {
|
|||
}
|
||||
|
||||
type Streamer struct {
|
||||
Uid int64 `field:"uid" uid:"auto"` //
|
||||
Platform Platform `field:"platform"` //
|
||||
UserId *string `field:"user_id"` //
|
||||
Uid int64 //
|
||||
Platform Platform //
|
||||
UserId string //
|
||||
|
||||
UserName *sql.NullString `field:"user_name"` //
|
||||
LiveUrl *sql.NullString `field:"live_url"` //
|
||||
Channel *sql.NullString `field:"channel"` //
|
||||
Tags interface{} `field:"tags"`
|
||||
Ext interface{} `field:"ext"` //
|
||||
// Comments interface{} `field:"comments"`
|
||||
UserName sql.NullString //
|
||||
LiveUrl sql.NullString //
|
||||
Channel sql.NullString //
|
||||
Tags interface{}
|
||||
Ext interface{} //
|
||||
|
||||
IsUpdateStreamer bool // 更新上面的内容
|
||||
IsUpdateUrl bool
|
||||
UpdateInterval int32 `field:"update_interval"`
|
||||
UpdateUrl interface{} `field:"update_url"` // TODO: nil
|
||||
LatestLogUid int64 `field:"latest_log_uid"`
|
||||
UpdateTime *sql.NullTime `field:"update_time"` //
|
||||
UpdateInterval int32
|
||||
UpdateUrl interface{}
|
||||
LatestLogUid int64
|
||||
UpdateTime sql.NullTime //
|
||||
|
||||
ErrorMsg *sql.NullString `field:"error_msg"`
|
||||
Operator int32 `field:"operator"`
|
||||
ErrorMsg sql.NullString
|
||||
Operator int32
|
||||
|
||||
LastOperator int32
|
||||
}
|
||||
|
|
@ -72,25 +75,24 @@ func (ai *Streamer) Set(field string, value interface{}) {
|
|||
}
|
||||
|
||||
type CollectLog struct {
|
||||
LogUid int64 `field:"log_uid"` // 日志id
|
||||
StreamerUid int64 `field:"streamer_uid"` // StreamerId 表id与
|
||||
LogUid int64 // 日志id
|
||||
StreamerUid int64 // StreamerId 表id与
|
||||
|
||||
Platform Platform `field:"platform"` //
|
||||
UserId string `field:"user_id"` // 平台的UserId
|
||||
IsLiveStreaming bool `field:"is_live_streaming"` //
|
||||
IsError bool `field:"is_error"` //
|
||||
Followers *sql.NullInt64 `field:"followers"` //
|
||||
Views *sql.NullInt64 `field:"views"` //
|
||||
Giver interface{} `field:"giver"` //
|
||||
Gratuity *sql.NullInt64 `field:"gratuity"` //
|
||||
LiveTitle *sql.NullString `field:"live_title"` //
|
||||
LiveStartTime *sql.NullTime `field:"live_start_time"` //
|
||||
LiveEndTime *sql.NullTime `field:"live_end_time"` //
|
||||
UpdateTime *sql.NullTime `field:"update_time"` //
|
||||
Tags interface{} `field:"tags"`
|
||||
Ext interface{} `field:"ext"` //
|
||||
ErrorMsg *sql.NullString `field:"error_msg"` //
|
||||
Comments interface{} `field:"comments"` //
|
||||
Platform string //
|
||||
UserId string // 平台的UserId
|
||||
IsLiveStreaming bool //
|
||||
IsError bool //
|
||||
Followers sql.NullInt64 //
|
||||
Views sql.NullInt64 //
|
||||
Giver interface{} //
|
||||
Gratuity sql.NullInt64 //
|
||||
LiveTitle sql.NullString //
|
||||
LiveStartTime sql.NullTime //
|
||||
LiveEndTime sql.NullTime //
|
||||
UpdateTime sql.NullTime //
|
||||
Tags interface{}
|
||||
Ext interface{} //
|
||||
ErrorMsg sql.NullString //
|
||||
}
|
||||
|
||||
// Get Simple Value
|
||||
|
|
@ -102,3 +104,36 @@ func (cl *CollectLog) Get(field string) interface{} {
|
|||
func (cl *CollectLog) Set(field string, value interface{}) {
|
||||
reflect.ValueOf(cl).Elem().FieldByName(field).Set(reflect.ValueOf(value))
|
||||
}
|
||||
|
||||
type ExtractorSource struct {
|
||||
source *gjson.Result
|
||||
extractor *hunter.Extractor
|
||||
}
|
||||
|
||||
func NewExtractorSource(gr *gjson.Result) *ExtractorSource {
|
||||
es := &ExtractorSource{}
|
||||
es.SetSource(gr)
|
||||
return es
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) SetSource(gr *gjson.Result) {
|
||||
es.source = gr
|
||||
es.extractor = nil
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) Clear() {
|
||||
es.source = nil
|
||||
es.extractor = nil
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) CreateExtractor() {
|
||||
es.extractor = hunter.NewExtractor([]byte(es.source.String()))
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) GetSource() *gjson.Result {
|
||||
return es.source
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) GetExtractor() *hunter.Extractor {
|
||||
return es.extractor
|
||||
}
|
||||
|
|
|
|||
12
go.mod
12
go.mod
|
|
@ -1,18 +1,18 @@
|
|||
module intimate
|
||||
|
||||
go 1.15
|
||||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/474420502/extractor v0.9.6
|
||||
github.com/474420502/extractor v0.2.2
|
||||
github.com/474420502/focus v0.12.0
|
||||
github.com/474420502/gcurl v0.4.5
|
||||
github.com/474420502/requests v1.9.1
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/474420502/gcurl v0.1.2
|
||||
github.com/474420502/hunter v0.3.4
|
||||
github.com/474420502/requests v1.6.0
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb
|
||||
github.com/tebeka/selenium v0.9.9
|
||||
github.com/tidwall/gjson v1.6.0
|
||||
github.com/tidwall/pretty v1.0.1 // indirect
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
)
|
||||
|
|
|
|||
33
go.sum
33
go.sum
|
|
@ -2,20 +2,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
|||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
|
||||
github.com/474420502/extractor v0.9.6 h1:mwwzwVeu/sZ4FV190Bl32ta4iVjERBWL6eGLXRDlAtg=
|
||||
github.com/474420502/extractor v0.9.6/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY=
|
||||
github.com/474420502/extractor v0.2.2 h1:hGao2iZt5CEI8oqYjQW938osQdHKgNWL/bwRJQNgHTM=
|
||||
github.com/474420502/extractor v0.2.2/go.mod h1:OVFijdKLDghigpIYISHzlognL5q8eeVenT2fRhCyFns=
|
||||
github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo=
|
||||
github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
|
||||
github.com/474420502/gcurl v0.4.4 h1:ZILu7RRjDBGHpTGmuWGKf1NZZbZsC7AHPlI8RHqs9As=
|
||||
github.com/474420502/gcurl v0.4.4/go.mod h1:7w4knyVJa1ia4I1xd0krG51fKLGwMmNn5sfG2zPWbqM=
|
||||
github.com/474420502/gcurl v0.4.5 h1:4y+NbbBGRIWDpfe/iojdSUzlShcZmnkHV4T4etiWQsw=
|
||||
github.com/474420502/gcurl v0.4.5/go.mod h1:7w4knyVJa1ia4I1xd0krG51fKLGwMmNn5sfG2zPWbqM=
|
||||
github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc=
|
||||
github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU=
|
||||
github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw=
|
||||
github.com/474420502/requests v1.7.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/474420502/requests v1.9.1 h1:gCDmBCW1ym8zOvKKBGjEG9wafMY7imYk2u28+Oy1WEc=
|
||||
github.com/474420502/requests v1.9.1/go.mod h1:x2T9l+e40R6kxxMvNm+YSZ9D6BHAXUDak4kQElIPJ9A=
|
||||
github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg=
|
||||
github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM=
|
||||
github.com/474420502/hunter v0.3.4 h1:fyLAgI84jWe3IcqsISC53j1w3CXI1FERxX//Potns0M=
|
||||
github.com/474420502/hunter v0.3.4/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA=
|
||||
github.com/474420502/libxml2 v0.0.0-20200803084225-29e441d26406 h1:nLvl2D2y+hxCglLnRmLqwRGwmUsXQt8ga46zGySTU1I=
|
||||
github.com/474420502/libxml2 v0.0.0-20200803084225-29e441d26406/go.mod h1:bUbcte7hFuLijGG6/+gGxurW3XvxE/CBdfAAlsIWj34=
|
||||
github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY=
|
||||
github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
|
||||
|
|
@ -24,8 +22,8 @@ github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnRO
|
|||
github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k=
|
||||
github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP3VrhEcx8=
|
||||
github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
|
||||
github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
|
||||
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
|
||||
github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
|
|
@ -42,8 +40,6 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
|
|||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
|
|
@ -113,9 +109,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
|
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
|
|||
|
|
@ -12,10 +12,4 @@ const (
|
|||
|
||||
// Ptwitcasting twitcasting 平台
|
||||
Ptwitcasting Platform = "twitcasting"
|
||||
|
||||
// PNimo PNimo 平台
|
||||
PNimo Platform = "nimo"
|
||||
|
||||
// PMirrativ PNimo 平台
|
||||
PMirrativ Platform = "mirrativ"
|
||||
)
|
||||
|
|
|
|||
80
store.go
80
store.go
|
|
@ -228,50 +228,6 @@ func NewStoreExtractor() *StoreExtractor {
|
|||
return &StoreExtractor{db: db}
|
||||
}
|
||||
|
||||
// PopNoWait 弹出一个不用按时间间隔更新的主播信息, 主要用来测试.
|
||||
func (store *StoreExtractor) PopNoWait(platform Platform, condition string, operators ...int32) (*Streamer, error) {
|
||||
tx, err := store.db.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var args = []interface{}{string(platform)}
|
||||
selectSQL := `select uid, update_time, user_id, tags, live_url, update_url, is_update_streamer, update_interval from ` + StreamerTable + ` where platform = ? and ` + condition
|
||||
if len(operators) == 0 {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, 0)
|
||||
} else {
|
||||
for _, operator := range operators {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, operator)
|
||||
}
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := tx.Commit()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
store.popCount++
|
||||
}()
|
||||
|
||||
// log.Println(selectSQL + ` limit 1 for update`)
|
||||
row := tx.QueryRow(selectSQL+` limit 1 for update`, args...)
|
||||
|
||||
s := &Streamer{}
|
||||
// uid, url, target_type, source, ext, operator
|
||||
err = row.Scan(&s.Uid, &s.UpdateTime, &s.UserId, &s.Tags, &s.LiveUrl, &s.UpdateUrl, &s.IsUpdateStreamer, &s.UpdateInterval)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Set("LastOperator", s.Operator)
|
||||
_, err = tx.Exec("update "+StreamerTable+" set operator = ? where uid = ?", OperatorWait, s.Uid)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Pop 弹出一条未处理的数据
|
||||
func (store *StoreExtractor) Pop(platform Platform, operators ...int32) (*Streamer, error) {
|
||||
|
||||
|
|
@ -280,7 +236,7 @@ func (store *StoreExtractor) Pop(platform Platform, operators ...int32) (*Stream
|
|||
return nil, err
|
||||
}
|
||||
var args = []interface{}{string(platform)}
|
||||
selectSQL := `select uid, update_time, user_id, tags, live_url, update_url, is_update_streamer, update_interval from ` + StreamerTable + ` where platform = ? and TIMESTAMPDIFF(MINUTE , update_time, CURRENT_TIMESTAMP()) >= update_interval`
|
||||
selectSQL := `select uid, update_time, user_id, update_url, is_update_streamer, update_interval from ` + StreamerTable + ` where platform = ? and TIMESTAMPDIFF(MINUTE , update_time, CURRENT_TIMESTAMP()) >= update_interval`
|
||||
if len(operators) == 0 {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, 0)
|
||||
|
|
@ -308,7 +264,7 @@ func (store *StoreExtractor) Pop(platform Platform, operators ...int32) (*Stream
|
|||
|
||||
s := &Streamer{}
|
||||
// uid, url, target_type, source, ext, operator
|
||||
err = row.Scan(&s.Uid, &s.UpdateTime, &s.UserId, &s.Tags, &s.LiveUrl, &s.UpdateUrl, &s.IsUpdateStreamer, &s.UpdateInterval)
|
||||
err = row.Scan(&s.Uid, &s.UpdateTime, &s.UserId, &s.UpdateUrl, &s.IsUpdateStreamer, &s.UpdateInterval)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -360,9 +316,9 @@ func (store *StoreExtractor) InsertStreamerList(streamerlist IGet) (isExists boo
|
|||
}
|
||||
|
||||
// InsertStreamer Streamer表, 插入数据
|
||||
func (store *StoreExtractor) InsertStreamer(streamer *Streamer) (isExists bool) {
|
||||
func (store *StoreExtractor) InsertStreamer(streamer IGet) (isExists bool) {
|
||||
// select uid from table where platform = ? and user_id = ?
|
||||
// selectSQL := "SELECT is_update_url, uid FROM " + StreamerTable + " WHERE platform = ? AND user_id = ?"
|
||||
selectSQL := "SELECT is_update_url, uid FROM " + StreamerTable + " WHERE platform = ? AND user_id = ?"
|
||||
tx, err := store.db.Begin()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -379,17 +335,18 @@ func (store *StoreExtractor) InsertStreamer(streamer *Streamer) (isExists bool)
|
|||
}
|
||||
}()
|
||||
|
||||
streamer.UpdateTime = &sql.NullTime{Time: time.Now().Add(-time.Hour * 100000), Valid: true}
|
||||
_, err = tx.Exec("INSERT IGNORE INTO "+StreamerTable+"(platform, user_id, user_name, live_url, update_url, tags, update_time) VALUES(?,?,?,?,?,?,?);",
|
||||
streamer.Platform,
|
||||
streamer.UserId,
|
||||
streamer.UserName,
|
||||
streamer.LiveUrl,
|
||||
streamer.UpdateUrl,
|
||||
streamer.Tags,
|
||||
streamer.UpdateTime,
|
||||
)
|
||||
row := tx.QueryRow(selectSQL+` LIMIT 1 FOR UPDATE`, streamer.Get("Platform"), streamer.Get("UserId"))
|
||||
var isUpdateUrl bool
|
||||
var Uid int64
|
||||
if err = row.Scan(&isUpdateUrl, &Uid); err == nil {
|
||||
if isUpdateUrl {
|
||||
tx.Exec("UPDATE "+StreamerTable+" SET update_url = ?", streamer.Get("UpdateUrl"))
|
||||
}
|
||||
streamer.(ISet).Set("Uid", Uid)
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = tx.Exec("INSERT INTO "+StreamerTable+"(platform, user_id, update_url, update_time) VALUES(?,?,?,?);", streamer.Get("Platform"), streamer.Get("UserId"), streamer.Get("UpdateUrl"), time.Now().Add(-time.Minute*60))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
@ -425,8 +382,7 @@ func (store *StoreExtractor) UpdateOperator(isource IGet) {
|
|||
|
||||
// UpdateStreamer Streamer表, 插入数据
|
||||
func (store *StoreExtractor) UpdateStreamer(streamer IGet) {
|
||||
// log.Printf("UPDATE "+StreamerTable+" SET user_name = %v, live_url = %v, channel = %v, latest_log_uid = %v, tags = %v, ext = %v, operator = %v, update_time = %v, update_interval = %v WHERE uid = %v", streamer.Get("UserName"), streamer.Get("LiveUrl"), streamer.Get("Channel"), streamer.Get("LatestLogUid"), streamer.Get("Tags"), streamer.Get("Ext"), streamer.Get("Operator"), streamer.Get("UpdateTime"), streamer.Get("UpdateInterval"), streamer.Get("Uid"))
|
||||
_, err := store.db.Exec("UPDATE "+StreamerTable+" SET user_name = ?, live_url = ?, channel = ?, latest_log_uid = ?, tags = ?, ext = ?, operator = ?, update_time = ?, update_interval = ? WHERE uid = ?",
|
||||
_, err := store.db.Exec("UPDATE "+StreamerTable+" SET user_name = ?, live_url = ?, channel = ?, latest_log_uid = ?, tags = ?, ext = ?, operator = ?, update_time = ?, update_interval = ? WHERE uid = ?;",
|
||||
streamer.Get("UserName"), streamer.Get("LiveUrl"), streamer.Get("Channel"), streamer.Get("LatestLogUid"), streamer.Get("Tags"), streamer.Get("Ext"), streamer.Get("Operator"), streamer.Get("UpdateTime"), streamer.Get("UpdateInterval"), streamer.Get("Uid"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -440,14 +396,12 @@ func (store *StoreExtractor) Update(streamer IGet, fieldvalues ...interface{}) {
|
|||
for i := 0; i < len(fieldvalues); i += 2 {
|
||||
field := fieldvalues[i]
|
||||
values = append(values, fieldvalues[i+1])
|
||||
updateSQL += field.(string) + " = ?,"
|
||||
updateSQL += field.(string) + " = ? "
|
||||
}
|
||||
updateSQL = updateSQL[0 : len(updateSQL)-1]
|
||||
updateSQL += "WHERE uid = ?"
|
||||
values = append(values, streamer.Get("Uid"))
|
||||
_, err := store.db.Exec(updateSQL, values...)
|
||||
if err != nil {
|
||||
log.Println(updateSQL)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ import (
|
|||
)
|
||||
|
||||
func TestStoreInsert(t *testing.T) {
|
||||
|
||||
// ht := hunter.NewHunter(openrecRanking)
|
||||
// ht.Execute()
|
||||
}
|
||||
|
||||
func TestStoreInsertCase1(t *testing.T) {
|
||||
|
|
|
|||
1
supervisor_conf/.gitignore
vendored
1
supervisor_conf/.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
conf.d
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
CURPATH=`pwd`
|
||||
BINPATH=$(dirname "$CURPATH")/bin
|
||||
find $BINPATH -type f -name 'log' -exec truncate -s 0 {} +
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
|
||||
CURPATH=`pwd`
|
||||
SUPPATH=$(dirname "$CURPATH")
|
||||
|
||||
mkdir -p $CURPATH/conf.d
|
||||
cp *.conf $CURPATH/conf.d/
|
||||
sed -i "s#MYPATH#$SUPPATH#g" $CURPATH/conf.d/*.conf
|
||||
ln -sf $CURPATH/conf.d/*.conf /etc/supervisor/conf.d/
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:mirrativ_extractor]
|
||||
directory = MYPATH/bin/mirrativ_extractor/
|
||||
command= MYPATH/bin/mirrativ_extractor/mirrativ_extractor
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/mirrativ_extractor/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=false
|
||||
|
||||
[program:mirrativ_task1]
|
||||
environment=DISPLAY=":99"
|
||||
directory = MYPATH/bin/mirrativ_task1/
|
||||
command= MYPATH/bin/mirrativ_task1/mirrativ_task1
|
||||
# process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
# numprocs=1 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/mirrativ_task1/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:nimo_extractor]
|
||||
directory = MYPATH/bin/nimo_extractor/
|
||||
command= MYPATH/bin/nimo_extractor/nimo_extractor
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/nimo_extractor/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=false
|
||||
|
||||
[program:nimo_task1]
|
||||
environment=DISPLAY=":99"
|
||||
directory = MYPATH/bin/nimo_task1/
|
||||
command= MYPATH/bin/nimo_task1/nimo_task1
|
||||
# process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
# numprocs=1 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/nimo_task1/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:openrec_extractor]
|
||||
directory = MYPATH/bin/openrec_extractor/
|
||||
command= MYPATH/bin/openrec_extractor/openrec_extractor
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/openrec_extractor/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:openrec_ranking]
|
||||
directory = MYPATH/bin/openrec_task1
|
||||
command= MYPATH/bin/openrec_task1/openrec_task1
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/openrec_task1/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=false
|
||||
|
||||
[program:twitcasting_extractor]
|
||||
environment=DISPLAY=":99"
|
||||
directory = MYPATH/bin/twitcasting_extractor/
|
||||
command= MYPATH/bin/twitcasting_extractor/twitcasting_extractor
|
||||
process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
numprocs=3 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/twitcasting_extractor/log
|
||||
# stderr_logfile=%(supervisorctl.var.directory)s/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=false
|
||||
|
||||
[program:twitcasting_task1]
|
||||
environment=DISPLAY=":99"
|
||||
directory = MYPATH/bin/twitcasting_task1/
|
||||
command= MYPATH/bin/twitcasting_task1/twitcasting_task1
|
||||
# process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
# numprocs=1 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/twitcasting_task1/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:twitch_extractor]
|
||||
environment=DISPLAY=":99"
|
||||
directory = MYPATH/bin/twitch_extractor
|
||||
command= MYPATH/bin/twitch_extractor/twitch_extractor
|
||||
process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
numprocs=5 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/twitch_extractor/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:twitch_extractor_p1]
|
||||
environment=DISPLAY=":99",pac_proxy=http://localhost:1090/pac
|
||||
directory = MYPATH/bin/twitch_extractor
|
||||
command= MYPATH/bin/twitch_extractor/twitch_extractor
|
||||
process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
numprocs=2 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/twitch_extractor/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon=false
|
||||
|
||||
[program:twitch_task1]
|
||||
environment=DISPLAY=":99"
|
||||
directory = MYPATH/bin/twitch_task1
|
||||
command= MYPATH/bin/twitch_task1/twitch_task1
|
||||
# process_name=%(program_name)s_%(process_num)02d ;多进程名称
|
||||
# numprocs=1 ;启动多个进程
|
||||
autorestart=true
|
||||
stderr_logfile=MYPATH/bin/twitch_task1/log
|
||||
stderr_logfile_maxbytes=0
|
||||
stopsignal=QUIT
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
|
||||
[supervisord]
|
||||
nodaemon=true
|
||||
|
||||
[program:xvfb-99]
|
||||
command=/usr/bin/Xvfb :99 -screen 0 1280x720x24 -ac -nolisten tcp -dpi 96 +extension RANDR -nolisten tcp
|
||||
autorestart=true
|
||||
|
||||
|
|
@ -12,7 +12,4 @@ const (
|
|||
|
||||
// STTwitcasting STTwitcasting源table名称
|
||||
STTwitcasting SourceTable = "source_twitcasting"
|
||||
|
||||
// STNimo nimo源table名称
|
||||
STNimo SourceTable = "source_nimo"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"intimate"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
func main() {
|
||||
bcurl := `curl 'https://www.mirrativ.com/api/live/catalog?id=2&cursor=%s' \
|
||||
-H 'authority: www.mirrativ.com' \
|
||||
-H 'accept: application/json' \
|
||||
-H 'x-timezone: Asia/Shanghai' \
|
||||
-H 'x-csrf-token: F3Ojd6RBtApP6YAZzVn-9jWN1of159VxAqOQL1Zn' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
|
||||
-H 'content-type: application/json' \
|
||||
-H 'sec-fetch-site: same-origin' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.mirrativ.com/' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9,ja;q=0.8' \
|
||||
-H 'cookie: f=A2D75F0E-D218-11EA-A042-452BF6D21CE8; _ga=GA1.2.689947597.1596081392; mr_id=kxb65LddGMZf5C28jkR_tGCZD_ZFOAepD5gfXO7eNjfPMB8EKYvU1Vg_Y29V1lsa; _gid=GA1.2.2116692650.1600139685; lang=ja' \
|
||||
--compressed`
|
||||
|
||||
curl := gcurl.Parse(bcurl)
|
||||
tp := curl.CreateTemporary(nil)
|
||||
cursor := tp.QueryParam(`cursor`)
|
||||
cursor.StringSet("")
|
||||
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
for !ps.IsClose() {
|
||||
|
||||
log.Println(tp.ParsedURL.String())
|
||||
resp, err := tp.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
apijson := gjson.ParseBytes(resp.Content())
|
||||
next := apijson.Get("next_cursor").String()
|
||||
|
||||
for _, liveinfo := range apijson.Get("list").Array() {
|
||||
var prekey string
|
||||
if liveinfo.Get("live_preview").Exists() {
|
||||
prekey = "live_preview"
|
||||
} else {
|
||||
prekey = "live"
|
||||
}
|
||||
owner := liveinfo.Get(prekey + ".owner")
|
||||
if guserid := owner.Get("user_id"); guserid.String() != "" {
|
||||
streamer := &intimate.Streamer{}
|
||||
streamer.Platform = intimate.PMirrativ
|
||||
streamer.Operator = 0
|
||||
streamer.UserId = &guserid.Str
|
||||
streamer.UserName = &sql.NullString{String: owner.Get("name").String(), Valid: true}
|
||||
streamer.UpdateInterval = 600
|
||||
streamer.UpdateTime = intimate.GetUpdateTimeNow()
|
||||
|
||||
err = intimate.TStreamer.InsertOrUpdate(
|
||||
streamer,
|
||||
intimate.DUpdate{Field: "update_time"},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if next == "" {
|
||||
ps.Wait(time.Minute * 10)
|
||||
} else {
|
||||
ps.Wait(time.Second * 2)
|
||||
}
|
||||
cursor.StringSet(next)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
package main
|
||||
|
||||
func main() {
|
||||
Execute()
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// estore 解析存储连接实例
|
||||
// var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
// Execute 执行
|
||||
func Execute() {
|
||||
|
||||
nimoapi := `curl 'https://api.nimo.tv/oversea/nimo/api/v2/liveRoom/liveRoomPage-1-100-/HK/1028/1000' \
|
||||
-H 'authority: api.nimo.tv' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
|
||||
-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary3bCA1lzvhj4kBR4Q' \
|
||||
-H 'accept: */*' \
|
||||
-H 'origin: https://www.nimo.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.nimo.tv/lives' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
--data-binary $'------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="keyType"\r\n\r\n0\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="body"\r\n\r\n{"deviceType":7,"requestSource":"WEB","iNetType":5}\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q--\r\n' \
|
||||
--compressed`
|
||||
|
||||
curl := gcurl.Parse(nimoapi)
|
||||
ses := curl.CreateSession()
|
||||
tp := curl.CreateTemporary(ses)
|
||||
|
||||
param := tp.PathParam("liveRoomPage-(1)-")
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
for !ps.IsClose() {
|
||||
|
||||
resp, err := tp.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
result := gjson.ParseBytes(resp.Content())
|
||||
roomlist := result.Get("data.result.liveRoomViewList")
|
||||
|
||||
if !roomlist.IsArray() {
|
||||
log.Println("json is error")
|
||||
log.Println(string(resp.Content()))
|
||||
break
|
||||
}
|
||||
|
||||
rooms := roomlist.Array()
|
||||
log.Println(tp.GetURLRawPath(), "rooms:", len(rooms))
|
||||
if len(rooms) == 0 {
|
||||
param.IntSet(1)
|
||||
time.Sleep(time.Minute * 15)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, room := range rooms {
|
||||
|
||||
streamer := &intimate.Streamer{}
|
||||
streamer.Platform = intimate.PNimo
|
||||
|
||||
if userid := room.Get("id").String(); userid != "" {
|
||||
|
||||
streamer.UserId = &userid
|
||||
streamer.LiveUrl = &sql.NullString{String: "https://www.nimo.tv/live/" + userid, Valid: true}
|
||||
|
||||
channel := room.Get("roomTypeName").String()
|
||||
streamer.Channel = &sql.NullString{String: channel, Valid: channel != ""}
|
||||
username := room.Get("anchorName").String()
|
||||
streamer.UserName = &sql.NullString{String: username, Valid: username != ""}
|
||||
|
||||
if rtags := room.Get("anchorLabels"); rtags.IsArray() {
|
||||
|
||||
var tags []string
|
||||
for _, r := range rtags.Array() {
|
||||
tag := r.Get("labelName").String()
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
|
||||
data, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
streamer.Tags = data
|
||||
|
||||
}
|
||||
|
||||
streamer.UpdateInterval = 120
|
||||
err = intimate.TStreamer.Insert(streamer)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
} else {
|
||||
log.Println("userid is null.", room.String())
|
||||
}
|
||||
}
|
||||
param.IntAdd(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
)
|
||||
|
||||
func CrawlContent(args ...interface{}) []byte {
|
||||
nimoapi := `curl 'https://api.nimo.tv/oversea/nimo/api/v2/liveRoom/liveRoomPage-%d-30-/HK/1028/1000' \
|
||||
-H 'authority: api.nimo.tv' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
|
||||
-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary3bCA1lzvhj4kBR4Q' \
|
||||
-H 'accept: */*' \
|
||||
-H 'origin: https://www.nimo.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.nimo.tv/lives' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
--data-binary $'------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="keyType"\r\n\r\n0\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="body"\r\n\r\n{"deviceType":7,"requestSource":"WEB","iNetType":5}\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q--\r\n' \
|
||||
--compressed`
|
||||
|
||||
curl := gcurl.Parse(fmt.Sprintf(nimoapi, 1))
|
||||
tp := curl.CreateTemporary(nil)
|
||||
|
||||
resp, err := tp.Execute()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return resp.Content()
|
||||
}
|
||||
|
||||
func openTestFile(...interface{}) []byte {
|
||||
f, err := os.Open("../../../testfile/nimo1.json")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
data, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
var Crawl func(...interface{}) []byte
|
||||
|
||||
func Test(t *testing.T) {
|
||||
|
||||
Execute()
|
||||
|
||||
}
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
package main
|
||||
|
||||
import "github.com/474420502/hunter"
|
||||
|
||||
func main() {
|
||||
Execute()
|
||||
ht := hunter.NewHunter(openrecRanking)
|
||||
ht.Execute()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,24 +4,29 @@ import (
|
|||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/474420502/hunter"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// // sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
// var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
|
||||
var openrecRanking *OpenrecRanking
|
||||
|
||||
// // estore 解析存储连接实例
|
||||
// var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
|
||||
|
||||
// Execute 执行方法
|
||||
func Execute() {
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
turl := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
func init() {
|
||||
|
||||
openrecRanking = &OpenrecRanking{}
|
||||
openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
-H 'authority: public.openrec.tv' \
|
||||
-H 'accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
|
||||
|
|
@ -34,25 +39,43 @@ func Execute() {
|
|||
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
|
||||
--compressed`
|
||||
|
||||
g := gcurl.Parse(turl)
|
||||
tp := g.Temporary()
|
||||
}
|
||||
|
||||
for !ps.IsClose() {
|
||||
// OpenrecRanking 获取排名任务
|
||||
type OpenrecRanking struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
resp, err := tp.Execute()
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
var loop int32 = 1
|
||||
|
||||
go func() {
|
||||
signalchan := make(chan os.Signal)
|
||||
signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
|
||||
log.Println("accept stop command:", <-signalchan)
|
||||
atomic.StoreInt32(&loop, 0)
|
||||
}()
|
||||
|
||||
for atomic.LoadInt32(&loop) > 0 {
|
||||
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
tp := cxt.Temporary()
|
||||
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 { //末页时没有内容返回, 末页退出
|
||||
finishpoint := time.Now()
|
||||
log.Println("任务Ranking UserId结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*120))
|
||||
for time.Now().Sub(finishpoint) < time.Minute*120 {
|
||||
time.Sleep(time.Second)
|
||||
if ps.IsClose() {
|
||||
if atomic.LoadInt32(&loop) <= 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -70,8 +93,15 @@ func Execute() {
|
|||
for _, User := range result.Array() {
|
||||
|
||||
userid := User.Get("channel.id").String()
|
||||
|
||||
// data := &intimate.Source{}
|
||||
// data.Source = sql.NullString{String: userid, Valid: len(userid) > 0}
|
||||
// data.Url = tp.GetRawURL()
|
||||
// data.TargetType = string(intimate.TTOpenrecUser)
|
||||
// sstore.Insert(data)
|
||||
|
||||
streamer := &intimate.Streamer{}
|
||||
streamer.UserId = &userid
|
||||
streamer.UserId = userid
|
||||
streamer.Platform = intimate.Popenrec
|
||||
|
||||
updateUrl := make(map[string]interface{})
|
||||
|
|
@ -83,18 +113,27 @@ func Execute() {
|
|||
|
||||
updateUrlBytes, err := json.Marshal(updateUrl)
|
||||
if err != nil {
|
||||
intimate.TStreamer.UpdateError(streamer, err)
|
||||
estore.UpdateError(streamer, err)
|
||||
continue
|
||||
}
|
||||
|
||||
streamer.UpdateUrl = updateUrlBytes
|
||||
intimate.TStreamer.Insert(streamer)
|
||||
estore.InsertStreamer(streamer)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("streamer count:", len(result.Array()), tp.ParsedURL.String())
|
||||
// 修改url query 参数的page递增. 遍历所有页面
|
||||
tp.QueryParam("page").IntAdd(1)
|
||||
querys := tp.GetQuery()
|
||||
page, err := strconv.Atoi(querys.Get("page"))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
page++
|
||||
querys.Set("page", strconv.Itoa(page))
|
||||
tp.SetQuery(querys)
|
||||
|
||||
time.Sleep(time.Second * 1)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,67 @@ package main
|
|||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
// OpenrecRanking 获取排名任务
|
||||
type OpenrecRankingTest struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
t := cxt.GetShare("test").(*testing.T)
|
||||
if !gjson.ValidBytes(resp.Content()) {
|
||||
t.Error("source is not json format.")
|
||||
}
|
||||
result := gjson.GetBytes(resp.Content(), "0.rank")
|
||||
if result.Int() != 1 {
|
||||
t.Error("rank is error. result raw is ", result.Raw)
|
||||
}
|
||||
|
||||
if cxt.Temporary().GetQuery().Get("page") != "1" {
|
||||
t.Error("Temporary page error")
|
||||
}
|
||||
// t.Error(string(resp.Content()))
|
||||
}
|
||||
|
||||
func TestRanking(t *testing.T) {
|
||||
curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
-H 'authority: public.openrec.tv' \
|
||||
-H 'accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
|
||||
-H 'origin: https://www.openrec.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.openrec.tv/ranking' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
|
||||
--compressed`
|
||||
|
||||
ort := &OpenrecRankingTest{}
|
||||
ort.PreCurlUrl = hunter.PreCurlUrl(curlBash)
|
||||
ht := hunter.NewHunter(ort)
|
||||
ht.SetShare("test", t)
|
||||
ht.Execute()
|
||||
}
|
||||
|
||||
func TestTimeAdd(t *testing.T) {
|
||||
finishpoint := time.Now()
|
||||
time.Sleep(time.Second * 2)
|
||||
t.Error(time.Now().Sub(finishpoint) > time.Second*1)
|
||||
}
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
main()
|
||||
func TestRankingInsert(t *testing.T) {
|
||||
ht := hunter.NewHunter(openrecRanking)
|
||||
ht.Execute()
|
||||
}
|
||||
|
|
|
|||
2
tasks/openrec/openrec_task2/.gitignore
vendored
Normal file
2
tasks/openrec/openrec_task2/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
openrec_task2
|
||||
log
|
||||
8
tasks/openrec/openrec_task2/main.go
Normal file
8
tasks/openrec/openrec_task2/main.go
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
package main
|
||||
|
||||
import "github.com/474420502/hunter"
|
||||
|
||||
func main() {
|
||||
ht := hunter.NewHunter(oer)
|
||||
ht.Execute()
|
||||
}
|
||||
178
tasks/openrec/openrec_task2/task_openrec.go
Normal file
178
tasks/openrec/openrec_task2/task_openrec.go
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
var oer *OpenrecExtratorRanking
|
||||
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
|
||||
|
||||
// estore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_extractor.sql
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
func init() {
|
||||
oer = &OpenrecExtratorRanking{}
|
||||
}
|
||||
|
||||
// OpenrecExtratorRanking 获取用户信息
|
||||
type OpenrecExtratorRanking struct {
|
||||
// Store *intimate.Store
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
var loop int32 = 1
|
||||
|
||||
go func() {
|
||||
signalchan := make(chan os.Signal)
|
||||
signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
|
||||
log.Println("accept stop command:", <-signalchan)
|
||||
atomic.StoreInt32(&loop, 0)
|
||||
}()
|
||||
|
||||
var lasterr error = nil
|
||||
|
||||
for atomic.LoadInt32(&loop) > 0 {
|
||||
|
||||
streamer, err := estore.Pop(intimate.Popenrec) //队列里弹出一个streamer行. 进行解析
|
||||
|
||||
if streamer == nil || err != nil {
|
||||
if err != lasterr {
|
||||
log.Println(err, lasterr)
|
||||
lasterr = err
|
||||
}
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
userId := streamer.UserId
|
||||
|
||||
var updateUrl map[string]string
|
||||
|
||||
err = json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl) // 反序列化update_url, 里面存了需要采集的url
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
// Check Userid
|
||||
|
||||
userUrl := updateUrl["user"]
|
||||
tp := cxt.Session().Get(userUrl) // 获取user url页面数据
|
||||
resp, err := tp.Execute()
|
||||
streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
estore.UpdateError(streamer, err)
|
||||
continue
|
||||
}
|
||||
|
||||
cookies := cxt.Session().GetCookies(tp.GetParsedURL())
|
||||
|
||||
scurl := updateUrl["supporters"] //获取打赏者的数据
|
||||
curl := gcurl.ParseRawCURL(scurl)
|
||||
supportersSession := curl.CreateSession()
|
||||
|
||||
temporary := curl.CreateTemporary(supportersSession)
|
||||
supportersSession.SetCookies(temporary.GetParsedURL(), cookies)
|
||||
var supporters []string
|
||||
for { // supporters 数据需要登录信息. 下面为赋值 supporters链接获取的uid token random码
|
||||
|
||||
supportersQuery := temporary.GetQuery()
|
||||
|
||||
for _, cookie := range cookies {
|
||||
if cookie.Name == "uuid" {
|
||||
supportersQuery.Set("Uuid", cookie.Value)
|
||||
continue
|
||||
}
|
||||
|
||||
if cookie.Name == "token" {
|
||||
supportersQuery.Set("Token", cookie.Value)
|
||||
continue
|
||||
}
|
||||
|
||||
if cookie.Name == "random" {
|
||||
supportersQuery.Set("Random", cookie.Value)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
supportersQuery.Set("identify_id", userId)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
|
||||
resp, err := temporary.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
supporterjson := gjson.ParseBytes(resp.Content())
|
||||
supporterdata := supporterjson.Get("data") //解析supporters获取的json数据
|
||||
if supporterdata.Type == gjson.Null {
|
||||
break
|
||||
}
|
||||
supporters = append(supporters, string(resp.Content()))
|
||||
|
||||
page := supportersQuery.Get("page_number") // page_number 加1
|
||||
pageint, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
pageint++
|
||||
page = strconv.Itoa(pageint)
|
||||
supportersQuery.Set("page_number", page)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
}
|
||||
|
||||
// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
ext := make(map[string]interface{})
|
||||
|
||||
ext["json_supporters"] = supporters
|
||||
ext["html_user"] = string(resp.Content())
|
||||
|
||||
liveUrl := updateUrl["live"]
|
||||
tp = cxt.Session().Get(liveUrl)
|
||||
resp, err = tp.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
estore.UpdateError(streamer, err)
|
||||
continue
|
||||
}
|
||||
ext["html_live"] = string(resp.Content())
|
||||
ext["var_user_id"] = userId
|
||||
|
||||
extJsonBytes, err := json.Marshal(ext)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
estore.UpdateError(streamer, err)
|
||||
continue
|
||||
}
|
||||
|
||||
streamer.Operator = int32(intimate.OperatorOK)
|
||||
|
||||
source := &intimate.Source{}
|
||||
source.Target = intimate.TOpenrecUser
|
||||
source.Ext = string(extJsonBytes)
|
||||
source.StreamerId = sql.NullInt64{Int64: streamer.Uid, Valid: true}
|
||||
sstore.Insert(source)
|
||||
|
||||
estore.UpdateOperator(streamer)
|
||||
}
|
||||
|
||||
}
|
||||
12
tasks/openrec/openrec_task2/task_openrec_test.go
Normal file
12
tasks/openrec/openrec_task2/task_openrec_test.go
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
func TestOpenrecUser(t *testing.T) {
|
||||
ht := hunter.NewHunter(oer)
|
||||
ht.Execute()
|
||||
}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
twitcasting_task1
|
||||
log
|
||||
|
|
@ -1,991 +0,0 @@
|
|||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
|
||||
<html>
|
||||
<head>
|
||||
<title>Live with Tag: vocaloid - TwitCasting</title>
|
||||
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
|
||||
<link rel="stylesheet" type="text/css" media="all" href="/css/mainstyle-us.css?1595400959">
|
||||
|
||||
|
||||
<meta name="viewport" content="width=device-width, user-scalable=no"/>
|
||||
|
||||
|
||||
<meta name="theme-color" content="#3381ff"/>
|
||||
<link href="/img/icon192.png" rel="icon" sizes="192x192"/>
|
||||
<link href="/img/icon72@2x.png" rel="apple-touch-icon" type="image/png"/>
|
||||
<link rel="stylesheet" href="/css/bootstrap.css?1596704154" type="text/css" media="all"/>
|
||||
<script type="text/javascript" src="/js/jquery-1.7.2.min.js"></script>
|
||||
<script crossorigin="anonymous" src="https://polyfill.io/v3/polyfill.min.js?features=default%2Ces2015%2Ces2016%2CIntersectionObserver%2CNodeList.prototype.forEach%2CNodeList.prototype.%40%40iterator%2CPromise.prototype.finally%2Cfetch"></script>
|
||||
<script type="text/javascript" src="/js/moiad.js?1526617899" defer></script>
|
||||
<script type="text/javascript" src="/js/Main.js?1597109494"></script>
|
||||
<meta name="format-detection" content="telephone=no">
|
||||
|
||||
<script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-51336-17', 'auto');
|
||||
ga('require', 'displayfeatures');
|
||||
ga('send', 'pageview'); ga('create', 'UA-51336-19', 'auto', {'name': 'usTracker'});
|
||||
ga('usTracker.require', 'displayfeatures');
|
||||
ga('usTracker.send', 'pageview');</script> <link rel="stylesheet" href="/css/search.css?1578279147" type="text/css" media="all"/>
|
||||
</head>
|
||||
<body data-is-mobile="false">
|
||||
|
||||
|
||||
<div class="tw-global-header-locator">
|
||||
<nav class="tw-global-header global-nav" data-lang="en"
|
||||
data-from-device="false"
|
||||
data-is-login="false"
|
||||
data-is-mobile="false"
|
||||
>
|
||||
<div class="tw-global-header-container header-container">
|
||||
<span class="tw-header-toggle-button">
|
||||
<a class="tw-hamburger-icon" href="#">
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
</a>
|
||||
</span>
|
||||
<div class="tw-global-header-primary">
|
||||
<h1 class="tw-global-header-logo">
|
||||
<a href="/">
|
||||
<div class="tw-global-header-logo-large">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1077.3 202"><defs/><g id="レイヤー_2" data-name="レイヤー 2"><g id="Layer_1" data-name="Layer 1"><path d="M946.48 0l-6.46 30.27h15.89l-1.56 7.32h-25.86L936.46 0h10.02zm40.29 0l-7.98 37.59h-10.01L976.75 0h10.02zm23.91 0c1.72 18.37 2.42 27.2 2.48 28.92h.1c.76-1.72 5.12-10.66 14.6-28.92h10.72l-20.74 37.59h-12.39L1000.5 0zm66.62 0l-1.4 7.05h-15.4l-1.62 7.81h14.55l-1.46 7.06h-14.6l-1.77 8.46h15.83l-1.4 7.21h-25.64L1052.36 0h24.94zM106.37 27.88l-5.11 23.65-34.07-.37L44.1 159.99H12.68L36.15 51.16 0 51.53l5.1-23.65h101.27z" fill="#0472fc"/><path class="cls-1" d="M97.84 160L86.1 62.13h29.72c4.16 57.73 4.73 69.47 4.92 75h.37c2.09-5.67 6.44-16.65 33.51-75h34.82c2.65 56.4 3 69.65 3 75.15h.38c2.46-6.25 7-16.85 35.77-75.15h31.05L206.1 160h-35.58c-3.22-59.25-3.41-69.65-3.41-74.77h-.38c-1.51 4.55-4.92 13.63-33.31 74.77zM307.9 45.29h-29.72l5.49-24.42h29.53zM254.52 160l20.83-97.86h29.71L284.24 160zM375.08 31.66l-6.62 30.47h23.66l-4.17 21H364.1l-8.51 39.94a32.22 32.22 0 0 0-.95 7.38c0 6.06 3.41 9.65 10.79 9.65a31.49 31.49 0 0 0 9.84-1.7l-3.78 21.2a62.77 62.77 0 0 1-15.14 1.89c-20.83 0-31.24-10.22-31.24-27.26a74.81 74.81 0 0 1 1.9-14.76L335 83.14h-17.6l4.54-21h17.6L344 40.93zm136.45 25.17c-7.76-4-17.41-6.06-28-6.06-30.85 0-50.34 22.34-50.34 51.11 0 23.09 15.14 35.4 38.42 35.4a68.1 68.1 0 0 0 25.93-5.11l-4.17 25.54c-8.13 2.46-17 3.79-28.76 3.79-35.59 0-63.79-18.36-63.79-57.73 0-43.72 34.45-77.22 81.2-77.22 13.63 0 26.69 2.27 36.53 6.25zm58.66 58.3c-22.14 0-31 6.25-31 16.28 0 7.19 5.3 11.16 12.49 11.16 14.39 0 24.8-11.35 26.88-26.87a65.78 65.78 0 0 0-8.37-.57zM575.3 145c-7.95 12.3-19.3 16.84-33.12 16.84-18.74 0-31.61-11-31.61-28.39 0-26.31 26.88-34.83 56.78-34.83 3.6 0 8.71.19 13.63.57a24.1 24.1 0 0 0 .38-4.92c0-10-7-14.58-20.25-14.58-9.47 0-21.58 2.28-30.85 7L532.91 65a109.76 109.76 0 0 1 33.88-5.3c26.31 0 41.64 10.22 41.64 31.23 0 11-7.19 40.51-8.9 52a115.06 115.06 0 0 0-1.7 17H574a98.43 98.43 0 0 1 1.7-15zM693 86.17a52.92 52.92 0 0 0-24-5.87c-9.09 0-14.58 2.46-14.58 8.33 0 5.11 3.22 7.38 13.82 13.25 13.82 7.57 21.2 14.57 21.2 27.83 0 24.41-22.15 32.74-44.86 32.74-12.11 0-24.6-2.28-33.69-6.06l6.06-22.15c8 4 18.92 6.82 27.44 6.82 10.41 0 15-3.22 15-9.46 0-4.74-3.21-8.14-15.14-14.39-14.2-7.57-19.87-14.95-19.87-26.12 0-20.06 18.73-31.42 44.86-31.42a79.88 79.88 0 0 1 29.9 5.68zm67-54.51l-6.63 30.47H777l-4.16 21H749l-8.52 39.94a32.09 32.09 0 0 0-.94 7.38c0 6.06 3.4 9.65 10.79 9.65a31.48 31.48 0 0 0 9.83-1.7l-3.78 21.2a62.77 62.77 0 0 1-15.14 1.89c-20.82 0-31.23-10.22-31.23-27.26a75.49 75.49 0 0 1 1.89-14.76l8-36.34h-17.6l4.54-21h17.6l4.56-21.2zm70.78 13.63h-29.71l5.48-24.42h29.53zM777.4 160l20.82-97.86h29.72L807.12 160zm98.22-82.35c8.33-12.3 19.88-18 32-18C928.44 59.67 940 72.54 940 92a85.47 85.47 0 0 1-2.08 18.36L927.49 160h-29c3.22-15.52 6.43-31 9.84-46.57 1.33-6.43 2.27-12.3 2.27-16.08 0-8.9-4.35-14.58-13.44-14.58-15.14 0-23.09 13.63-29 40.89L860.48 160h-29.14l16.09-76.66c1.32-6.43 2.64-14 3.59-21.2h26.88c-.38 4.36-1.71 11-2.65 15.52zm146.87 24.42c0-12.49-6.63-21.39-18-21.39-14.77 0-27.07 16.84-27.07 36.53 0 12.11 5.49 21.2 17.41 21.2 15.92 0 27.66-17.6 27.66-36.34zm9.27-39.94H1060l-20.06 93.32c-6.43 30.09-23.28 46.55-59.62 46.55a114.39 114.39 0 0 1-38-7l6.62-22.9c10.6 4.92 21.77 7.76 31.8 7.76 20.06 0 30.09-9.84 35.2-36.91h-.37c-7.2 10.6-17.8 16.66-31.24 16.66-23.46 0-36.53-16.47-36.53-39.75 0-32.74 21.2-60.19 51.68-60.19 14.76 0 24.22 6.63 29 17.79h.37z" fill="#0472fc"/><path fill="#0472fc" d="M919.73 195H5.84l2.97-14h913.9l-2.98 14z" fill="#0472fc"/></g></g></svg>
|
||||
</div>
|
||||
<div class="tw-global-header-logo-small">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g data-name="レイヤー 2"><g data-name="Layer 1"><path fill="#fff" fill-rule="evenodd" d="M7.65 14.38a1.07 1.07 0 1 1 2.13 0 1.07 1.07 0 0 1-2.13 0z"/><path class="cls-1" fill="#fff" d="M17.79 21.89a11 11 0 0 1-1.33.67A11.43 11.43 0 0 1 3.9 20.1c-.14-.14-.28-.28-.41-.43q.31-.51.72-1.05a5.09 5.09 0 0 1 1-1A3 3 0 0 1 7.7 17c1 .23.68 1 .68 1 .87.34.32 1.38.32 1.38a.88.88 0 0 1 .05.15h.1a1.29 1.29 0 0 0 1.29-1.29v-1.33l1.73 1.16a.52.52 0 0 0 .51 0l1.8-1.21-2 .6h-.09l-1.93-.57v-.32l2 .58 2.29-.68-.06-.06-2-1.22a.53.53 0 0 0-.52 0l-1.72 1.07v-2a1.29 1.29 0 0 0-1.29-1.29H6.49a1.29 1.29 0 0 0-1.29 1.25v2.39A7.79 7.79 0 0 1 8.94 6l2.74-2.41c.15-.13.49-.43.74-.3s.11.4.09.45a4.27 4.27 0 0 0-.15 1.58 2.81 2.81 0 0 1 1.85-1c.43-.06.37.2.31.26-.67.78.77 1.41.77 1.41a7.78 7.78 0 0 1 .65 13.88 6.84 6.84 0 0 1 1.85 2.02z"/><path d="M14.91 13.63a.81.81 0 1 0 .81.81.81.81 0 0 0-.81-.81zM23.73 12a11.78 11.78 0 0 1-5.82 10.14A11.73 11.73 0 0 0 23.73 12zm-15 1.34a1.07 1.07 0 1 0 0 2.13 1.07 1.07 0 0 0 0-2.13z" fill="#0073ff" fill-rule="evenodd"/><path class="cls-3" fill="#0073ff" d="M23.73 12a11.78 11.78 0 0 1-5.82 10.14A11.73 11.73 0 0 0 23.73 12z"/><path class="cls-3" fill="#0073ff" d="M23.06 7.33A12 12 0 0 0 16.67.94a12.07 12.07 0 0 0-9.34 0A12 12 0 0 0 .94 7.33a12.07 12.07 0 0 0 0 9.34 12 12 0 0 0 6.39 6.39 12.07 12.07 0 0 0 9.34 0 12 12 0 0 0 6.39-6.39 12.07 12.07 0 0 0 0-9.34zm-6.6 15.23A11.43 11.43 0 0 1 3.9 20.1c-.14-.14-.28-.28-.41-.43q.31-.51.72-1.05a5.09 5.09 0 0 1 1-1A3 3 0 0 1 7.7 17c1 .23.68 1 .68 1 .87.34.32 1.38.32 1.38a.88.88 0 0 1 .05.15h.1a1.29 1.29 0 0 0 1.29-1.29v-1.33l1.73 1.16a.52.52 0 0 0 .51 0l1.8-1.21-2 .6h-.09l-1.93-.57v-.32l2 .58 2.29-.68-.06-.06-2-1.22a.53.53 0 0 0-.52 0l-1.72 1.07v-2a1.29 1.29 0 0 0-1.29-1.29H6.49a1.29 1.29 0 0 0-1.29 1.25v2.39A7.79 7.79 0 0 1 8.94 6l2.74-2.41c.15-.13.49-.43.74-.3s.11.4.09.45a4.27 4.27 0 0 0-.15 1.58 2.81 2.81 0 0 1 1.85-1c.43-.06.37.2.31.26-.67.78.77 1.41.77 1.41a7.78 7.78 0 0 1 .65 13.88 6.84 6.84 0 0 1 1.85 2.05 11 11 0 0 1-1.33.64zm-8.81-8.18a1.07 1.07 0 1 1 2.13 0 1.07 1.07 0 0 1-2.13 0zm10.27 7.75A11.78 11.78 0 0 0 23.73 12a11.73 11.73 0 0 1-5.81 10.13z"/><path class="cls-1" fill="#fff" d="M8.72 13.31a1.07 1.07 0 1 0 0 2.13 1.07 1.07 0 0 0 0-2.13zm0 1.62a.55.55 0 0 1-.55-.55.55.55 0 1 1 .55.55z"/></g></g></svg> </div>
|
||||
</a>
|
||||
</h1>
|
||||
<form class="tw-global-header-mobile-search-bar" id="global-nav-search" method="post"
|
||||
action="/search/text/">
|
||||
<input type="search" class="tw-global-header-mobile-search-input" id="search-query" name="q"
|
||||
value=""
|
||||
placeholder="Search">
|
||||
<button type="submit" class="tw-global-header-mobile-search-button" aria-label="Search">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M17.36 15.837l6.326 6.109a1.029 1.029 0 1 1-1.43 1.48l-6.344-6.127a9.707 9.707 0 0 1-6.164 2.196C4.364 19.495 0 15.131 0 9.748A9.748 9.748 0 0 1 9.748 0c5.383 0 9.747 4.364 9.747 9.748a9.71 9.71 0 0 1-2.135 6.089zm-7.612 1.6a7.69 7.69 0 1 0 0-15.38 7.69 7.69 0 0 0 0 15.38z" fill="#8D9296"/></svg>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
<div class="tw-global-header-secondary">
|
||||
|
||||
<div class="tw-global-header-nav-item tw-global-header-nav-item-broadcast" data-can-hover="true">
|
||||
<a class="tw-global-header-nav-item-default" href="/indexloginwindow.php?next=broadcaster">
|
||||
<span class="tw-global-header-icon-broadcast">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24"><path d="M22.286 17.031L18 15.108v-4.275-.942l4.287-1.922v9.062zm-6-2.864c0 2.046 0 3.4-.395 3.782s-1.785.387-3.889.387H6c-2.1 0-3.492 0-3.887-.387s-.4-1.736-.4-3.782v-3.334c0-2.046 0-3.4.4-3.782S3.9 6.664 6 6.664h6c2.1 0 3.494 0 3.89.387s.394 1.736.394 3.782zm1.654-6.084a3.248 3.248 0 0 0-.84-2.208C16.2 5 14.8 5 12 5H6c-2.8 0-4.2 0-5.1.872S0 8.112 0 10.833v3.334c0 2.72 0 4.089.9 4.96S3.2 20 6 20h6c2.8 0 4.2 0 5.1-.872a3.259 3.259 0 0 0 .84-2.211l6.06 2.72V5.363z" fill-rule="evenodd"/></svg>
|
||||
</span>
|
||||
<span>Broadcast</span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="tw-global-header-nav-item tw-global-header-nav-item-notifier" data-can-hover="true" aria-selected="false">
|
||||
<a class="tw-global-header-nav-item-default"
|
||||
href="/tcairindex.php">Subscriptions</a>
|
||||
</div>
|
||||
|
||||
<div class="tw-global-header-nav-item tw-global-header-nav-item-login">
|
||||
<span class="tw-global-header-nav-item-highlight">
|
||||
Log In <span class="tw-global-header-icon-down"><svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M.531 8.035c-.708-.708-.708-1.911 0-2.62l.39-.388c.707-.708 1.91-.708 2.619 0l8.46 8.46 8.46-8.46c.708-.708 1.912-.708 2.62 0l.389.389c.708.708.708 1.911 0 2.62L12 19.503.531 8.035z" fill="#000" fill-rule="evenodd"/></svg>
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="tw-global-header-login-popup-locator">
|
||||
<div class="tw-global-header-login-popup">
|
||||
<div class="tw-global-header-block">
|
||||
<ul class="tw-global-header-login-button-list">
|
||||
<li>
|
||||
<a class="tw-global-header-login-button" href="#" data-social="twitter"
|
||||
onclick="go_auth(); return false;">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M7.548 21.501c9.056 0 14.01-7.503 14.01-14.01 0-.213 0-.425-.015-.636A10.019 10.019 0 0 0 24 4.305a9.828 9.828 0 0 1-2.828.776 4.941 4.941 0 0 0 2.165-2.724 9.868 9.868 0 0 1-3.127 1.195 4.929 4.929 0 0 0-8.391 4.491A13.98 13.98 0 0 1 1.67 2.9a4.928 4.928 0 0 0 1.525 6.573A4.887 4.887 0 0 1 .96 8.855v.063a4.926 4.926 0 0 0 3.95 4.827 4.916 4.916 0 0 1-2.223.084 4.93 4.93 0 0 0 4.6 3.42A9.88 9.88 0 0 1 0 19.289a13.94 13.94 0 0 0 7.548 2.209" fill="#FFF" fill-rule="evenodd"/></svg>
|
||||
Twitter </a>
|
||||
</li>
|
||||
<li>
|
||||
<a class="tw-global-header-login-button" href="#" data-social="facebook"
|
||||
onclick="go_auth('f'); return false;">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M22.676 0H1.324C.593 0 0 .593 0 1.324v21.352C0 23.407.593 24 1.324 24h11.504v-9.281H9.703v-3.633h3.125V8.412c0-3.099 1.895-4.787 4.659-4.787.931-.002 1.862.045 2.789.14v3.24h-1.904c-1.506 0-1.8.712-1.8 1.763v2.313h3.6l-.467 3.633h-3.153V24h6.124c.731 0 1.324-.593 1.324-1.324V1.324C24 .593 23.407 0 22.676 0z" fill="#FFF" fill-rule="evenodd"/></svg>
|
||||
Facebook </a>
|
||||
</li>
|
||||
<li>
|
||||
<a class="tw-global-header-login-button" href="#" data-social="cas"
|
||||
onclick="go_auth('c'); return false;">
|
||||
<img src="/img/icon192.png">
|
||||
TwitCasting </a>
|
||||
</li>
|
||||
</ul>
|
||||
<span class="tw-global-header-terms">
|
||||
By logging in, you are indicating that you have read and agreed to our <a href="/indexlicense.php">Terms of Service.</a> </span>
|
||||
<a class="tw-global-header-other-login" href="/indexloginwindow.php">Login with other SNS</a>
|
||||
</div>
|
||||
<div class="tw-global-header-block">
|
||||
<span class="tw-global-header-block-title">Create New Account</span>
|
||||
<a href="/indexcasregister.php" class="tw-button-secondary tw-button-large" data-is-block="true">Sign Up</a>
|
||||
<form class="tw-global-header-login-form">
|
||||
<input type="hidden" id="keep" checked>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div class="tw-sidebar">
|
||||
<div class="tw-sidebar-content" data-tablet="false">
|
||||
<section class="tw-sidebar-main-column">
|
||||
<ul class="tw-sidebar-main-navi">
|
||||
<li class="tw-sidebar-live" >
|
||||
<a href="/?ch0">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 17"><path d="M19.7 5.6c-.1-1-1-1.8-2-1.9h-.2c-1.5-.1-3.1-.3-4.7-.4-1.6-.1-3.3-.1-5 0h-.4l1.2-1.5h.2c.6 0 1-.4 1-1 0-.5-.4-1-1-1-.5 0-1 .5-1 1V1L4.4 3.5c-.7 0-1.3.1-1.9.2-1.2.2-2 .9-2.1 2.1-.1.3-.2.7-.2 1.1v.7c-.1.8-.2 1.7-.2 2.6 0 1.3.1 2.6.3 4.1.1 1 .9 1.8 1.9 1.9 1 .1 1.9.2 2.7.3 1.3.1 2.6.2 3.6.2h.9c3.4 0 5.9-.2 8.3-.5 1.1-.1 1.9-.9 2-1.9.1-1 .2-2.3.3-3.7 0-1.7-.2-3.5-.3-5zm-1.5 5c-.1 1-.2 2.1-.3 3.6 0 .2-.2.3-.5.4-2.3.3-4.7.5-8 .5h-.9c-1 0-2.2-.1-3.5-.2-.8-.1-1.6-.1-2.6-.3-.1 0-.3-.2-.3-.4-.2-1.5-.3-2.7-.3-3.9 0-.9.1-1.7.1-2.6L2 7c0-.3.1-.7.1-1 .1-.4.2-.5.6-.5C4.3 5.2 6 5.1 7.9 5h4.8c1.5.1 3.1.3 4.6.4h.2c.2 0 .3.2.3.4.2 1.5.4 3.2.4 4.8z" fill-rule="evenodd" clip-rule="evenodd" fill="#9ca1a6"/></svg> Live </a>
|
||||
</li>
|
||||
<li class="tw-sidebar-category tw-sidebar-category-category">
|
||||
<a href="#">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 16"><path class="st0" d="M1 0h3c.3 0 .5.2.5.5v3c0 .3-.2.5-.5.5H1c-.3 0-.5-.2-.5-.5v-3C.5.2.7 0 1 0zm6 1h12c.3 0 .5.2.5.5v1c0 .3-.2.5-.5.5H7c-.3 0-.5-.2-.5-.5v-1c0-.3.2-.5.5-.5zM1 6h3c.3 0 .5.2.5.5v3c0 .3-.2.5-.5.5H1c-.3 0-.5-.2-.5-.5v-3c0-.3.2-.5.5-.5zm6 1h12c.3 0 .5.2.5.5v1c0 .3-.2.5-.5.5H7c-.3 0-.5-.2-.5-.5v-1c0-.3.2-.5.5-.5zm-6 5h3c.3 0 .5.2.5.5v3c0 .3-.2.5-.5.5H1c-.3 0-.5-.2-.5-.5v-3c0-.3.2-.5.5-.5zm6 1h12c.3 0 .5.2.5.5v1c0 .3-.2.5-.5.5H7c-.3 0-.5-.2-.5-.5v-1c0-.3.2-.5.5-.5z" fill="#9ca1a6"/></svg> Category </a>
|
||||
</li>
|
||||
<li class="tw-sidebar-ranking" >
|
||||
<a href="/rankingindex.php">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 18"><path d="M17.8 5.4c-.4 0-.7-.3-.7-.7s.3-.7.7-.7.7.3.7.7-.3.7-.7.7zm-2.3 10.5c0 .1-.1.1-.1.1H4.6c-.1 0-.1-.1-.1-.1v-2.1h11v2.1zM1.5 4.6c0-.4.3-.7.7-.7s.7.3.7.7-.3.8-.7.8-.7-.4-.7-.8zm4.9 2.2c.4 0 .8-.1 1.1-.4l2-1.6c.1 0 .3.1.5.1h.3l2 1.6c.3.3.7.4 1.1.4.2 0 .4 0 .6-.1l2.1-.8.4.4-.8 5.7H4.4l-.8-5.7.3-.3 2 .7h.5zM10 2c.4 0 .7.3.7.7s-.3.7-.7.7-.7-.3-.7-.7.3-.7.7-.7zm7.8.5c-1.1 0-2.1.9-2.2 2l-2.2.7h-.1L11.8 4c.2-.4.4-.8.4-1.3 0-1.2-1-2.2-2.2-2.2s-2.2 1-2.2 2.2c0 .4.1.8.3 1.1L6.5 5.1h-.1l-2-.7c-.1-1.1-1-2-2.2-2C1 2.5 0 3.4 0 4.6c0 1.1.9 2.1 2 2.2l.8 6.2v2.9c0 1 .8 1.8 1.8 1.8h10.7c1 0 1.8-.8 1.8-1.8V13l.9-6.2c1.1-.1 2-1.1 2-2.2 0-1.2-1-2.1-2.2-2.1z" fill-rule="evenodd" clip-rule="evenodd" fill="#9ca1a6"/></svg> Ranking </a>
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<div class="tw-sidebar-category-list-wrapper">
|
||||
<ul class="tw-sidebar-category-list">
|
||||
|
||||
<li class="tw-sidebar-category-list-item">
|
||||
<span class="tw-sidebar-category-list-title">Category</span>
|
||||
<ul class="tw-sidebar-category-sub-list">
|
||||
<li class="tw-sidebar-category-sub-list-item">
|
||||
<a href="/?genre=category_gaming">
|
||||
Gaming <span class="tw-sidebar-category-live-count">(100)</span>
|
||||
</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="tw-sidebar-background"></div>
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="tw-global-header-spacer"></div>
|
||||
|
||||
<script type="text/javascript" src="/js/GlobalHeader.js?1593738718" defer></script>
|
||||
|
||||
|
||||
<div id="mainwrapper" class="clearfix">
|
||||
|
||||
<div class="tw-page tw-basic-page clearfix">
|
||||
|
||||
<div id="showadsheader"
|
||||
style="text-align:center;padding-top:8px;padding-bottom:8px;margin-top:8px;margin-bottom:-2px;overflow:hidden;max-height:90px;"><script type="text/javascript">
|
||||
window.addEventListener("DOMContentLoaded", function () {
|
||||
window.adLoader = Moiad.AdLoaderFactory.create(
|
||||
window,
|
||||
"https:\/\/gox.twitcasting.tv\/",
|
||||
"https:\/\/gox.twitcasting.tv\/",
|
||||
200
|
||||
);
|
||||
});
|
||||
</script><script type="text/javascript">
|
||||
window.addEventListener("DOMContentLoaded", function () {
|
||||
var tag = new Moiad.SingleAdTag();
|
||||
tag.setElementId("showadsheader");
|
||||
tag.setPlaceName("tcx_general_header_pc_responsive");
|
||||
adLoader.activate(tag);
|
||||
});
|
||||
</script></div>
|
||||
<div class="tw-basic-page-left-column">
|
||||
|
||||
<h2>
|
||||
Tag Search :
|
||||
<a href="/search/tag/vocaloid" class="tag tag-large tag-info">vocaloid</a> </h2>
|
||||
|
||||
<span class="tw-search-result-count">
|
||||
<a href="#tw-search-result-live">Live(0)</a> /
|
||||
<a href="#tw-search-result-user">User(28)</a>
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="tw-search-result-live" id="tw-search-result-live">
|
||||
</div>
|
||||
|
||||
<div class="tw-search-result-user" id="tw-search-result-user">
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/lhymecyas">
|
||||
<img src="//s01.twitcasting.tv/img/twitter_bigger_6.png" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/lhymecyas"><span class="username">月詠來夢@<wbr>きゃす</span></a>
|
||||
|
||||
<div class="fullname">@LhymeCyas</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">月詠來夢のきゃす垢</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E6%AD%8C%E3%81%A3%E3%81%A6%E3%81%BF%E3%81%9F" class="tag tag-mini tag-info">歌ってみた</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E9%BC%BB%E5%A3%B0" class="tag tag-mini tag-info">鼻声</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">初見さん大歓迎</a> <a href="/search/tag/%E9%9B%91%E8%AB%87" class="tag tag-mini tag-info">雑談</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:shiratama_dango">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/89/a3/54ac2466ad751_64.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:shiratama_dango"><span class="username">SHIRATAMA</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>shiratama_<wbr>dango</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">A creature which enjoy singing, humming while walking, and eating.</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/japanese" class="tag tag-mini tag-info">japanese</a> <a href="/search/tag/thai" class="tag tag-mini tag-info">thai</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/zunguri1459">
|
||||
<img src="//s01.twitcasting.tv/img/twitter_bigger_6.png" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/zunguri1459"><span class="username">ずんちゃ</span></a>
|
||||
|
||||
<div class="fullname">@zunguri145<wbr>9</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">はろはわゆ</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E3%83%9C%E3%82%AB%E3%83%AD" class="tag tag-mini tag-info">ボカロ</a> <a href="/search/tag/%E3%81%BF%E3%81%A3%E3%81%B1%E3%81%84" class="tag tag-mini tag-info">みっぱい</a> <a href="/search/tag/%E5%88%9D%E5%BF%83%E8%80%85" class="tag tag-mini tag-info">初心者</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/kyabet001">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/780186868507828224/8i-n-KG9_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/kyabet001"><span class="username">したばま <wbr>ましか</span></a>
|
||||
|
||||
<div class="fullname">@kyabet001</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">2Dホラーアクションシューティング「CARLA」を製作中です PV: https://t.co/PXBDoaNFUV よかったらフォロ/フォロバオナシャス</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/anime" class="tag tag-mini tag-info">anime</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/game" class="tag tag-mini tag-info">game</a> <a href="/search/tag/%E8%BB%8A" class="tag tag-mini tag-info">車</a> <a href="/search/tag/5bp." class="tag tag-mini tag-info">5bp.</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:grog">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/24/32/56c86d014226d_64.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:grog"><span class="username">ぐろぐ</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>grog</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc"></div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/MUSIC" class="tag tag-mini tag-info">MUSIC</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/utaite" class="tag tag-mini tag-info">utaite</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/nayu_notyou">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/860291445026037760/Ig-6IP48_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/nayu_notyou"><span class="username">んぁゆ <wbr>(<wbr>NaYu)<wbr></span></a>
|
||||
|
||||
<div class="fullname">@NaYu_<wbr>NotYou</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">🇮🇩🛫🇦🇺 || Main Account || Korean Related: @NaYu_ha || Pull me into more hells- || icon: @AyaminTwT</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/%E6%AD%8C%E3%81%86" class="tag tag-mini tag-info">歌う</a> <a href="/search/tag/%E3%83%9C%E3%82%AB%E3%83%AD" class="tag tag-mini tag-info">ボカロ</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/0konoha39">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/679256076361969664/-dQl81Pj_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/0konoha39"><span class="username">このは@<wbr>モンハン勢</span></a>
|
||||
|
||||
<div class="fullname">@0konoha39</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">DIVA、白猫、モンハンが大好きな大学生です(((((└(:D」┌)┘)))))))</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E3%81%93%E3%81%AE%E3%81%AF" class="tag tag-mini tag-info">このは</a> <a href="/search/tag/DIVA" class="tag tag-mini tag-info">DIVA</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E5%88%9D%E5%BF%83%E8%80%85" class="tag tag-mini tag-info">初心者</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/f:100000242133373">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100000242133373/picture?type=square" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/f:100000242133373"><span class="username">f:<wbr>Egao <wbr>No <wbr>ShouJou</span></a>
|
||||
|
||||
<div class="fullname">@f:<wbr>1000002421<wbr>33373</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc"></div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/sumairu" class="tag tag-mini tag-info">sumairu</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/song" class="tag tag-mini tag-info">song</a> <a href="/search/tag/live" class="tag tag-mini tag-info">live</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/rubierin_sna">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/828473387009740801/_fhSN6z3_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/rubierin_sna"><span class="username">みゆ</span></a>
|
||||
|
||||
<div class="fullname">@rubierin_<wbr>sna</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">sing! やってます✨</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/Thai" class="tag tag-mini tag-info">Thai</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/Anime" class="tag tag-mini tag-info">Anime</a> <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a> <a href="/search/tag/Smule" class="tag tag-mini tag-info">Smule</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/f:100004051574775">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100004051574775/picture?type=square" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/f:100004051574775"><span class="username">f:<wbr>Yuu'll <wbr>Be <wbr>Back</span></a>
|
||||
|
||||
<div class="fullname">@f:<wbr>1000040515<wbr>74775</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc"></div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/anime" class="tag tag-mini tag-info">anime</a> <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/english" class="tag tag-mini tag-info">english</a> <a href="/search/tag/thai" class="tag tag-mini tag-info">thai</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:mongdang">
|
||||
<img src="https://ssl.twitcasting.tv/img/twitcas_normal_0.png" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:mongdang"><span class="username">Mongdang</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>Mongdang</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc"></div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/Karaoke" class="tag tag-mini tag-info">Karaoke</a> <a href="/search/tag/Anisong" class="tag tag-mini tag-info">Anisong</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/Anime" class="tag tag-mini tag-info">Anime</a> <a href="/search/tag/%E6%AD%8C%E3%81%A3%E3%81%A6%E3%81%BF%E3%81%9F" class="tag tag-mini tag-info">歌ってみた</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/kamiy_ari">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/1235041741155078144/tIYD3zkT_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/kamiy_ari"><span class="username">狼谷ありー</span></a>
|
||||
|
||||
<div class="fullname">@kamiy_<wbr>ari</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">歌が大好きです。</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E3%82%AD%E3%83%A3%E3%82%B9%E5%88%9D%E5%BF%83%E8%80%85%E3%81%A7%E3%81%99" class="tag tag-mini tag-info">キャス初心者です</a> <a href="/search/tag/%E3%83%92%E3%83%88%E3%82%AB%E3%83%A9" class="tag tag-mini tag-info">ヒトカラ</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/f:100000714321056">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100000714321056/picture?type=square" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/f:100000714321056"><span class="username">f:<wbr>Thamanan <wbr>Wittayacha<wbr>mnankul</span></a>
|
||||
|
||||
<div class="fullname">@f:<wbr>1000007143<wbr>21056</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">อ่าา ชอบเกรียนที่สุดเบยย อ๊ายยย ขอให้ได้เกรียน เกรียนในสิ่งที่ชอบ และไม่เดือดร้อนคนอื่น โอเคโน๊ะ อยากจะเกรียนเพลงใส่กันเชิญจ้า กร๊ากก ผมเกรียนได้ทุกคนอยู่แระ ..</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/Anime" class="tag tag-mini tag-info">Anime</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/hikamody">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/976466557734813696/8jowWEM1_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/hikamody"><span class="username">hika</span></a>
|
||||
|
||||
<div class="fullname">@hikamody</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">성인/잡덕/Bi🌈/Feminist/Flexitarian</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/j-pop" class="tag tag-mini tag-info">j-pop</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/katie_diva_xoxo">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/649306799061884928/yBwmfjuO_normal.png" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/katie_diva_xoxo"><span class="username">totally <wbr>katië <wbr>123</span></a>
|
||||
|
||||
<div class="fullname">@katie_<wbr>diva_<wbr>xoxo</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">❤️❤️❤️zoella</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/Music" class="tag tag-mini tag-info">Music</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:vlemvpe">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/26/7f/54c62b8de296a_64.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:vlemvpe"><span class="username">Fujisaki <wbr>Hitomi</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>vlemvpe</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">Fujisaki Hitomi
|
||||
https://www.facebook.com/ciazfah2
|
||||
คนไทยค่า :D</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/thai" class="tag tag-mini tag-info">thai</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:hatune3625">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/9a/a5/57d36c0a68ee1_64.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:hatune3625"><span class="username">初音@<wbr>yu卍nan</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>hatune3625<wbr></div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">初音ミク大好き</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E9%9F%B3%E3%82%B2%E3%83%BC" class="tag tag-mini tag-info">音ゲー</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E3%82%B3%E3%83%A1%E3%83%B3%E3%83%88%E6%B0%97%E8%BB%BD%E3%81%AB" class="tag tag-mini tag-info">コメント気軽に</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:sara21ren">
|
||||
<img src="https://ssl.twitcasting.tv/img/twitcas_normal_6.png" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:sara21ren"><span class="username">Sarah</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>sara21ren</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">Hello. It's Sarah (: I play the piano and the guitar. Sometimes sing. Ttm hmu!!</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/Teen" class="tag tag-mini tag-info">Teen</a> <a href="/search/tag/english" class="tag tag-mini tag-info">english</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/kawazu1816">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/519776816798498817/2mQH2_CT_normal.jpeg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/kawazu1816"><span class="username">カワズ先輩≠カエル先輩</span></a>
|
||||
|
||||
<div class="fullname">@kawazu1816<wbr></div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">愛したって、愛されたっていいじゃないか</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E5%88%9D%E5%BF%83%E8%80%85" class="tag tag-mini tag-info">初心者</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E%21" class="tag tag-mini tag-info">初見さん大歓迎!</a> <a href="/search/tag/%E3%82%AB%E3%83%A9%E3%82%AA%E3%82%B1" class="tag tag-mini tag-info">カラオケ</a> <a href="/search/tag/%E7%8C%AB" class="tag tag-mini tag-info">猫</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/varenturu">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/846293033062563840/Hfw-YvYT_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/varenturu"><span class="username">ヴァレン（<wbr>実況者）<wbr></span></a>
|
||||
|
||||
<div class="fullname">@varenturu</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">不定期で実況動画あげていく予定です。 よろしくお願いします！！ 異常なほどな誤字脱字etc...沢山のハプニングがあると思います。 温かい目で見てやってくれればありが..</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E6%AD%8C%E3%81%A3%E3%81%A6%E3%81%BF%E3%81%9F" class="tag tag-mini tag-info">歌ってみた</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> <a href="/search/tag/Fate" class="tag tag-mini tag-info">Fate</a> <a href="/search/tag/%E6%9D%B1%E6%96%B9" class="tag tag-mini tag-info">東方</a> <a href="/search/tag/%E3%82%A2%E3%83%8B%E3%82%BD%E3%83%B3" class="tag tag-mini tag-info">アニソン</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:iasoa020">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/75/c8/5f257e6bce867_64.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:iasoa020"><span class="username">BARI</span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>iasoa020</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">歌を歌います。少しでもプラスに慣れればと思います。</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E3%83%9F%E3%83%A5%E3%83%BC%E3%82%B8%E3%82%AB%E3%83%AB" class="tag tag-mini tag-info">ミュージカル</a> <a href="/search/tag/%E5%A3%B0%E6%A5%BD" class="tag tag-mini tag-info">声楽</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/f:100007528892324">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100007528892324/picture?type=square" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/f:100007528892324"><span class="username">f:<wbr>Xio <wbr>Steph</span></a>
|
||||
|
||||
<div class="fullname">@f:<wbr>1000075288<wbr>92324</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc"></div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/anime" class="tag tag-mini tag-info">anime</a> <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/radio" class="tag tag-mini tag-info">radio</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/japan" class="tag tag-mini tag-info">japan</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/f:489672901190991">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/489672901190991/picture?type=square" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/f:489672901190991"><span class="username">f:<wbr>Kama <wbr>Jewell <wbr>Greco</span></a>
|
||||
|
||||
<div class="fullname">@f:<wbr>4896729011<wbr>90991</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc"></div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/Singing" class="tag tag-mini tag-info">Singing</a> <a href="/search/tag/acoustic" class="tag tag-mini tag-info">acoustic</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/singerwongwriter" class="tag tag-mini tag-info">singerwongwriter</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/kiryu_hanabusa">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/1189080132373204992/YF4bSxAB_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/kiryu_hanabusa"><span class="username">英 <wbr>姫柳/<wbr>かんじむずかしい</span></a>
|
||||
|
||||
<div class="fullname">@Kiryu_<wbr>Hanabusa</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">YouTuberになるために準備中です。 基本受けの体制なので話し掛けてもらうまでROMってることが多いです。気軽にお声掛けください！ codevein/競馬/麻雀/雀魂/FPS #姫柳の..</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/BUMPOFCHICKEN" class="tag tag-mini tag-info">BUMPOFCHICKEN</a> <a href="/search/tag/%E3%82%A2%E3%83%8B%E3%83%A1%E3%82%BD%E3%83%B3%E3%82%B0" class="tag tag-mini tag-info">アニメソング</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> <a href="/search/tag/%E3%82%B3%E3%83%A1%E3%83%B3%E3%83%88%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">コメント歓迎</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">初見さん大歓迎</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/mujamiku">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/972119417747161089/XiTi7dBX_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/mujamiku"><span class="username">ムジャオ＠<wbr>3/<wbr>9アイコン変わりました</span></a>
|
||||
|
||||
<div class="fullname">@mujamiku</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">ただのザコなボカロファンです。</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E3%83%9F%E3%82%AF%E3%82%B3%E3%83%AC" class="tag tag-mini tag-info">ミクコレ</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E9%9B%91%E8%AB%87" class="tag tag-mini tag-info">雑談</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">初見さん大歓迎</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/porkyuupine">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/838292064991236096/dx8lqDcW_bigger.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/porkyuupine"><span class="username">(<wbr>๑•̀ω•́๑)<wbr></span></a>
|
||||
|
||||
<div class="fullname">@porkyuupin<wbr>e</div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">❤ VOCALOID ｜ 歌い手 ｜ アニメと漫画 ❤ シンガポールからのファンです、よろしく〜☆ </div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/bored" class="tag tag-mini tag-info">bored</a> <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E3%83%9C%E3%82%AB%E3%83%AD" class="tag tag-mini tag-info">ボカロ</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row">
|
||||
<div class="tw-search-result-row-user">
|
||||
<div class="clearfix">
|
||||
<div class="indent">
|
||||
<div class="userimage32">
|
||||
<a href="/c:mimingdayo">
|
||||
<img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/29/14/586d949bef0e6_64.jpg" width="32" height="32" class="icon32">
|
||||
</a>
|
||||
</div>
|
||||
<div class="usertext">
|
||||
|
||||
<a href="/c:mimingdayo"><span class="username">miming(<wbr>みみん)<wbr></span></a>
|
||||
|
||||
<div class="fullname">@c:<wbr>mimingdayo<wbr></div> </div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-search-result-row-user-desc">韓国人です。日本語勉強はじめたのは10年くらい前からで
|
||||
留学はしたことありません。
|
||||
まぁ気ままにやります。
|
||||
ヾ(⌒(_*'ω'*)_</div>
|
||||
<div class="tw-search-result-row-user-tags">
|
||||
<a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> <a href="/search/tag/J-POP" class="tag tag-mini tag-info">J-POP</a> <a href="/search/tag/%E6%97%A5%E6%9C%AC%E8%AA%9E" class="tag tag-mini tag-info">日本語</a> <a href="/search/tag/%E9%9F%93%E5%9B%BD%E4%BA%BA" class="tag tag-mini tag-info">韓国人</a> </div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="tw-search-result-next-tags">
|
||||
<span class="tw-search-result-next-tags-title">Find From Live Tags</span>
|
||||
<a href="/search/tag/RTPDenVIVO" class="tag tag-info">RTPDenVIVO</a> <a href="/search/tag/Apex" class="tag tag-info">Apex</a> <a href="/search/tag/gdgd" class="tag tag-info">gdgd</a> <a href="/search/tag/DBD" class="tag tag-info">DBD</a> <a href="/search/tag/PUBG" class="tag tag-info">PUBG</a> <a href="/search/tag/ApexLegends" class="tag tag-info">ApexLegends</a> <a href="/search/tag/YouTube" class="tag tag-info">YouTube</a> <a href="/search/tag/VALORANT" class="tag tag-info">VALORANT</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-info">歌</a> <a href="/search/tag/v%E7%B3%BB" class="tag tag-info">v系</a> <a href="/search/tag/AAA" class="tag tag-info">AAA</a> <a href="/search/tag/MW" class="tag tag-info">MW</a> <a href="/search/tag/WoWs" class="tag tag-info">WoWs</a> <a href="/search/tag/WarThunder" class="tag tag-info">WarThunder</a> <a href="/search/tag/%E7%8A%AC" class="tag tag-info">犬</a> <a href="/search/tag/%EC%86%8C%ED%86%B5" class="tag tag-info">소통</a> <a href="/search/tag/%ED%9E%99%ED%95%A9" class="tag tag-info">힙합</a> </div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div id="navigation">
|
||||
<div id="navibox" class="tw-basic-page-right-column">
|
||||
<h2>Live Tags</h2>
|
||||
<div class="box">
|
||||
<p class="taglist">
|
||||
<a href="/search/tag/RTPDenVIVO" class="tag tag-info">RTPDenVIVO</a> <a href="/search/tag/Apex" class="tag tag-info">Apex</a> <a href="/search/tag/DBD" class="tag tag-info">DBD</a> <a href="/search/tag/PUBG" class="tag tag-info">PUBG</a> <a href="/search/tag/ApexLegends" class="tag tag-info">ApexLegends</a> <a href="/search/tag/YouTube" class="tag tag-info">YouTube</a> <a href="/search/tag/VALORANT" class="tag tag-info">VALORANT</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-info">歌</a> <a href="/search/tag/v%E7%B3%BB" class="tag tag-info">v系</a> <a href="/search/tag/AAA" class="tag tag-info">AAA</a> <a href="/search/tag/MW" class="tag tag-info">MW</a> <a href="/search/tag/WoWs" class="tag tag-info">WoWs</a> <a href="/search/tag/WarThunder" class="tag tag-info">WarThunder</a> <a href="/search/tag/%E7%8A%AC" class="tag tag-info">犬</a> <a href="/search/tag/%EC%86%8C%ED%86%B5" class="tag tag-info">소통</a> <a href="/search/tag/%ED%9E%99%ED%95%A9" class="tag tag-info">힙합</a> <a href="/search/tag/gdgd" class="tag tag-info">gdgd</a> </p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div id="ad_footer" style="padding:10px 1px;clear:left;width: auto;"><script type="text/javascript">
|
||||
window.addEventListener("DOMContentLoaded", function () {
|
||||
var tag = new Moiad.SingleAdTag();
|
||||
tag.setElementId("ad_footer");
|
||||
tag.setPlaceName("tcx_adfooter_pc_300x250");
|
||||
adLoader.activate(tag);
|
||||
});
|
||||
</script></div><div style="clear:both"></div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<footer class="tw-footer">
|
||||
<div class="tw-footer-container">
|
||||
<section class="tw-footer-application-area">
|
||||
<div class="tw-footer-column">
|
||||
<h3 class="tw-footer-column-title">Viewer App</h3>
|
||||
<div class="tw-footer-application">
|
||||
<div class="tw-footer-application-icon"><img src="/img/top/icon_application01.png" alt="Viewer App"></div>
|
||||
<ul class="tw-footer-application-list">
|
||||
<li class="tw-footer-application-list-item"><a href="https://itunes.apple.com/app/twitcasting-viewer/id365235835?mt=8"><img
|
||||
src="/img/badge_app_store.png" alt="App Store"></a></li>
|
||||
<li class="tw-footer-application-list-item"><a href="https://play.google.com/store/apps/details?id=com.sidefeed.TCViewer"><img
|
||||
src="/img/badge_android_market.png" alt="Google Play"></a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-footer-column">
|
||||
<h3 class="tw-footer-column-title">Live App</h3>
|
||||
<div class="tw-footer-application">
|
||||
<div class="tw-footer-application-icon"><img src="/img/top/icon_application02.png" alt="Live App"></div>
|
||||
<ul class="tw-footer-application-list">
|
||||
<li class="tw-footer-application-list-item"><a href="https://itunes.apple.com/app/twitcasting-live/id352756035?mt=8"><img
|
||||
src="/img/badge_app_store.png" alt="App Store"></a></li>
|
||||
<li class="tw-footer-application-list-item"><a href="https://play.google.com/store/apps/details?id=com.sidefeed.TCLive"><img
|
||||
src="/img/badge_android_market.png" alt="Google Play"></a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tw-footer-column">
|
||||
<h3 class="tw-footer-column-title">TwitCasting Games</h3>
|
||||
<div class="tw-footer-application">
|
||||
<div class="tw-footer-application-icon"><img src="/img/top/icon_application03.png" alt="TwitCasting Games"></div>
|
||||
<ul class="tw-footer-application-list">
|
||||
<li class="tw-footer-application-list-item"><a href="https://itunes.apple.com/app/twitcasting-live/id1444520376?mt=8"><img
|
||||
src="/img/badge_app_store.png" alt="App Store"></a></li>
|
||||
<li class="tw-footer-application-list-item"><a href="https://play.google.com/store/apps/details?id=com.sidefeed.screencas"><img
|
||||
src="/img/badge_android_market.png" alt="Google Play"></a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section class="tw-footer-navi-area">
|
||||
<div class="tw-footer-column">
|
||||
<div class="tw-footer-row">
|
||||
<h4 class="tw-footer-column-title"><label
|
||||
for="tw-footer-navi-visibility">About</label></h4>
|
||||
<input class="tw-footer-column-visibility" id="tw-footer-navi-visibility" type="checkbox"
|
||||
checked="checked">
|
||||
<ul class="tw-footer-navi">
|
||||
<li><a href="/indexprivacy.php">Privacy Policy</a></li>
|
||||
<li><a href="/indexlicense.php">Terms of Service</a></li>
|
||||
<li><a href="/indexattribution.php">Attribution</a></li>
|
||||
<li><a href="/indexapi.php">Developer API</a></li>
|
||||
</ul>
|
||||
<span class="tw-global-footer-icon-down"><svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M.531 8.035c-.708-.708-.708-1.911 0-2.62l.39-.388c.707-.708 1.91-.708 2.619 0l8.46 8.46 8.46-8.46c.708-.708 1.912-.708 2.62 0l.389.389c.708.708.708 1.911 0 2.62L12 19.503.531 8.035z" fill="#000" fill-rule="evenodd"/></svg>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="tw-footer-column">
|
||||
<h4 class="tw-footer-column-title"><label for="tw-footer-language-visibility">Country</label></h4>
|
||||
<input class="tw-footer-column-visibility" id="tw-footer-language-visibility" type="checkbox"
|
||||
checked="checked">
|
||||
<ul class="tw-footer-navi tw-footer-lang-list" role="listbox">
|
||||
<li>
|
||||
<a href="http://ja.twitcasting.tv/search/tag/vocaloid?&hl=ja" role="option" aria-selected="false">
|
||||
<img class="tw-global-header-lang-icon" src="/img/country/jp_icon.png" alt="">
|
||||
日本語 </a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="http://es.twitcasting.tv/search/tag/vocaloid?&hl=es" role="option" aria-selected="false">
|
||||
<img class="tw-global-header-lang-icon" src="/img/country/mx_icon.png" alt="">
|
||||
Español </a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="http://pt.twitcasting.tv/search/tag/vocaloid?&hl=pt" role="option" aria-selected="false">
|
||||
<img class="tw-global-header-lang-icon" src="/img/country/br_icon.png" alt="">
|
||||
Português </a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="http://en.twitcasting.tv/search/tag/vocaloid?&hl=en" role="option" aria-selected="true">
|
||||
<img class="tw-global-header-lang-icon" src="/img/country/us_icon.png" alt="">
|
||||
English </a>
|
||||
</li>
|
||||
</ul>
|
||||
<span class="tw-global-footer-icon-down"><svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M.531 8.035c-.708-.708-.708-1.911 0-2.62l.39-.388c.707-.708 1.91-.708 2.619 0l8.46 8.46 8.46-8.46c.708-.708 1.912-.708 2.62 0l.389.389c.708.708.708 1.911 0 2.62L12 19.503.531 8.035z" fill="#000" fill-rule="evenodd"/></svg>
|
||||
</span>
|
||||
</div>
|
||||
</section>
|
||||
<section class="tw-footer-copy-area">
|
||||
Copyright (c) 2009-2020 <a href="https://about.moi.st/">Moi Corp.</a> (Moi Labs inc.)
|
||||
</section>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
<!-- Facebook Pixel Code -->
|
||||
<script>
|
||||
!function(f,b,e,v,n,t,s)
|
||||
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
|
||||
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
|
||||
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
|
||||
n.queue=[];t=b.createElement(e);t.async=!0;
|
||||
t.src=v;s=b.getElementsByTagName(e)[0];
|
||||
s.parentNode.insertBefore(t,s)}(window, document,'script',
|
||||
'https://connect.facebook.net/en_US/fbevents.js');
|
||||
fbq('init', '264399894717032');
|
||||
fbq('track', 'PageView');
|
||||
</script>
|
||||
<noscript><img height="1" width="1" style="display:none"
|
||||
src="https://www.facebook.com/tr?id=264399894717032&ev=PageView&noscript=1"/></noscript>
|
||||
<!-- End Facebook Pixel Code -->
|
||||
|
||||
<!-- Twitter universal website tag code -->
|
||||
<script>
|
||||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||||
// Insert Twitter Pixel ID and Standard Event data below
|
||||
twq('init','o3l79');
|
||||
twq('track','PageView');
|
||||
</script>
|
||||
<!-- End Twitter universal website tag code -->
|
||||
|
||||
<!-- Global site tag (gtag.js) - Google Ads: 649663333 -->
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=AW-649663333"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'AW-649663333');
|
||||
</script>
|
||||
|
||||
<!-- LINE Tag Base Code -->
|
||||
<!-- Do Not Modify -->
|
||||
<script>
|
||||
(function(g,d,o){
|
||||
g._ltq=g._ltq||[];g._lt=g._lt||function(){g._ltq.push(arguments)};
|
||||
var h=location.protocol==='https:'?'https://d.line-scdn.net':'http://d.line-cdn.net';
|
||||
var s=d.createElement('script');s.async=1;
|
||||
s.src=o||h+'/n/line_tag/public/release/v1/lt.js';
|
||||
var t=d.getElementsByTagName('script')[0];t.parentNode.insertBefore(s,t);
|
||||
})(window, document);
|
||||
_lt('init', {
|
||||
customerType: 'lap',
|
||||
tagId: 'c0619881-5f46-411d-b23a-a077e3506417'
|
||||
});
|
||||
_lt('send', 'pv', ['c0619881-5f46-411d-b23a-a077e3506417']);
|
||||
</script>
|
||||
<noscript>
|
||||
<img height="1" width="1" style="display:none"
|
||||
src="https://tr.line.me/tag.gif?c_t=lap&t_id=c0619881-5f46-411d-b23a-a077e3506417&e=pv&noscript=1" />
|
||||
</noscript>
|
||||
<!-- End LINE Tag Base Code -->
|
||||
<script async src="https://s.yimg.jp/images/listing/tool/cv/ytag.js"></script>
|
||||
<script>
|
||||
window.yjDataLayer = window.yjDataLayer || [];
|
||||
function ytag() { yjDataLayer.push(arguments); }
|
||||
ytag({"type":"ycl_cookie"});
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,5 +1 @@
|
|||
package main
|
||||
|
||||
func main() {
|
||||
Execute()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,108 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"intimate"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/extractor"
|
||||
"github.com/474420502/focus/compare"
|
||||
"github.com/474420502/focus/tree/heap"
|
||||
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/474420502/requests"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) {
|
||||
rawurl := "https://twitcasting.tv/你好"
|
||||
u, _ := url.Parse(rawurl)
|
||||
t.Error(u.EscapedPath())
|
||||
t.Error(u.String())
|
||||
}
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitcasting))
|
||||
|
||||
func TestUpdateTime(t *testing.T) {
|
||||
// streamer := &intimate.Streamer{}
|
||||
// streamer.Uid = 420153
|
||||
// streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
|
||||
// estore.Update(streamer, "update_time", streamer.UpdateTime)
|
||||
}
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
main()
|
||||
|
||||
searchurl := "https://twitcasting.tv/rankingindex.php"
|
||||
queuedict := make(map[string]bool)
|
||||
queue := heap.New(compare.String)
|
||||
queue.Put(searchurl)
|
||||
queuedict[searchurl] = true
|
||||
|
||||
for surl, ok := queue.Pop(); ok; surl, ok = queue.Pop() {
|
||||
|
||||
ses := requests.NewSession()
|
||||
resp, err := ses.Get(surl.(string)).Execute()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
etor := extractor.ExtractXml(resp.Content())
|
||||
|
||||
// doc, err := libxml2.ParseHTML(resp.Content())
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
// defer doc.Free()
|
||||
|
||||
result, err := etor.XPath("//*[contains(@class, 'tag')]/@href")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// result, err := doc.Find("//*[contains(@class, 'tag')]/@href")
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
// defer result.Free()
|
||||
|
||||
iter := result.NodeIter()
|
||||
for iter.Next() {
|
||||
|
||||
wurl := "https://twitcasting.tv" + iter.Node().NodeValue()
|
||||
if ok := queuedict[wurl]; !ok {
|
||||
log.Println(wurl)
|
||||
sl := &intimate.StreamerList{}
|
||||
sl.Platform = intimate.Ptwitcasting
|
||||
sl.Url = wurl
|
||||
sl.Operator = 0
|
||||
sl.UpdateInterval = 120
|
||||
sl.UpdateTime = time.Now()
|
||||
estore.InsertStreamerList(sl)
|
||||
queue.Put(wurl)
|
||||
queuedict[wurl] = true
|
||||
}
|
||||
}
|
||||
|
||||
// doc.Find("//div[@class='tw-search-result-row']")
|
||||
xps, err := etor.XPaths("//div[@class='tw-search-result-row']")
|
||||
if err != nil {
|
||||
log.Println(surl, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// xps.ForEachTag(SearchProfile{})
|
||||
|
||||
// texts, errs := xps.ForEachText(".//span[@class='username']")
|
||||
// if len(errs) > 0 {
|
||||
// t.Error(errs)
|
||||
// }
|
||||
var splist = xps.ForEachTag(SearchProfile{})
|
||||
for _, isp := range splist {
|
||||
sp := isp.(*SearchProfile)
|
||||
sp.UserId = sp.LiveUrl[1:]
|
||||
// log.Println(sp.(SearchProfile))
|
||||
}
|
||||
|
||||
for _, isp := range splist {
|
||||
log.Println(isp.(*SearchProfile))
|
||||
}
|
||||
|
||||
log.Println("finish remain", queue.Size())
|
||||
}
|
||||
}
|
||||
|
||||
type SearchProfile struct {
|
||||
UserName string `exp:".//span[@class='username']" method:"Text"`
|
||||
UserId string // `exp:".//span[@class='fullname']" method:"Text"`
|
||||
LiveUrl string `exp:".//div[@class='usertext']/a[@href]" method:"Attribute,href Value"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,145 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/extractor"
|
||||
"github.com/474420502/focus/compare"
|
||||
"github.com/474420502/focus/tree/heap"
|
||||
"github.com/474420502/requests"
|
||||
)
|
||||
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitcasting))
|
||||
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
type SearchProfile struct {
|
||||
UserName string `exp:".//span[@class='username']" method:"Text"`
|
||||
UserId string // `exp:".//span[@class='fullname']" method:"Text"`
|
||||
LiveUrl string `exp:".//div[@class='usertext']/a[@href]" method:"AttributeValue,href"`
|
||||
Tag []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"Text"`
|
||||
TagUrl []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"AttributeValue,href"`
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
homeurl := "https://twitcasting.tv"
|
||||
searchurl := "https://twitcasting.tv/rankingindex.php"
|
||||
queuedict := make(map[string]bool)
|
||||
queue := heap.New(compare.String)
|
||||
queue.Put(searchurl)
|
||||
queuedict[searchurl] = true
|
||||
ses := requests.NewSession()
|
||||
ses.Config().SetTimeout(15)
|
||||
|
||||
var surl interface{}
|
||||
var ok bool
|
||||
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
|
||||
for surl, ok = queue.Pop(); ok && !ps.IsClose(); surl, ok = queue.Pop() {
|
||||
u, err := url.Parse(surl.(string))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := ses.Get(u.String()).Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
log.Println(u.String(), surl)
|
||||
continue
|
||||
// log.Panic(err)
|
||||
}
|
||||
|
||||
etor := extractor.ExtractHtml(resp.Content())
|
||||
result, err := etor.XPaths("//p[@class='taglist']/a[contains(@class, 'tag')]/@href")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, href := range result.GetTexts() {
|
||||
|
||||
wurl := homeurl + href
|
||||
if ok := queuedict[wurl]; !ok {
|
||||
log.Println(wurl)
|
||||
sl := &intimate.StreamerList{}
|
||||
sl.Platform = string(intimate.Ptwitcasting)
|
||||
sl.Url = wurl
|
||||
sl.Operator = 0
|
||||
sl.UpdateInterval = 120
|
||||
sl.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
sl.UrlHash = intimate.GetUrlHash(sl.Url)
|
||||
|
||||
intimate.TStreamerList.Insert(sl)
|
||||
// estore.InsertStreamerList(sl)
|
||||
|
||||
queue.Put(wurl)
|
||||
queuedict[wurl] = true
|
||||
}
|
||||
}
|
||||
|
||||
xps, err := etor.XPaths("//div[@class='tw-search-result-row']")
|
||||
if err != nil {
|
||||
log.Println(surl, err)
|
||||
continue
|
||||
}
|
||||
|
||||
var splist = xps.ForEachObjectByTag(SearchProfile{})
|
||||
for _, isp := range splist {
|
||||
sp := isp.(*SearchProfile)
|
||||
if sp.LiveUrl == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
sp.UserId = sp.LiveUrl[1:]
|
||||
for i := 0; i < len(sp.TagUrl); i++ {
|
||||
wurl := homeurl + sp.TagUrl[i]
|
||||
sp.TagUrl[i] = wurl
|
||||
if ok := queuedict[wurl]; !ok {
|
||||
sl := &intimate.StreamerList{}
|
||||
sl.Platform = string(intimate.Ptwitcasting)
|
||||
sl.Url = wurl
|
||||
sl.Operator = 0
|
||||
sl.UpdateInterval = 120
|
||||
sl.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
|
||||
sl.UrlHash = intimate.GetUrlHash(sl.Url)
|
||||
intimate.TStreamerList.Insert(sl)
|
||||
|
||||
queue.Put(wurl)
|
||||
queuedict[wurl] = true
|
||||
}
|
||||
}
|
||||
// log.Println(sp.(SearchProfile))
|
||||
}
|
||||
|
||||
log.Println("find user:", len(splist))
|
||||
for _, isp := range splist {
|
||||
sp := isp.(*SearchProfile)
|
||||
// log.Println(sp)
|
||||
streamer := &intimate.Streamer{}
|
||||
streamer.Platform = intimate.Ptwitcasting
|
||||
streamer.LiveUrl = &sql.NullString{String: sp.LiveUrl, Valid: true}
|
||||
if btags, err := json.Marshal(sp.Tag); err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
streamer.Tags = btags
|
||||
}
|
||||
streamer.UpdateInterval = 120
|
||||
streamer.UpdateTime = intimate.GetUpdateTimeNow()
|
||||
streamer.UserName = &sql.NullString{String: sp.UserName, Valid: true}
|
||||
streamer.UserId = &sp.UserId
|
||||
streamer.Operator = 0
|
||||
// estore.InsertStreamer(streamer)
|
||||
intimate.TStreamer.Insert(streamer)
|
||||
}
|
||||
|
||||
log.Println("finish remain", queue.Size())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
package main
|
||||
|
||||
func main() {
|
||||
Execute()
|
||||
e := ChannelLink{}
|
||||
e.Execute()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"intimate"
|
||||
"log"
|
||||
"time"
|
||||
|
|
@ -8,113 +9,104 @@ import (
|
|||
"github.com/tebeka/selenium"
|
||||
)
|
||||
|
||||
// // sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
// var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
|
||||
|
||||
// // estore 解析存储连接实例
|
||||
// var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
// 获取类型的所有频道链接
|
||||
|
||||
// ChannelLink 频道链接
|
||||
type ChannelLink struct {
|
||||
}
|
||||
|
||||
// Execute 执行任务
|
||||
func Execute() {
|
||||
|
||||
func (cl *ChannelLink) Execute() {
|
||||
var err error
|
||||
wd := intimate.GetChromeDriver(3030)
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
var adriver *intimate.AutoCloseDriver
|
||||
|
||||
for !ps.IsClose() {
|
||||
|
||||
var err error
|
||||
adriver = intimate.GetChromeDriver()
|
||||
wd := adriver.Webdriver
|
||||
|
||||
weburl := "https://www.twitch.tv/directory?sort=VIEWER_COUNT"
|
||||
err = wd.Get(weburl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cardCondition := func(wd selenium.WebDriver) (bool, error) {
|
||||
elements, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return len(elements) > 0, nil
|
||||
}
|
||||
wd.WaitWithTimeout(cardCondition, time.Second*15)
|
||||
time.Sleep(time.Second)
|
||||
|
||||
e, err := wd.FindElement(selenium.ByXPATH, "//button[@data-a-target='browse-sort-menu']")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
e.Click()
|
||||
|
||||
var lasthreflen = 0
|
||||
var hrefs map[string]bool = make(map[string]bool)
|
||||
var delayerror = 5
|
||||
for i := 0; i <= 200; i++ {
|
||||
cards, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
|
||||
if len(hrefs) == lasthreflen {
|
||||
delayerror--
|
||||
if delayerror <= 0 {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
delayerror = 7
|
||||
}
|
||||
lasthreflen = len(hrefs)
|
||||
|
||||
for ii := 0; ii < 10; ii++ {
|
||||
for _, card := range cards {
|
||||
href, err := card.GetAttribute("href")
|
||||
if err != nil {
|
||||
log.Println(href, err)
|
||||
continue
|
||||
} else {
|
||||
hrefs[href] = true
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if ps.IsClose() {
|
||||
break
|
||||
}
|
||||
|
||||
if len(cards) > 10 {
|
||||
log.Println(len(cards))
|
||||
wd.ExecuteScript(`items = document.evaluate("//div[@data-target='directory-page__card-container']/../self::div[@data-target and @style]", document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
||||
for (var i = 0; i < items.snapshotLength - 10; i++) { item = items.snapshotItem(i); item.remove() ;};`, nil)
|
||||
}
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyDown(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyUp(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 2500)
|
||||
}
|
||||
|
||||
for href := range hrefs {
|
||||
|
||||
sl := &intimate.StreamerList{}
|
||||
sl.Url = href
|
||||
sl.UrlHash = intimate.GetUrlHash(sl.Url)
|
||||
sl.Platform = string(intimate.Ptwitch)
|
||||
sl.UpdateTime = intimate.GetUpdateTimeNow()
|
||||
err := intimate.TStreamerList.Insert(sl)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("hrefs len:", len(hrefs))
|
||||
adriver.Close()
|
||||
ps.Wait(time.Minute * 5)
|
||||
weburl := "https://www.twitch.tv/directory?sort=VIEWER_COUNT"
|
||||
err = wd.Get(weburl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cardCondition := func(wd selenium.WebDriver) (bool, error) {
|
||||
elements, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return len(elements) > 0, nil
|
||||
}
|
||||
wd.WaitWithTimeout(cardCondition, time.Second*15)
|
||||
time.Sleep(time.Second)
|
||||
|
||||
e, err := wd.FindElement(selenium.ByXPATH, "//button[@data-a-target='browse-sort-menu']")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
e.Click()
|
||||
|
||||
var hrefs map[string]bool = make(map[string]bool)
|
||||
var delayerror = 5
|
||||
for i := 0; i <= 200; i++ {
|
||||
cards, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
|
||||
if len(hrefs) == 0 {
|
||||
delayerror--
|
||||
if delayerror <= 0 {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
delayerror = 5
|
||||
}
|
||||
|
||||
for ii := 0; ii < 10; ii++ {
|
||||
for _, card := range cards {
|
||||
href, err := card.GetAttribute("href")
|
||||
if err != nil {
|
||||
log.Println(href, err)
|
||||
continue
|
||||
} else {
|
||||
hrefs[href] = true
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if ps.IsClose() {
|
||||
break
|
||||
}
|
||||
|
||||
if len(cards) > 10 {
|
||||
log.Println(len(cards))
|
||||
wd.ExecuteScript(`items = document.evaluate("//div[@data-target='directory-page__card-container']/../self::div[@data-target and @style]", document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
||||
for (var i = 0; i < items.snapshotLength - 10; i++) { item = items.snapshotItem(i); item.remove() ;};`, nil)
|
||||
}
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyDown(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyUp(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 2500)
|
||||
}
|
||||
|
||||
for href := range hrefs {
|
||||
|
||||
// TODO: Save href
|
||||
source := &intimate.Source{}
|
||||
source.Source = sql.NullString{String: href, Valid: true}
|
||||
source.Operator = 0
|
||||
source.Target = intimate.TTwitchChannel
|
||||
source.Url = weburl
|
||||
sstore.Insert(source)
|
||||
}
|
||||
|
||||
log.Println("hrefs len:", len(hrefs))
|
||||
sstore.Deduplicate(intimate.TTwitchChannel, "source")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ import (
|
|||
)
|
||||
|
||||
func TestCase1(t *testing.T) {
|
||||
Execute()
|
||||
e := ChannelLink{}
|
||||
e.Execute()
|
||||
}
|
||||
|
||||
func TestLiveUrl(t *testing.T) {
|
||||
|
|
|
|||
2
tasks/twitch/twitch_task2/.gitignore
vendored
Normal file
2
tasks/twitch/twitch_task2/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
twitch_task2
|
||||
log
|
||||
6
tasks/twitch/twitch_task2/main.go
Normal file
6
tasks/twitch/twitch_task2/main.go
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
package main
|
||||
|
||||
func main() {
|
||||
ul := UserList{}
|
||||
ul.Execute()
|
||||
}
|
||||
180
tasks/twitch/twitch_task2/task_twitch.go
Normal file
180
tasks/twitch/twitch_task2/task_twitch.go
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/tebeka/selenium"
|
||||
)
|
||||
|
||||
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
|
||||
|
||||
// estore 解析存储连接实例
|
||||
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
|
||||
|
||||
// 获取类型的所有频道链接
|
||||
|
||||
// UserList 频道链接
|
||||
type UserList struct {
|
||||
}
|
||||
|
||||
// Execute 执行任务
|
||||
func (cl *UserList) Execute() {
|
||||
// DELETE FROM source_twitch WHERE uid NOT IN (SELECT MAX(s.uid) FROM (SELECT uid, source FROM source_twitch ) s GROUP BY s.source) ;
|
||||
//article//a[@data-a-target='preview-card-title-link']
|
||||
|
||||
wd := intimate.GetChromeDriver(3030)
|
||||
defer wd.Close()
|
||||
defer wd.Quit()
|
||||
ps := intimate.NewPerfectShutdown()
|
||||
counter := intimate.NewCounter()
|
||||
counter.SetMaxLimit(100)
|
||||
counter.SetMaxToDo(func(olist ...interface{}) error {
|
||||
owd := olist[0].(*selenium.WebDriver)
|
||||
if err := (*owd).Quit(); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
*owd = intimate.GetChromeDriver(3030)
|
||||
return nil
|
||||
}, &wd)
|
||||
|
||||
for !ps.IsClose() {
|
||||
|
||||
var err error
|
||||
sourceChannel, err := sstore.Pop(intimate.TTwitchChannel)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
weburl := sourceChannel.Source.String + "?sort=VIEWER_COUNT"
|
||||
err = wd.Get(weburl)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
sstore.UpdateError(sourceChannel, err)
|
||||
time.Sleep(time.Second * 10)
|
||||
continue
|
||||
}
|
||||
|
||||
wd.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) {
|
||||
_, err := wd.FindElement(selenium.ByXPATH, "(//div/p[@class=''])[last()]")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}, time.Second*10)
|
||||
|
||||
btn, err := wd.FindElement(selenium.ByXPATH, "//button[@data-a-target='browse-sort-menu']")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
btn.Click()
|
||||
|
||||
var elements []selenium.WebElement
|
||||
var liveurls = 0
|
||||
var delayerror = 2
|
||||
for i := 0; i < 200 && !ps.IsClose(); i++ {
|
||||
elements, err = wd.FindElements(selenium.ByXPATH, "(//div/p[@class=''])[last()]")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyDown(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
wd.KeyUp(selenium.EndKey)
|
||||
time.Sleep(time.Millisecond * 2000)
|
||||
if len(elements) == liveurls {
|
||||
delayerror--
|
||||
if delayerror <= 0 {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
delayerror = 2
|
||||
}
|
||||
liveurls = len(elements)
|
||||
}
|
||||
articles, err := wd.FindElements(selenium.ByXPATH, "//article")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, article := range articles {
|
||||
|
||||
e, err := article.FindElement(selenium.ByXPATH, ".//a[@data-a-target='preview-card-title-link' and @href]")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
href, err := e.GetAttribute("href")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
btns, err := article.FindElements(selenium.ByXPATH, ".//div[@class='tw-full-width tw-inline-block']//button")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
var tags []string
|
||||
for _, btn := range btns {
|
||||
tag, err := btn.GetAttribute("data-a-target")
|
||||
if err == nil {
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
}
|
||||
|
||||
streamer := &intimate.Streamer{}
|
||||
|
||||
matches := regexp.MustCompile(`https://www.twitch.tv/(\w+)`).FindStringSubmatch(href)
|
||||
if len(matches) == 2 {
|
||||
streamer.UserId = matches[1]
|
||||
} else {
|
||||
log.Println(href)
|
||||
continue
|
||||
}
|
||||
|
||||
jtags, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
streamer.Tags = jtags
|
||||
}
|
||||
|
||||
streamer.Platform = intimate.Ptwitch
|
||||
|
||||
updateUrl := make(map[string]string)
|
||||
updateUrl["live"] = href
|
||||
streamer.LiveUrl = sql.NullString{String: href, Valid: true}
|
||||
data, err := json.Marshal(updateUrl)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
streamer.UpdateUrl = data
|
||||
streamer.Operator = 0
|
||||
if estore.InsertStreamer(streamer) {
|
||||
// log.Println("streamer update tags", streamer.Uid, tags)
|
||||
estore.Update(streamer, "Tags", streamer.Tags)
|
||||
}
|
||||
}
|
||||
log.Println("streamer find", len(articles))
|
||||
if len(articles) == 0 {
|
||||
sourceChannel.Operator = 5
|
||||
sstore.UpdateOperator(sourceChannel)
|
||||
}
|
||||
counter.AddWithReset(1)
|
||||
}
|
||||
|
||||
wd.Close()
|
||||
wd.Quit()
|
||||
}
|
||||
|
|
@ -1,8 +1,6 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
import "testing"
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
10205
testfile/nimo1.json
10205
testfile/nimo1.json
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
154
utils.go
154
utils.go
|
|
@ -1,14 +1,11 @@
|
|||
package intimate
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
|
@ -31,42 +28,10 @@ func init() {
|
|||
|
||||
}
|
||||
|
||||
// GetUpdateTimeNow 获取马上更新时间. 与第一次连用
|
||||
func GetUpdateTimeNow() *sql.NullTime {
|
||||
return &sql.NullTime{Time: time.Now().Add(-time.Hour * 100000), Valid: true}
|
||||
}
|
||||
|
||||
func GetUrlHash(urlstr string) string {
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(urlstr)))
|
||||
}
|
||||
|
||||
// ParseNumber 去逗号解析数字
|
||||
func ParseNumber(num string) (int64, error) {
|
||||
num = strings.Trim(num, " ")
|
||||
num = strings.ReplaceAll(num, ",", "")
|
||||
return strconv.ParseInt(num, 10, 64)
|
||||
}
|
||||
|
||||
// ParseNumberEx 解析带字符的数字
|
||||
func ParseNumberEx(num string) (float64, error) {
|
||||
num = strings.Trim(num, " ")
|
||||
num = strings.ReplaceAll(num, ",", "")
|
||||
last := num[len(num)-1]
|
||||
factor := 1.0
|
||||
switch {
|
||||
case last == 'k' || last == 'K':
|
||||
factor = 1000.0
|
||||
num = num[0 : len(num)-1]
|
||||
case last == 'm' || last == 'M':
|
||||
factor = 1000000.0
|
||||
num = num[0 : len(num)-1]
|
||||
}
|
||||
i, err := strconv.ParseFloat(num, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return i * factor, nil
|
||||
func ParseNumber(number string) (int64, error) {
|
||||
number = strings.ReplaceAll(number, ",", "")
|
||||
return strconv.ParseInt(number, 10, 64)
|
||||
}
|
||||
|
||||
// ParseDuration time to duration eg: 1:40:00 -> time.Duration
|
||||
|
|
@ -94,39 +59,7 @@ func ParseDuration(dt string) (time.Duration, error) {
|
|||
return tdt.Sub(zeroTime), nil
|
||||
}
|
||||
|
||||
type AutoCloseDriver struct {
|
||||
Webdriver selenium.WebDriver
|
||||
Port int
|
||||
}
|
||||
|
||||
func (adriver *AutoCloseDriver) Close() {
|
||||
|
||||
data, err := exec.Command("/bin/bash", "-c", fmt.Sprintf(`pgrep -f "port=%d"`, adriver.Port)).Output()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
log.Println(string(data))
|
||||
return
|
||||
}
|
||||
// log.Println(string(data))
|
||||
|
||||
killshell := fmt.Sprintf("pkill -9 -P %s", data)
|
||||
err = exec.Command("/bin/bash", "-c", killshell).Run()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
err = exec.Command("/bin/bash", "-c", fmt.Sprintf("kill -9 %s", data)).Run()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func GetChromeDriver() *AutoCloseDriver {
|
||||
|
||||
port := GetFreePort()
|
||||
|
||||
func GetChromeDriver(port int) selenium.WebDriver {
|
||||
var err error
|
||||
caps := selenium.Capabilities{"browserName": "chrome"}
|
||||
|
||||
|
|
@ -134,7 +67,6 @@ func GetChromeDriver() *AutoCloseDriver {
|
|||
for _, epath := range []string{"../../../crx/myblock.crx", "../../crx/myblock.crx"} {
|
||||
_, err := os.Stat(epath)
|
||||
if err == nil {
|
||||
|
||||
err := chromecaps.AddExtension(epath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -162,27 +94,28 @@ func GetChromeDriver() *AutoCloseDriver {
|
|||
|
||||
chromecaps.ExcludeSwitches = append(chromecaps.ExcludeSwitches, "enable-automation")
|
||||
caps.AddChrome(chromecaps)
|
||||
|
||||
_, err = selenium.NewChromeDriverService("/usr/bin/chromedriver", port)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
wd, err := selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", port))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
adriver := &AutoCloseDriver{}
|
||||
adriver.Port = port
|
||||
adriver.Webdriver = wd
|
||||
|
||||
runtime.SetFinalizer(wd, func(obj interface{}) {
|
||||
if err := obj.(selenium.WebDriver).Close(); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if err := obj.(selenium.WebDriver).Quit(); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
})
|
||||
wd.ExecuteScript("windows.navigator.webdriver = undefined", nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return adriver
|
||||
return wd
|
||||
}
|
||||
|
||||
// PerfectShutdown 完美关闭程序
|
||||
|
|
@ -210,18 +143,6 @@ func (ps *PerfectShutdown) IsClose() bool {
|
|||
return atomic.LoadInt32(&ps.loop) == 0
|
||||
}
|
||||
|
||||
// Wait 判断是否要关闭
|
||||
func (ps *PerfectShutdown) Wait(tm time.Duration) bool {
|
||||
now := time.Now()
|
||||
for time.Now().Sub(now) <= tm {
|
||||
if ps.IsClose() {
|
||||
return false
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type Counter struct {
|
||||
dcount int
|
||||
count int
|
||||
|
|
@ -314,50 +235,3 @@ func (c *Counter) Add(n int) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type WaitFor struct {
|
||||
WebDriver selenium.WebDriver
|
||||
}
|
||||
|
||||
func NewWaitFor(wd selenium.WebDriver) *WaitFor {
|
||||
return &WaitFor{WebDriver: wd}
|
||||
}
|
||||
|
||||
func (wf *WaitFor) Default(xpath string, do func(elements ...selenium.WebElement) bool) error {
|
||||
return wf.WaitWithTimeout(xpath, 15*time.Second, do)
|
||||
}
|
||||
|
||||
func (wf *WaitFor) WaitWithTimeout(xpath string, timeout time.Duration, do func(elements ...selenium.WebElement) bool) error {
|
||||
return wf.WebDriver.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) {
|
||||
elements, err := wd.FindElements(selenium.ByXPATH, xpath)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(elements) > 0 {
|
||||
if do == nil {
|
||||
return true, nil
|
||||
}
|
||||
if do(elements...) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
|
||||
}, timeout)
|
||||
}
|
||||
|
||||
func GetFreePort() int {
|
||||
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
l, err := net.ListenTCP("tcp", addr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer l.Close()
|
||||
return l.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user