test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mirrativ finish 90%

Browse Source
This commit is contained in:
eson 2020-09-16 18:53:26 +08:00
parent ee28db4ad8
commit fb3b09fc38
6 changed files with 320 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
autostore.go
View File

@ -145,7 +145,7 @@ func (queue *Queue) Pop() (result interface{}, err error) {
rows, err := tx.Query(selectsql, queue.cond.CondArgs...)
if err != nil {
return nil, fmt.Errorf("table: %s queue is empty", queue.table.name)
return nil, fmt.Errorf("table: %s queue is empty. %s", queue.table.name, err.Error())
}
var fields = make([]interface{}, len(queue.fieldIndex))

116
extractor/mirrativ_extractor/mirrativ_extractor_test.go
View File

@ -1,22 +1,39 @@
package main
import (
"database/sql"
"encoding/json"
"intimate"
"log"
"testing"
"time"
"github.com/474420502/gcurl"
"github.com/tidwall/gjson"
)
func main() {
sessionstr := `
-H 'authority: www.mirrativ.com'
-H 'accept: application/json'
-H 'x-timezone: Asia/Shanghai'
-H 'x-csrf-token: F3Ojd6RBtApP6YAZzVn-9jWN1of159VxAqOQL1Zn'
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
-H 'content-type: application/json'
-H 'sec-fetch-site: same-origin'
-H 'sec-fetch-mode: cors'
-H 'sec-fetch-dest: empty'
-H 'referer: https://www.mirrativ.com/live/O5Ia4iX9c5CeZj7DFtg52Q'
-H 'accept-language: zh-CN,zh;q=0.9,ja;q=0.8'
-H 'cookie: f=A2D75F0E-D218-11EA-A042-452BF6D21CE8; _ga=GA1.2.689947597.1596081392; mr_id=kxb65LddGMZf5C28jkR_tGCZD_ZFOAepD5gfXO7eNjfPMB8EKYvU1Vg_Y29V1lsa; _gid=GA1.2.2116692650.1600139685; lang=ja'`
ps := intimate.NewPerfectShutdown()
gprofile := gcurl.Parse(`https://www.mirrativ.com/api/user/profile?user_id=103383701`)
gprofile := gcurl.Parse(`curl 'https://www.mirrativ.com/api/user/profile?user_id=103383701'` + sessionstr)
tpProfile := gprofile.CreateTemporary(nil)
tpProfileUserID := tpProfile.QueryParam("user_id")
g := gcurl.Parse(`https://www.mirrativ.com/api/live/live?live_id=O5Ia4iX9c5CeZj7DFtg52Q`)
g := gcurl.Parse(`culr 'https://www.mirrativ.com/api/live/live?live_id=O5Ia4iX9c5CeZj7DFtg52Q'` + sessionstr)
tpLive := g.CreateTemporary(nil)
tpLiveID := tpLive.QueryParam("live_id")
@ -28,9 +45,11 @@ func main() {
time.Sleep(time.Second * 2)
continue
}
now := &sql.NullTime{Time: time.Now(), Valid: true}
streamer := istreamer.(*intimate.Streamer)
streamer.UpdateTime = now
userid := *streamer.UserId
log.Println(userid)
tpProfileUserID.StringSet(userid)
resp, err := tpProfile.Execute()
@ -40,10 +59,99 @@ func main() {
continue
}
profilejson := gcurl.Parse(string(resp.Content()))
clog := &intimate.CollectLog{}
clog.Platform = intimate.PMirrativ
clog.UpdateTime = now
clog.UserId = userid
clog.StreamerUid = streamer.Uid
profilejson := gjson.ParseBytes(resp.Content())
if result := profilejson.Get("follower_num"); result.Exists() {
clog.Followers = &sql.NullInt64{Int64: result.Int(), Valid: true}
}
if result := profilejson.Get("onlive.live_id"); result.Exists() {
liveID := result.String()
tpLiveID.StringSet(liveID)
resp, err = tpLive.Execute()
if err != nil {
log.Println(err)
continue
}
livejson := gjson.ParseBytes(resp.Content())
if result := livejson.Get("total_viewer_num"); result.Exists() {
clog.Views = &sql.NullInt64{Int64: result.Int(), Valid: true}
}
if result := livejson.Get("title"); result.Exists() {
clog.LiveTitle = &sql.NullString{String: result.String(), Valid: true}
}
if result := livejson.Get("started_at"); result.Exists() {
clog.LiveStartTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true}
}
if result := livejson.Get("heartbeated_at"); result.Exists() {
clog.LiveEndTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true}
}
if result := livejson.Get("app_id"); result.Exists() {
streamer.Channel = &sql.NullString{String: result.String(), Valid: true}
}
if result := livejson.Get("timeline.#.app.short_title"); result.Exists() {
for _, tl := range result.Array() {
var tags []string = []string{tl.String()}
jtags, _ := json.Marshal(tags)
streamer.Tags = jtags
clog.Tags = jtags
break
}
} else {
log.Println(string(resp.Content()))
return
}
if result := livejson.Get("gift_ranking_url"); result.Exists() {
// streamer.Channel = &sql.NullString{String: result.String(), Valid: true}
gifturl := "curl '" + result.String() + "&type=monthly&cursor='" + sessionstr
ggift := gcurl.Parse(gifturl)
tp := ggift.CreateTemporary(nil)
tp.SetURLRawPath("/api/gift/ranking")
pcursor := tp.QueryParam("cursor")
var gratuity int64 = 0
for {
giftdata, err := tp.Execute()
giftjson := gjson.ParseBytes(giftdata.Content())
if err != nil {
log.Println(err)
} else {
for _, rpoint := range giftjson.Get("ranking.#.point").Array() {
gratuity += rpoint.Int()
}
}
ncursor := giftjson.Get("next_cursor").String()
if ncursor == "" {
break
}
pcursor.StringSet(ncursor)
}
// https://www.mirrativ.com/gift/ranking?live_id=O5Ia4iX9c5CeZj7DFtg52Q&obfuscated_user_id=PgIBEgc6jVc
clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true}
}
cid, err := intimate.TClog.InsertRetAutoID(clog)
if err != nil {
log.Println(err)
}
streamer.LatestLogUid = cid
}
intimate.TStreamer.Update(streamer)
time.Sleep(time.Second * 2)
}
}
func TestDo(t *testing.T) {

2
extractor_field.go
View File

@ -46,7 +46,7 @@ type Streamer struct {
Channel *sql.NullString `field:"channel"` //
Tags interface{} `field:"tags"`
Ext interface{} `field:"ext"` //
Comments interface{} `field:"comments"`
// Comments interface{} `field:"comments"`
IsUpdateStreamer bool // 更新上面的内容
IsUpdateUrl bool

7
tasks/mirrativ/mirrativ_task1/mirrativ_task1.go
View File

@ -57,14 +57,11 @@ func main() {
streamer.Platform = intimate.PMirrativ
streamer.Operator = 0
streamer.UserId = &guserid.Str
streamer.UserName = &sql.NullString{String: liveinfo.Get("name").String(), Valid: true}
streamer.UserName = &sql.NullString{String: owner.Get("name").String(), Valid: true}
streamer.UpdateInterval = 60
streamer.UpdateTime = intimate.GetUpdateTimeNow()
err = intimate.TStreamer.InsertOrUpdate(
streamer,
intimate.DUpdate{Field: "update_time"},
)
err = intimate.TStreamer.Insert(streamer)
if err != nil {
log.Println(err)
panic(err)

2
tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go
View File

@ -5,5 +5,5 @@ import (
)
func TestMain(t *testing.T) {
main()
}

201
testfile/mirrativ.json
View File

@ -2362,3 +2362,204 @@
"image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/9bfba1b91a486106882577717aab4ccb4113b200d07dfb7127b087e4e52d8869_m.jpeg?1600164645",
"orientation_v2": "6"
}
{
"obfuscated_user_id": "PgIBEgc6jVc",
"current_user_rank": null,
"ranking": [
{
"point": 10,
"user": {
"share_url": "https://www.mirrativ.com/user/105481237",
"profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/2d16160a4158c99daf92f287313a7b4df7797b9f1e07ae0f8983df42fbecec97_m.jpeg?1600205857",
"name": "wwwww",
"description": "",
"properties": [],
"badges": [],
"is_continuous_streamer": 0,
"is_new": 0,
"user_id": "105481237",
"onlive": null
}
},
{
"point": 1,
"user": {
"share_url": "https://www.mirrativ.com/user/103544585",
"profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/40cbd7501613cf08f45c56704b1a80796413d7ac0f9e06851d7a4ebe6f1fe7de_m.jpeg?1600217414",
"name": "ꗼ𝒓𝑦𝒖ꗼ",
"description": "俺のファンか仲良い人にしかギフトとかはあげません！まぁ俺にファンなんて居ないけど",
"properties": [],
"badges": [
{
"image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_master_holiday_L.en.png?v1",
"small_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_master_holiday_S.en.png?v1"
}
],
"is_continuous_streamer": 1,
"is_new": 0,
"catalog_label_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_status_master_holiday_en@3x.png?v5",
"user_id": "103544585",
"onlive": null
}
}
],
"status": {
"msg": "",
"ok": 1,
"error": "",
"captcha_url": "",
"error_code": 0,
"message": ""
},
"next_cursor": "",
"current_cursor": "bW9udGhseToxMDYwNjk5MTY6MjY2NzA0NzMvMA"
}
{
"streaming_url_hls": "http://hls-cdn22.mirrativ.com/liveedge/c4d32ce6edec9a37e7b20cff4b656a95714b5b9ee872964d9e435debe74a9518/playlist.m3u8",
"is_gift_supported": 1,
"live_id": "9LtbLw58uSrzig_NTxutIw",
"is_mirrorable": 0,
"description": "いつも拝聴ありがとうございます！\nクラッカーは配信が落ちてしまうので、禁止でお願い致しますm(_ _)m\n",
"total_viewer_num": 1127,
"thumbnail_image_url": "",
"is_archive": 0,
"is_singing_karaoke": 0,
"title": "雑談しながらゲームする！",
"max_online_viewer_num": 212,
"created_at": 1600194426,
"is_live": 1,
"started_at": 1600242640,
"blur_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_m.jpeg?blur=80&1600244617",
"preview_blur_image_url": "",
"live_mos": null,
"thumbnail_blur_image_url": "",
"image_url_without_letterbox": "",
"diamonds": 0,
"joined_live_thumbnail_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_m.jpeg?1600244617",
"template_comments": [],
"tags": [],
"broadcast_host": "online.mirrativ.com",
"live_user_key": "",
"bcsvr_key": "1abe77db:Cp3UZxJW",
"heartbeated_at": 1600244742,
"shares": {
"twitter": {
"maxlength": 84,
"card": {
"site": "mirrativ.com",
"image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/custom_thumbnail/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_share.jpeg?1600244617",
"title": "雑談しながらゲームする！",
"description": "🐉そあこ💖爽鱗🐉"
},
"text": "",
"placeholder": "この配信についてひとこと！"
},
"others": {
"text": "雑談しながらゲームする！ #プラエデ #ミラティブ #配信 | https://www.mirrativ.com/live/9LtbLw58uSrzig_NTxutIw"
},
"title": "🐉そあこ💖爽鱗🐉さんの配信を友達にシェアしよう！",
"description": "友達が見に来てくれると配信者さんに伝わるよ！"
},
"is_private": 0,
"collab_supported": 1,
"sticker_enabled": 0,
"collab_has_vacancy": 0,
"streaming_key": "c4d32ce6edec9a37e7b20cff4b656a95714b5b9ee872964d9e435debe74a9518",
"stamp_num": 0,
"linked_live": {
"owner_badge_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_S.ja.png?v6",
"live_id": "-L6kxuJm_HgJN7aBY5JjYg",
"owner_id": "100895240",
"owner_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/2a6dab73acec21af7d4d08f53f854e4c3a429d0ba73cd3c8f7618f90ff3c4f9b_m.jpeg?1600184061",
"owner_name": "🦔NICO*🦥🥈感謝"
},
"collab_online_user_num": 0,
"share_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/custom_thumbnail/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_share.jpeg?1600244617",
"broadcast_key": "1abe77db:Cp3UZxJW",
"gift_ranking_url": "https://www.mirrativ.com/gift/ranking?live_id=9LtbLw58uSrzig_NTxutIw&obfuscated_user_id=PgIBEgc6jVc",
"collab_mos": null,
"archive_url_hls": "",
"remaining_coins": 0,
"ended_at": 0,
"sticker_category_ids": [],
"online_user_num": 174,
"announcement_url": "",
"anniversary_bot_comment": "",
"is_emomo_wipe_enabled": 0,
"share_url": "https://www.mirrativ.com/live/9LtbLw58uSrzig_NTxutIw",
"status": {
"msg": "",
"ok": 1,
"error": "",
"captcha_url": "",
"error_code": 0,
"message": ""
},
"orientation": 2,
"app_id": "jp.co.yoozoo.projectyellow",
"app_icon_urls": [
"https://cdn.mirrativ.com/mirrorman-prod/image/app_icon/999ce961e79bfe6d7fff3db363375a17a112810791a692b8858970efa599ca99.png"
],
"timeline": [
{
"app": {
"is_my_app": 0,
"icon_url": "https://cdn.mirrativ.com/mirrorman-prod/image/app_icon/999ce961e79bfe6d7fff3db363375a17a112810791a692b8858970efa599ca99.png",
"store_url": "https://go.onelink.me/app/c8e9133",
"app_id": "jp.co.yoozoo.projectyellow",
"short_title": "プラエデ",
"is_category": 0,
"title": "レッド：プライドオブエデン",
"id": 3139
},
"timestamp": 1600242607,
"title": "雑談しながらゲームする！"
}
],
"enable_clap": 0,
"is_paid_sticker_supported": 0,
"announcement_urls": {},
"sticker_num": 0,
"max_collab_user_num": 3,
"comment_num": 145,
"owner": {
"share_url": "https://www.mirrativ.com/user/104326294",
"is_able_continuous_stream_holiday": 1,
"profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/1046538382ddfccff9b118699ec605cb33cc3c61feb94122f79d5812221e99b5_m.jpeg?1600194531",
"birthday_from": 1615561200,
"is_birthday_editable": 1,
"badges": [
{
"image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_L.ja.png?v6",
"small_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_S.ja.png?v6"
}
],
"is_new": 0,
"catalog_label_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_status_super_holiday_ja@3x.png?v5",
"birthday": "0314",
"birthday_to": 1615820399,
"name": "🐉そあこ💖爽鱗🐉",
"is_birthday": 0,
"description": "色んなゲームを17:30に定期配信してます♡ギフト当たった人は欲しい時にDM求む",
"birthday_editable_date": "",
"properties": [],
"is_continuous_streamer": 1,
"user_id": "104326294",
"live_request_num": "70",
"onlive": null
},
"broadcast_port": 1883,
"recommend_sticker_ids": [],
"sticker_display_type": 0,
"archive_comment_enabled": 0,
"streaming_url_edge": "rtmp://edge-210-140-191-149.mirrativ.com:1935/liveedge/c4d32ce6edec9a37e7b20cff4b656a95714b5b9ee872964d9e435debe74a9518",
"collab_enabled": 0,
"image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_m.jpeg?1600244617",
"orientation_v2": "6"
}
// curl 'https://www.mirrativ.com/api/gift/ranking?obfuscated_user_id=PgIBEgc6jVc&live_id=O5Ia4iX9c5CeZj7DFtg52Q&type=monthly&cursor=' \