intimate/store.go

218 lines
6.6 KiB
Go
Raw Normal View History

2020-07-07 10:39:24 +00:00
package intimate
import (
"database/sql"
"errors"
2020-07-06 09:58:24 +00:00
"log"
_ "github.com/go-sql-driver/mysql"
)
// OperatorFlag 标志
type OperatorFlag int32
const (
2020-07-09 03:38:51 +00:00
// OperatorOK 等待被处理
OperatorOK OperatorFlag = 100
// OperatorWait 等待被处理
OperatorWait OperatorFlag = 1000
// OperatorError 错误标志
OperatorError OperatorFlag = 10000
)
2020-07-10 04:05:33 +00:00
// SourceStore 储存
type SourceStore struct {
2020-07-09 03:38:51 +00:00
table string
db *sql.DB
errorCount int
errorLimit int
}
2020-07-10 04:05:33 +00:00
// NewSourceStore 创建一个存储实例
func NewSourceStore(table string) *SourceStore {
db, err := sql.Open("mysql", InitConfig.Database.SourceURI)
if err != nil {
panic(err)
}
2020-07-10 04:05:33 +00:00
return &SourceStore{table: table, db: db}
}
2020-07-10 04:05:33 +00:00
func (store *SourceStore) errorAlarm(err error) {
2020-07-06 09:58:24 +00:00
if err != nil {
2020-07-09 03:38:51 +00:00
log.Println("store error: ", err)
// 报警. 如果数据插入有问题
store.errorCount++
if store.errorCount >= store.errorLimit {
// 数据库频繁操作初问题 报警, 减少没意义的请求
}
} else {
if store.errorCount > 0 {
store.errorCount--
}
2020-07-06 09:58:24 +00:00
}
}
2020-07-10 04:05:33 +00:00
// Insert 插入数据
func (store *SourceStore) Insert(isource IGetSource) {
_, err := store.db.Exec("insert into "+store.table+"(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg())
2020-07-09 03:38:51 +00:00
store.errorAlarm(err)
}
2020-07-10 04:05:33 +00:00
// Update 更新数据
func (store *SourceStore) Update(isource IUpdateSource) {
2020-07-13 10:10:48 +00:00
_, err := store.db.Exec("update "+store.table+" set ext = ?, pass_gob = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetPassGob(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
2020-07-09 03:38:51 +00:00
store.errorAlarm(err)
}
2020-07-13 10:10:48 +00:00
// UpdateError 更新错误数据
func (store *SourceStore) UpdateError(isource IUpdateSource, err error) {
isource.SetOperator(int32(OperatorError))
isource.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
_, dberr := store.db.Exec("update "+store.table+" set operator = ?, error_msg = ? where uid = ?", isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
store.errorAlarm(dberr)
}
2020-07-10 04:05:33 +00:00
// Restore 恢复Operator数据状态
func (store *SourceStore) Restore(isource IUpdateSource) {
_, err := store.db.Exec("update "+store.table+" set operator = ? where uid = ?", isource.GetLastOperator(), isource.GetUid())
store.errorAlarm(err)
}
// Pop 弹出一条未处理的数据
func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSource, error) {
tx, err := store.db.Begin()
if err != nil {
log.Println(err, targetType)
return nil, err
}
var args = []interface{}{targetType}
selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?`
if len(operators) == 0 {
selectSQL += " and operator = ?"
args = append(args, 0)
} else {
for _, operator := range operators {
selectSQL += " and operator = ?"
args = append(args, operator)
}
}
// log.Println(selectSQL + ` limit 1 for update`)
row := tx.QueryRow(selectSQL+` limit 1 for update`, args...)
defer func() {
err := tx.Commit()
if err != nil {
log.Println(err)
err = tx.Rollback()
if err != nil {
log.Println(err)
}
}
}()
if row != nil {
s := &Source{}
2020-07-10 04:05:33 +00:00
// uid, url, target_type, source, ext, operator
err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator)
2020-07-10 04:05:33 +00:00
s.SetLastOperator(s.Operator)
if err != nil {
log.Println(err, targetType)
_, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid)
if err != nil {
log.Println(err)
}
return nil, err
}
_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
return s, nil
}
return nil, errors.New("TaskQueue is nil")
}
2020-07-10 04:05:33 +00:00
// AnchorTable 主播表名称
const AnchorTable string = "anchor_info"
// CollectLogTable 采集日志表
const CollectLogTable string = "collect_log"
type ExtractorStore struct {
db *sql.DB
errorCount int
errorLimit int
}
func (store *ExtractorStore) errorAlarm(err error) {
2020-07-10 04:05:33 +00:00
if err != nil {
2020-07-10 10:31:17 +00:00
log.Panic("store error: ", err)
// 报警. 如果数据插入有问题
store.errorCount++
if store.errorCount >= store.errorLimit {
// 数据库频繁操作初问题 报警, 减少没意义的请求
}
} else {
if store.errorCount > 0 {
store.errorCount--
}
2020-07-10 04:05:33 +00:00
}
}
2020-07-10 10:31:17 +00:00
func NewExtractorStore() *ExtractorStore {
db, err := sql.Open("mysql", InitConfig.Database.ExtractorURI)
if err != nil {
panic(err)
}
return &ExtractorStore{db: db}
}
/*
`uid` bigint,
`platform` varchar(255) NOT NULL,
`anchor_id` varchar(255) NOT NULL,
`anchor_name` varchar(255) NOT NULL,
`live_url` text,
`channel` varchar(128) DEFAULT NULL,
`show_type` varchar(255) DEFAULT NULL,
*/
// InsertAnchorInfo AnchorInfo表, 插入数据
func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) {
_, err := store.db.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt())
store.errorAlarm(err)
}
/*
`uid` bigint,
`platform` varchar(255) NOT NULL,
`anchor_id` varchar(255) NOT NULL,
`is_showing` tinyint(1) DEFAULT NULL,
`is_error` tinyint(1) DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`views` int(11) DEFAULT NULL,
`giver` json DEFAULT NULL,
`gratuity` int(11) DEFAULT NULL,
`show_title` text DEFAULT NULL,
`show_start_time` timestamp NULL DEFAULT NULL,
`show_end_time` timestamp NULL DEFAULT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, --时间戳从源数据里获取
`ext` json DEFAULT NULL,
2020-07-10 10:31:17 +00:00
`error_msg` text DEFAULT NULL,
*/
// InsertCollectLog CollectLog表插入数据
2020-07-10 10:31:17 +00:00
func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) {
_, err := store.db.Exec("insert into "+CollectLogTable+"(uid, platform, anchor_id, is_showing, is_error, followers, views, giver, gratuity, show_title, show_start_time, show_end_time, update_time, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsShowing(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetShowTitle(), isource.GetShowStartTime(), isource.GetShowEndTime(), isource.GetUpdateTime(), isource.GetExt(), isource.GetErrorMsg(),
)
store.errorAlarm(err)
2020-07-10 04:05:33 +00:00
}