add: duplicate tags

add: field sorted fix: head 100 sorted sql
2020-08-24 15:12:52 +08:00 · 2020-08-24 15:12:52 +08:00 · 524daccaf3
commit 524daccaf3
parent e91995fc65
7 changed files with 188 additions and 99 deletions
--- a/goserver/count_tag.go
+++ b/goserver/count_tag.go
@ -91,14 +91,9 @@ func countTagInfo(cxt *gin.Context, ret func(cxt *gin.Context, cw *tagcounter))
 	})

 	var other = &taginfo{Name: "Other...", Value: 0}
-	var i = 0
+
 	for _, v := range cw.CountWord {
-		if i <= 99 {
-			heap.Put(v)
-		} else {
-			break
-		}
-		i++
+		heap.Put(v)
 	}

 	if heap.Size() == 0 {
@ -106,15 +101,18 @@ func countTagInfo(cxt *gin.Context, ret func(cxt *gin.Context, cw *tagcounter))
 		return
 	}

-	// heap.Put(other)
-	cw.PQueue = heap.Values()
+	for i := 0; i <= 100; i++ {
+		if v, ok := heap.Pop(); ok {
+			cw.PQueue = append(cw.PQueue, v)
+		}
+	}
+
 	other.Value = cw.PQueue[len(cw.PQueue)-1].(*taginfo).Value - 1
 	if other.Value == 0 {
 		other.Value = 1
 	}

 	cw.PQueue = append(cw.PQueue, other)
-	// cxt.JSON(200, cw.PQueue)
 	ret(cxt, cw)
 	cw.LastTime = time.Now()
 	tagCounter.Store(platform, cw)
--- a/goserver/main.go
+++ b/goserver/main.go
@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"strconv"
 	"strings"
@ -93,20 +94,37 @@ func Query(cxt *gin.Context, platform string) {
 		cxt.Error(err)
 		return
 	}
-	if psize > 100 {
-		cxt.Error(fmt.Errorf("page size <= 100"))
+	if psize > 5000 {
+		cxt.Error(fmt.Errorf("page size <= 5000"))
 		return
 	}

+	// filter := cxt.Query("filter")
+	orderfield := cxt.Query("orderfield")
+	ordertype := cxt.Query("ordertype")
+
+	log.Println(orderfield, ordertype)
+
 	start := (page - 1) * psize
 	// end := start + 200

-	ssql := fmt.Sprintf(SqlQuery, platform, strconv.Itoa(start), strconv.Itoa(psize))
+	var orderstr string
+	switch ordertype {
+	case "ascend":
+		orderstr = fmt.Sprintf("ORDER BY %s ASC", orderfield)
+	case "descend":
+		orderstr = fmt.Sprintf("ORDER BY %s DESC", orderfield)
+	default:
+		orderstr = ""
+	}
+
+	ssql := fmt.Sprintf(SqlQuery, platform, orderstr, strconv.Itoa(start), strconv.Itoa(psize))
 	rows, err := StoreStreamer.Query(ssql)
 	if err != nil {
 		cxt.Error(err)
 		return
 	}
+
 	var ots []*ObjectQuery
 	for rows.Next() {
 		ot := &ObjectQuery{}
--- a/goserver/main_test.go
+++ b/goserver/main_test.go
@ -42,3 +42,39 @@ func estCountTag(t *testing.T) {
 	}

 }
+
+func estDupTag(t *testing.T) {
+	querysql := "select uid, tags from streamer where tags is not null"
+	rows, err := StoreStreamer.Query(querysql)
+	if err != nil {
+		panic(err)
+	}
+	for rows.Next() {
+		var uid int64
+		var tagsbuf []byte
+		var tags []string
+		rows.Scan(&uid, &tagsbuf)
+		json.Unmarshal(tagsbuf, &tags)
+		if len(tags) > 0 {
+			var newtags []string
+			m := make(map[string]int)
+			for _, t := range tags {
+				if _, ok := m[t]; !ok {
+					newtags = append(newtags, t)
+					m[t] = 1
+				} else {
+					m[t]++
+				}
+			}
+			if len(newtags) != len(tags) {
+				t.Error(uid, tags)
+				newtagsbuf, err := json.Marshal(newtags)
+				if err == nil {
+					StoreStreamer.Exec("update streamer set tags = ? where uid = ?", newtagsbuf, uid)
+				} else {
+					panic(err)
+				}
+			}
+		}
+	}
+}
--- a/goserver/var_sql.go
+++ b/goserver/var_sql.go
@ -27,8 +27,8 @@ FROM
 	intimate_extractor.streamer
 WHERE
 	platform = "%s"
-	AND operator = 0
-	AND latest_log_uid is not NULL limit %s,%s) ie
+	AND operator = 0 
+	AND latest_log_uid is not NULL ) ie
 JOIN intimate_extractor.collect_log cl
 WHERE
-ie.latest_log_uid = cl.log_uid; `
+ie.latest_log_uid = cl.log_uid %s limit %s,%s;`
--- a/src/ChartsCount.js
+++ b/src/ChartsCount.js
@ -29,7 +29,7 @@ function parseData(cw = {}) {
        return b.value - a.value
    })

-    for (var i = 0; i < seriesData.length; i++) {
+    for (i = 0; i < seriesData.length; i++) {
        var o = seriesData[i];
        legendData.push(o.name);
        selected[o.name] = i <= 20;
--- a/src/Table.js
+++ b/src/Table.js
@ -5,78 +5,7 @@ import apihost from './Var';

 const { Option } = Select;

-const columns = [
-  {
-    title: '平台',
-    dataIndex: 'Platform',
-    editable: false,
-    key: 'Platform',
-    //   width: "8%",
-  },
-  {
-    title: 'userid',
-    dataIndex: 'UserId',
-    editable: false,
-    key: 'UserId',
-    //   width: "7%",
-  },
-  {
-    title: '名称',
-    dataIndex: 'UserName',
-    editable: false,
-    key: 'UserName',
-    // width: "7%",
-  },
-  {
-    title: '标签',
-    dataIndex: 'Tags',
-    editable: false,
-    render: tags => (
-      <>
-        {
-          tags != null ?
-            tags.map(tag => {
-              let color = "purple";
-              if (tag.length < 3) {
-                color = 'green';
-              } else if (tag.length < 6) {
-                color = 'geekblue';
-              } else if (tag.length < 9) {
-                color = 'volcano';
-              }

-              return (
-                <Tag color={color} key={tag}>
-                  {tag}
-                </Tag>
-              );
-            }
-
-            ) : null}
-      </>)
-  },
-  {
-    title: '粉丝数(关注)',
-    dataIndex: 'Followers',
-    key: 'Followers',
-    width: "8%",
-  },
-  {
-    title: '礼物数(币|钱)',
-    dataIndex: 'Gratuity',
-    key: 'Gratuity',
-    width: "8%",
-    sorter: (a, b) => a.Gratuity - b.Gratuity,
-    // sortOrder: sortedInfo.columnKey === 'Gratuity' && sortedInfo.order,
-    ellipsis: true,
-  },
-  {
-    title: '数据更新时间',
-    dataIndex: 'UpdateTime',
-    key: 'UpdateTime',
-    // width: "7%",
-  },
-];



@ -96,6 +25,8 @@ class DataTable extends React.Component {
      position: ["topLeft"],
    },
    loading: false,
+    filters: null,
+    sorter: null,
  };

  expandedRow = (record) => {
@ -105,26 +36,31 @@ class DataTable extends React.Component {
        title: 'uid',
        dataIndex: 'Uid',
        key: 'Uid',
+        dataField:'uid',
      },
      {
        title: '直播地址',
        dataIndex: 'LiveUrl',
        key: 'LiveUrl',
+        dataField: 'live_url'
      },
      {
        title: '直播标题',
        dataIndex: 'LiveTitle',
        key: 'LiveTitle',
+        dataField:'live_title',
      },
      {
        title: '近直播开始时间',
        dataIndex: 'LiveStartTime',
        key: 'LiveStartTime',
+        dataField:'live_start_time',
      },
      {
        title: '近直播结束时间',
        dataIndex: 'LiveEndTime',
        key: 'LiveEndTime',
+        dataField:'live_end_time',
      },
    ];

@ -134,11 +70,15 @@ class DataTable extends React.Component {

  updatePlatform(p) {

-    const { pagination } = this.state;
+    const { pagination, sorter, filters } = this.state;
+    console.log("filters", filters); // ${filters?`&filters=${filters}`:""}
+    console.log("sorter", sorter);
    pagination.current = 1;
    this.setState({ platform: p }, () => {
      this.fetchapi({
-        pagination
+        pagination,
+        sorter,
+        filters
      });
    });

@ -150,20 +90,22 @@ class DataTable extends React.Component {
  }

  handleTableChange = (pagination, filters, sorter) => {
-    console.log(filters, sorter);
+ 
    this.fetchapi({
-      sortField: sorter.field,
-      sortOrder: sorter.order,
      pagination,
-      ...filters,
+      sorter,
+      filters,
    });
  };

  fetchapi = (params = {}) => {
+ 
+    const { pagination, sorter, filters } = params;
+    const { platform,  } = this.state;
    this.setState({ loading: true });
-    const { platform, pagination } = this.state;
-
-    fetch(`${apihost}/${platform}/query?page=${pagination.current}&psize=${pagination.pageSize}`, { mode: "cors" }).then((response) => {
+    console.log("filters", filters); // ${filters?`&filters=${filters}`:""}
+    console.log("sorter", sorter);
+    fetch(`${apihost}/${platform}/query?page=${pagination.current}&psize=${pagination.pageSize}${sorter?`&orderfield=${sorter.column?sorter.column.dataField:""}&ordertype=${sorter.order}`:""}`, { mode: "cors" }).then((response) => {
      // console.log(response);
      response.json().then(
        (data) => {
@ -175,6 +117,8 @@ class DataTable extends React.Component {
              ...params.pagination,
              total: 100000,
            },
+            filters: filters, 
+            sorter: sorter
          });
        }
      )
@ -183,7 +127,98 @@ class DataTable extends React.Component {

  render() {

-    const { data, pagination, loading } = this.state;
+    let { data, pagination, loading, sorter, filters } = this.state;
+    sorter = sorter || {};
+    filters = filters || {};
+
+    const columns = [
+        {
+          title: '平台',
+          dataIndex: 'Platform',
+          editable: false,
+          key: 'Platform',
+          //   width: "8%",
+          dataField: 'platform',
+        },
+        {
+          title: 'userid',
+          dataIndex: 'UserId',
+          editable: false,
+          key: 'UserId',
+          dataField: 'user_id',
+          //   width: "7%",
+        },
+        {
+          title: '名称',
+          dataIndex: 'UserName',
+          editable: false,
+          key: 'UserName',
+          dataField: 'user_name',
+          // width: "7%",
+        },
+        {
+          title: '标签',
+          dataIndex: 'Tags',
+          editable: false,
+          dataField: 'tags',
+          render: tags => (
+            <>
+              {
+                tags != null ?
+                  tags.map(tag => {
+                    let color = "purple";
+                    if (tag.length < 3) {
+                      color = 'green';
+                    } else if (tag.length < 6) {
+                      color = 'geekblue';
+                    } else if (tag.length < 9) {
+                      color = 'volcano';
+                    }
+      
+                    return (
+                      <Tag color={color} key={tag}>
+                        {tag}
+                      </Tag>
+                    );
+                  }
+      
+                  ) : null}
+            </>)
+        },
+        {
+          title: '粉丝数(关注)',
+          dataIndex: 'Followers',
+          key: 'Followers',
+          width: "8%",
+          dataField: 'followers',
+          sorter: (a, b) => a.Followers - b.Followers,
+          //   sortOrder: sorter.columnKey === 'Gratuity' && sorter.order,
+        sortDirections: ['descend', 'ascend'],
+        ellipsis: false,
+        },
+        {
+          title: '礼物数(币|钱)',
+          dataIndex: 'Gratuity',
+          key: 'Gratuity',
+          width: "8%",
+          dataField: 'gratuity',
+          // defaultSortOrder: 'ascend',
+         //  filterMultiple: false,
+          // onFilter: (value, record)=>{ console.log(record, value) ;return record.Gratuity >= value;},
+          sorter: (a, b) => a.Gratuity - b.Gratuity,
+        //   sortOrder: sorter.columnKey === 'Gratuity' && sorter.order,
+          sortDirections: ['descend', 'ascend'],
+          ellipsis: false,
+        },
+        {
+          title: '数据更新时间',
+          dataIndex: 'UpdateTime',
+          key: 'UpdateTime',
+          dataField: 'update_time',
+          // width: "7%",
+        },
+      ];
+
    return (
      <div>

@ -208,6 +243,8 @@ class DataTable extends React.Component {
      </Col>
  </Row> */}

+
+
      <Table
        bordered={true}
        size={"middle"}
--- a/src/Var.js
+++ b/src/Var.js
@ -1,5 +1,5 @@


-var apihost = "http://192.168.31.208:5500" 
+var apihost = "http://192.168.16.130:5500" 

 export default apihost;