From d71ec2a56232c774c4deffb651d96e41fcfd0efc Mon Sep 17 00:00:00 2001 From: huangsimin Date: Wed, 4 Mar 2020 12:24:53 +0800 Subject: [PATCH] fix: time and test --- go.mod | 1 + go.sum | 10 ++++++++ main.go | 69 ++++++++++++++++++++++++++++++++++------------------- put_data.go | 12 ++++------ 4 files changed, 60 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 7b542ac..ebc9335 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module rocksdb-api go 1.14 require ( + github.com/474420502/focus v0.7.5 github.com/Pallinder/go-randomdata v1.2.0 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect diff --git a/go.sum b/go.sum index 70c19f7..c0faab7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,17 @@ +github.com/474420502/focus v0.7.3 h1:I76H/HYt9mK6w8/B4viRwI9MqH4JXVgYxWkbN80IY2U= +github.com/474420502/focus v0.7.3/go.mod h1:Rnsdajih5yVIPvDg0tpQjVdEYz+zCZRjlzGfk+Sx3q8= +github.com/474420502/focus v0.7.4 h1:ayL/sIoss07V/PGIwe4MZxA+nW17/Z5X94fK5btsF5k= +github.com/474420502/focus v0.7.4/go.mod h1:Rnsdajih5yVIPvDg0tpQjVdEYz+zCZRjlzGfk+Sx3q8= +github.com/474420502/focus v0.7.5 h1:NZWVnhLjmcnfVAtGdQdJhn19R0c2oiWMa9kLAdaomiQ= +github.com/474420502/focus v0.7.5/go.mod h1:Rnsdajih5yVIPvDg0tpQjVdEYz+zCZRjlzGfk+Sx3q8= +github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= diff --git a/main.go b/main.go index 4a73cdf..8e25b6c 100644 --- a/main.go +++ b/main.go @@ -1,79 +1,100 @@ package main import ( - "encoding/binary" "log" "sync" + "time" + + "github.com/474420502/focus/compare" + pqueuekey "github.com/474420502/focus/priority_queuekey" "github.com/tecbot/gorocksdb" ) -func CreateData() { +func CreateData(gthread int, COUNT int) { db, cfs := OpenDataBase() - GCOUNT := 10 - wg := &sync.WaitGroup{} - wg.Add(GCOUNT) + wg.Add(gthread) - for i := 0; i < GCOUNT; i++ { - go putData(wg, db, cfs) + for i := 0; i < gthread; i++ { + go putData(wg, db, cfs, COUNT) } wg.Wait() } -func queryData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) { +func queryData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle, COUNT int) { defer wg.Done() - // now := time.Now().Unix() + now := time.Now().Unix() ro := gorocksdb.NewDefaultReadOptions() - COUNT := 100 - cf := cfs[len(cfs)-1] iter := db.NewIteratorCF(ro, cf) + defer iter.Close() - temp := make([]byte, 8) - binary.BigEndian.PutUint64(temp, 10) - iter.Seek(temp) - + iter.SeekToFirst() + // temp := make([]byte, 8) + // binary.BigEndian.PutUint64(temp, 5) + // iter.Seek(temp) log.Println(iter.Valid()) - for i := 0; i < COUNT && iter.Valid(); i++ { - bkey := iter.Key().Data() - key := string(bkey) - value := string(iter.Value().Data()) + pq := pqueuekey.New(compare.ByteArray) - log.Println(binary.BigEndian.Uint64(bkey[0:8]), "\nkey:", key, "\n", "value:", value) + for i := 0; i < COUNT && iter.Valid(); i++ { + + bkey := iter.Key().Data() + bvalue := iter.Value().Data() + + pq.Push(bkey, bvalue) + + // key := string(bkey) + // value := string(bvalue) + + // log.Println(binary.BigEndian.Uint64(bkey[0:8]), "\nkey:", key, "\n", "value:", value) iter.Next() } - // log.Println(int64(COUNT) / (time.Now().Unix() - now)) + log.Println("size:", pq.Size()) + log.Println("qps:", int64(COUNT)/(time.Now().Unix()-now)) + + piter := pq.Iterator() + for i := 0; piter.Next() && i < 10; i++ { + println("key:", string(piter.Key().([]byte)), "value:", string(piter.Value().([]byte))) + } + } func QueryData() { + db, cfs := OpenDataBase() - GCOUNT := 1 + GCOUNT := 10 + COUNT := 1000000 + + now := time.Now().Unix() wg := &sync.WaitGroup{} wg.Add(GCOUNT) for i := 0; i < GCOUNT; i++ { - go queryData(wg, db, cfs) + + go queryData(wg, db, cfs, COUNT) } wg.Wait() + + log.Println("total qps:", int64(COUNT*GCOUNT)/(time.Now().Unix()-now)) } func main() { QueryData() - // CreateData() + // CreateData(50, 100000) // 10 * 10000 // db.Put(wo, []byte("foo"), []byte("bar")) diff --git a/put_data.go b/put_data.go index 307f246..f748007 100644 --- a/put_data.go +++ b/put_data.go @@ -10,13 +10,12 @@ import ( "github.com/tecbot/gorocksdb" ) -func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) { +func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle, COUNT int) { defer wg.Done() now := time.Now().Unix() wo := gorocksdb.NewDefaultWriteOptions() - COUNT := 100000 for i := 0; i < COUNT; i++ { temp := make([]byte, 8) @@ -39,15 +38,12 @@ func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamily keybuf := append(putbuf, '\x01') binary.BigEndian.PutUint16(temp, uint16(age)) - keybuf = append(name, temp[0:2]...) + keybuf = append(keybuf, temp[0:2]...) keybuf = append(keybuf, '\x01') binary.BigEndian.PutUint16(temp, uint16(level)) - keybuf = append(name, temp[0:2]...) + keybuf = append(keybuf, temp[0:2]...) - putbuf = append(putbuf, '\x01') - putbuf = append(keybuf, []byte(task)...) - - db.PutCF(wo, cfs[len(cfs)-1], keybuf, putbuf) + db.PutCF(wo, cfs[len(cfs)-1], keybuf, []byte(task)) } log.Println(int64(COUNT) / (time.Now().Unix() - now))