From a511f1b1e699431221c8e5598711400d868ac0d4 Mon Sep 17 00:00:00 2001 From: eson <474420502@qq.com> Date: Wed, 4 Mar 2020 03:32:07 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA=E6=AD=A3=E5=B8=B8?= =?UTF-8?q?=E7=9A=84=E9=80=BB=E8=BE=91=E5=AD=98=E5=82=A8=20=E4=BA=8C?= =?UTF-8?q?=E8=BF=9B=E5=88=B6=E6=95=B0=E5=AD=97=E7=9B=B8=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- base.go | 66 +++++++++++++++++++++++++++++++ go.mod | 2 +- main.go | 112 +++++++++------------------------------------------- put_data.go | 54 +++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 95 deletions(-) create mode 100644 base.go create mode 100644 put_data.go diff --git a/base.go b/base.go new file mode 100644 index 0000000..5fcdc0e --- /dev/null +++ b/base.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "log" + "os" + "time" + + "github.com/tecbot/gorocksdb" +) + +func OpenDataBase() (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle) { + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + + bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) + bbto.SetCacheIndexAndFilterBlocksWithHighPriority(true) + + opts := gorocksdb.NewDefaultOptions() + + bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(16)) + + opts.SetBlockBasedTableFactory(bbto) + opts.SetCreateIfMissing(true) + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCompression(gorocksdb.LZ4Compression) + + year, month, day := time.Now().Date() + + timeCFStr := fmt.Sprintf("%d-%d-%d", year, int64(month), day) + + f, err := os.OpenFile("./log", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0660) + if err != nil { + panic(err) + } + log.SetOutput(f) + + log.Println(timeCFStr) + names, err := gorocksdb.ListColumnFamilies(opts, ".rocksdb") + if err != nil { + log.Println(".rocksdb 文件不存在") + names = append(names, "default") + } + + isadd := true + + var opslist []*gorocksdb.Options + for _, name := range names { + opslist = append(opslist, opts) + if timeCFStr == name { + isadd = false + } + } + + if isadd { + names = append(names, timeCFStr) + opslist = append(opslist, opts) + } + + log.Println(names) + + db, cfs, err := gorocksdb.OpenDbColumnFamilies(opts, ".rocksdb", names, opslist) + if err != nil { + panic(err) + } + return db, cfs +} diff --git a/go.mod b/go.mod index 372fc04..7b542ac 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module queue +module rocksdb-api go 1.14 diff --git a/main.go b/main.go index 4a75265..4a73cdf 100644 --- a/main.go +++ b/main.go @@ -1,98 +1,13 @@ package main import ( - "fmt" + "encoding/binary" "log" - "os" - "strconv" "sync" - "time" - "github.com/Pallinder/go-randomdata" "github.com/tecbot/gorocksdb" ) -func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) { - defer wg.Done() - - now := time.Now().Unix() - - wo := gorocksdb.NewDefaultWriteOptions() - COUNT := 1000000 - - for i := 0; i < COUNT; i++ { - name := strconv.Itoa(i) + "-" + randomdata.FirstName(-1) + "-" + randomdata.FullName(-1) + strconv.Itoa(randomdata.Number(0, 10000000)) - age := randomdata.Number(0, 105) - level := randomdata.Number(0, 10) - task := randomdata.Email() + randomdata.Adjective() + randomdata.City() - - // tx.Exec("insert into user(name, age,level, task, attr, taskid) values(?, ?, ?, ?, ?, ?)", name, age, level, task, attr, taskid) - putbuf := []byte(name) - keybuf := append(putbuf, '\x01', byte(age), '\x01', byte(level), '\x01') - - putbuf = append(keybuf, []byte(task)...) - db.PutCF(wo, cfs[1], keybuf, putbuf) - } - - log.Println(int64(COUNT) / (time.Now().Unix() - now)) -} - -func OpenDataBase() (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle) { - bbto := gorocksdb.NewDefaultBlockBasedTableOptions() - - bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) - bbto.SetCacheIndexAndFilterBlocksWithHighPriority(true) - - opts := gorocksdb.NewDefaultOptions() - - bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(16)) - - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) - opts.SetCompression(gorocksdb.LZ4Compression) - - year, month, day := time.Now().Date() - - timeCFStr := fmt.Sprintf("%d-%d-%d", year, int64(month), day) - - f, err := os.OpenFile("./log", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0660) - if err != nil { - panic(err) - } - log.SetOutput(f) - - log.Println(timeCFStr) - names, err := gorocksdb.ListColumnFamilies(opts, ".rocksdb") - if err != nil { - log.Println(".rocksdb 文件不存在") - names = append(names, "default") - } - - isadd := true - - var opslist []*gorocksdb.Options - for _, name := range names { - opslist = append(opslist, opts) - if timeCFStr == name { - isadd = false - } - } - - if isadd { - names = append(names, timeCFStr) - opslist = append(opslist, opts) - } - - log.Println(names) - - db, cfs, err := gorocksdb.OpenDbColumnFamilies(opts, ".rocksdb", names, opslist) - if err != nil { - panic(err) - } - return db, cfs -} - func CreateData() { db, cfs := OpenDataBase() @@ -116,15 +31,24 @@ func queryData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFami ro := gorocksdb.NewDefaultReadOptions() - COUNT := 10 + COUNT := 100 - iter := db.NewIteratorCF(ro, cfs[2]) - iter.SeekForPrev([]byte("100")) + cf := cfs[len(cfs)-1] + iter := db.NewIteratorCF(ro, cf) - for i := 0; i < COUNT; i++ { - key := string(iter.Key().Data()) + temp := make([]byte, 8) + binary.BigEndian.PutUint64(temp, 10) + iter.Seek(temp) + + log.Println(iter.Valid()) + + for i := 0; i < COUNT && iter.Valid(); i++ { + bkey := iter.Key().Data() + key := string(bkey) value := string(iter.Value().Data()) - log.Println("\nkey:", key, "\n", "value:", value) + + log.Println(binary.BigEndian.Uint64(bkey[0:8]), "\nkey:", key, "\n", "value:", value) + iter.Next() } @@ -148,8 +72,8 @@ func QueryData() { func main() { - // QueryData() - CreateData() + QueryData() + // CreateData() // db.Put(wo, []byte("foo"), []byte("bar")) diff --git a/put_data.go b/put_data.go new file mode 100644 index 0000000..307f246 --- /dev/null +++ b/put_data.go @@ -0,0 +1,54 @@ +package main + +import ( + "encoding/binary" + "log" + "sync" + "time" + + "github.com/Pallinder/go-randomdata" + "github.com/tecbot/gorocksdb" +) + +func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) { + defer wg.Done() + + now := time.Now().Unix() + + wo := gorocksdb.NewDefaultWriteOptions() + COUNT := 100000 + + for i := 0; i < COUNT; i++ { + temp := make([]byte, 8) + name := make([]byte, 8) + binary.BigEndian.PutUint64(name, uint64(i)) + name = append(name, '-') + name = append(name, []byte(randomdata.FirstName(-1))...) + name = append(name, '-') + name = append(name, []byte(randomdata.FullName(-1))...) + name = append(name, '-') + binary.BigEndian.PutUint64(temp, uint64(randomdata.Number(0, 10000000))) + name = append(name, temp...) + + age := randomdata.Number(0, 105) + level := randomdata.Number(0, 10) + task := randomdata.Email() + randomdata.Adjective() + randomdata.City() + + // tx.Exec("insert into user(name, age,level, task, attr, taskid) values(?, ?, ?, ?, ?, ?)", name, age, level, task, attr, taskid) + putbuf := name + + keybuf := append(putbuf, '\x01') + binary.BigEndian.PutUint16(temp, uint16(age)) + keybuf = append(name, temp[0:2]...) + keybuf = append(keybuf, '\x01') + binary.BigEndian.PutUint16(temp, uint16(level)) + keybuf = append(name, temp[0:2]...) + + putbuf = append(putbuf, '\x01') + putbuf = append(keybuf, []byte(task)...) + + db.PutCF(wo, cfs[len(cfs)-1], keybuf, putbuf) + } + + log.Println(int64(COUNT) / (time.Now().Unix() - now)) +}