From 5a6d78bfb9ecbba2406fdf8b98230888160d4249 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 3 Mar 2020 18:31:48 +0800 Subject: [PATCH] first --- go.mod | 13 ++++ go.sum | 23 +++++++ install_rocksdb.sh | 1 + main.go | 166 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 install_rocksdb.sh create mode 100644 main.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..372fc04 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module queue + +go 1.14 + +require ( + github.com/Pallinder/go-randomdata v1.2.0 + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect + github.com/stretchr/testify v1.5.1 // indirect + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c + golang.org/x/text v0.3.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..70c19f7 --- /dev/null +++ b/go.sum @@ -0,0 +1,23 @@ +github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= +github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/install_rocksdb.sh b/install_rocksdb.sh new file mode 100644 index 0000000..4c074e7 --- /dev/null +++ b/install_rocksdb.sh @@ -0,0 +1 @@ +CGO_CFLAGS="-I/usr/include/rocksdb" CGO_LDFLAGS="-L/usr/lib/x86_64-linux-gnu -lrocksdb -lstdc++ -lm -lz -llz4 -lzstd" go get -v github.com/tecbot/gorocksdb diff --git a/main.go b/main.go new file mode 100644 index 0000000..6437d3a --- /dev/null +++ b/main.go @@ -0,0 +1,166 @@ +package main + +import ( + "fmt" + "log" + "os" + "strconv" + "sync" + "time" + + "github.com/Pallinder/go-randomdata" + "github.com/tecbot/gorocksdb" +) + +func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) { + defer wg.Done() + + now := time.Now().Unix() + + wo := gorocksdb.NewDefaultWriteOptions() + COUNT := 1000000 + + for i := 0; i < COUNT; i++ { + name := strconv.Itoa(i) + "-" + randomdata.FirstName(-1) + "-" + randomdata.FullName(-1) + strconv.Itoa(randomdata.Number(0, 10000000)) + age := randomdata.Number(0, 105) + level := randomdata.Number(0, 10) + task := randomdata.Email() + randomdata.Adjective() + randomdata.City() + + // tx.Exec("insert into user(name, age,level, task, attr, taskid) values(?, ?, ?, ?, ?, ?)", name, age, level, task, attr, taskid) + putbuf := []byte(name) + keybuf := append(putbuf, '\x01', byte(age), '\x01', byte(level), '\x01') + + putbuf = append(keybuf, []byte(task)...) + db.PutCF(wo, cfs[1], keybuf, putbuf) + } + + log.Println(int64(COUNT) / (time.Now().Unix() - now)) +} + +func OpenDataBase() (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle) { + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + + bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) + bbto.SetCacheIndexAndFilterBlocksWithHighPriority(true) + + opts := gorocksdb.NewDefaultOptions() + + bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(16)) + + opts.SetBlockBasedTableFactory(bbto) + opts.SetCreateIfMissing(true) + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCompression(gorocksdb.LZ4Compression) + + year, month, day := time.Now().Date() + + timeCFStr := fmt.Sprintf("%d-%d-%d", year, int64(month), day) + + f, err := os.OpenFile("./log", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0660) + if err != nil { + panic(err) + } + log.SetOutput(f) + + log.Println(timeCFStr) + names, err := gorocksdb.ListColumnFamilies(opts, "./.rocksdb") + if err != nil { + panic(err) + } + + isadd := true + + var opslist []*gorocksdb.Options + for _, name := range names { + opslist = append(opslist, opts) + if timeCFStr == name { + isadd = false + } + } + + if isadd { + names = append(names, timeCFStr) + opslist = append(opslist, opts) + } + + log.Println(names) + + db, cfs, err := gorocksdb.OpenDbColumnFamilies(opts, "./.rocksdb", names, opslist) + if err != nil { + panic(err) + } + return db, cfs +} + +func CreateData() { + + db, cfs := OpenDataBase() + + GCOUNT := 100 + + wg := &sync.WaitGroup{} + wg.Add(GCOUNT) + + for i := 0; i < GCOUNT; i++ { + go putData(wg, db, cfs) + } + + wg.Wait() +} + +func queryData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) { + defer wg.Done() + + // now := time.Now().Unix() + + ro := gorocksdb.NewDefaultReadOptions() + + COUNT := 10 + + iter := db.NewIteratorCF(ro, cfs[2]) + iter.SeekForPrev([]byte("100")) + + for i := 0; i < COUNT; i++ { + key := string(iter.Key().Data()) + value := string(iter.Value().Data()) + log.Println("\nkey:", key, "\n", "value:", value) + iter.Next() + } + + // log.Println(int64(COUNT) / (time.Now().Unix() - now)) +} + +func QueryData() { + db, cfs := OpenDataBase() + + GCOUNT := 1 + + wg := &sync.WaitGroup{} + wg.Add(GCOUNT) + + for i := 0; i < GCOUNT; i++ { + go queryData(wg, db, cfs) + } + + wg.Wait() +} + +func main() { + + // QueryData() + CreateData() + + // db.Put(wo, []byte("foo"), []byte("bar")) + + // ro := gorocksdb.NewDefaultReadOptions() + // db, err := gorocksdb.OpenDb(opts, "./.rocksdb") + // if err != nil { + // panic(err) + // } + // s, err := db.Get(ro, []byte("foo")) + // if err != nil { + // panic(err) + // } + // log.Println(string(s.Data())) + +}