first

go.mod

@@ -0,0 +1,13 @@
+module queue
+
+go 1.14
+
+require (
+	github.com/Pallinder/go-randomdata v1.2.0
+	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
+	github.com/stretchr/testify v1.5.1 // indirect
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
+	golang.org/x/text v0.3.2 // indirect
+)

go.sum

@@ -0,0 +1,23 @@
+github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
+github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
+github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

install_rocksdb.sh

@@ -0,0 +1 @@
+CGO_CFLAGS="-I/usr/include/rocksdb" CGO_LDFLAGS="-L/usr/lib/x86_64-linux-gnu -lrocksdb -lstdc++ -lm -lz  -llz4 -lzstd" go get -v github.com/tecbot/gorocksdb

main.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/Pallinder/go-randomdata"
+	"github.com/tecbot/gorocksdb"
+)
+
+func putData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) {
+	defer wg.Done()
+
+	now := time.Now().Unix()
+
+	wo := gorocksdb.NewDefaultWriteOptions()
+	COUNT := 1000000
+
+	for i := 0; i < COUNT; i++ {
+		name := strconv.Itoa(i) + "-" + randomdata.FirstName(-1) + "-" + randomdata.FullName(-1) + strconv.Itoa(randomdata.Number(0, 10000000))
+		age := randomdata.Number(0, 105)
+		level := randomdata.Number(0, 10)
+		task := randomdata.Email() + randomdata.Adjective() + randomdata.City()
+
+		// tx.Exec("insert into user(name, age,level, task, attr, taskid) values(?, ?, ?, ?, ?, ?)", name, age, level, task, attr, taskid)
+		putbuf := []byte(name)
+		keybuf := append(putbuf, '\x01', byte(age), '\x01', byte(level), '\x01')
+
+		putbuf = append(keybuf, []byte(task)...)
+		db.PutCF(wo, cfs[1], keybuf, putbuf)
+	}
+
+	log.Println(int64(COUNT) / (time.Now().Unix() - now))
+}
+
+func OpenDataBase() (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle) {
+	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
+
+	bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30))
+	bbto.SetCacheIndexAndFilterBlocksWithHighPriority(true)
+
+	opts := gorocksdb.NewDefaultOptions()
+
+	bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(16))
+
+	opts.SetBlockBasedTableFactory(bbto)
+	opts.SetCreateIfMissing(true)
+	opts.SetCreateIfMissingColumnFamilies(true)
+	opts.SetCompression(gorocksdb.LZ4Compression)
+
+	year, month, day := time.Now().Date()
+
+	timeCFStr := fmt.Sprintf("%d-%d-%d", year, int64(month), day)
+
+	f, err := os.OpenFile("./log", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0660)
+	if err != nil {
+		panic(err)
+	}
+	log.SetOutput(f)
+
+	log.Println(timeCFStr)
+	names, err := gorocksdb.ListColumnFamilies(opts, "./.rocksdb")
+	if err != nil {
+		panic(err)
+	}
+
+	isadd := true
+
+	var opslist []*gorocksdb.Options
+	for _, name := range names {
+		opslist = append(opslist, opts)
+		if timeCFStr == name {
+			isadd = false
+		}
+	}
+
+	if isadd {
+		names = append(names, timeCFStr)
+		opslist = append(opslist, opts)
+	}
+
+	log.Println(names)
+
+	db, cfs, err := gorocksdb.OpenDbColumnFamilies(opts, "./.rocksdb", names, opslist)
+	if err != nil {
+		panic(err)
+	}
+	return db, cfs
+}
+
+func CreateData() {
+
+	db, cfs := OpenDataBase()
+
+	GCOUNT := 100
+
+	wg := &sync.WaitGroup{}
+	wg.Add(GCOUNT)
+
+	for i := 0; i < GCOUNT; i++ {
+		go putData(wg, db, cfs)
+	}
+
+	wg.Wait()
+}
+
+func queryData(wg *sync.WaitGroup, db *gorocksdb.DB, cfs []*gorocksdb.ColumnFamilyHandle) {
+	defer wg.Done()
+
+	// now := time.Now().Unix()
+
+	ro := gorocksdb.NewDefaultReadOptions()
+
+	COUNT := 10
+
+	iter := db.NewIteratorCF(ro, cfs[2])
+	iter.SeekForPrev([]byte("100"))
+
+	for i := 0; i < COUNT; i++ {
+		key := string(iter.Key().Data())
+		value := string(iter.Value().Data())
+		log.Println("\nkey:", key, "\n", "value:", value)
+		iter.Next()
+	}
+
+	// log.Println(int64(COUNT) / (time.Now().Unix() - now))
+}
+
+func QueryData() {
+	db, cfs := OpenDataBase()
+
+	GCOUNT := 1
+
+	wg := &sync.WaitGroup{}
+	wg.Add(GCOUNT)
+
+	for i := 0; i < GCOUNT; i++ {
+		go queryData(wg, db, cfs)
+	}
+
+	wg.Wait()
+}
+
+func main() {
+
+	// QueryData()
+	CreateData()
+
+	//  db.Put(wo, []byte("foo"), []byte("bar"))
+
+	// ro := gorocksdb.NewDefaultReadOptions()
+	// db, err := gorocksdb.OpenDb(opts, "./.rocksdb")
+	// if err != nil {
+	// 	panic(err)
+	// }
+	// s, err := db.Get(ro, []byte("foo"))
+	// if err != nil {
+	// 	panic(err)
+	// }
+	// log.Println(string(s.Data()))
+
+}