diff --git a/build.sh b/build.sh index 0c15cd7..4e3015f 100755 --- a/build.sh +++ b/build.sh @@ -28,5 +28,6 @@ cd ${ROOT_DIR}/demo/add_go_license && go build -ldflags "$LDFlags" -o ${ROOT_DIR cd ${ROOT_DIR}/demo/taskpool && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/taskpool && cd ${ROOT_DIR}/demo/bufferpool && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/bufferpool && cd ${ROOT_DIR}/demo/samefile && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/samefile && +cd ${ROOT_DIR}/demo/slicebytepool && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/slicebytepool && ls -lrt ${ROOT_DIR}/bin && echo 'build done.' diff --git a/demo/slicebytepool/main.go b/demo/slicebytepool/main.go new file mode 100644 index 0000000..b3f8e02 --- /dev/null +++ b/demo/slicebytepool/main.go @@ -0,0 +1,142 @@ +// Copyright 2019, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package main + +import ( + "flag" + "fmt" + "math/rand" + "os" + "runtime" + "runtime/pprof" + "sync" + "sync/atomic" + "time" + + "github.com/q191201771/naza/pkg/slicebytepool" + + "github.com/q191201771/naza/pkg/nazalog" +) + +var bp slicebytepool.SliceBytePool + +//var count int32 +var doneCount uint32 +var tmpSliceByte []byte + +var gorutineNum = 1000 +var loopNum = 1000 +var sleepMSec = time.Duration(10) * time.Millisecond + +func size() int { + return random(1, 256*1024) + + //return 128 * 1024 + + //ss := []int{1000, 2000, 5000} + //////ss := []int{128, 1024, 4096, 16384} + //atomic.AddInt32(&count, 1) + //return ss[count % 3] + + //count++ + //if count > 128 * 1024 { + // count = 1 + //} + //return count +} + +func random(l, r int) int { + return l + (rand.Int() % (r - l)) +} + +func originFunc() { + size := size() + buf := make([]byte, size) + tmpSliceByte = buf + atomic.AddUint32(&doneCount, 1) + time.Sleep(sleepMSec) +} + +func bufferPoolFunc() { + size := size() + buf := bp.Get(size) + tmpSliceByte = buf + time.Sleep(sleepMSec) + bp.Put(buf) + atomic.AddUint32(&doneCount, 1) +} + +func main() { + strategy := parseFlag() + nazalog.Debugf("strategy: %d", strategy) + + //cpufd, err := os.Create("/tmp/cpu.prof") + //nazalog.FatalIfErrorNotNil(err) + //pprof.StartCPUProfile(cpufd) + //defer pprof.StopCPUProfile() + + rand.Seed(time.Now().Unix()) + + if strategy != 3 { + bp = slicebytepool.NewSliceBytePool(slicebytepool.Strategy(strategy)) + } + + go func() { + for { + if strategy != 3 { + nazalog.Debugf("time. done=%d, pool=%+v", atomic.LoadUint32(&doneCount), bp.RetrieveStatus()) + time.Sleep(1 * time.Second) + } else { + nazalog.Debugf("time. done=%d", atomic.LoadUint32(&doneCount)) + time.Sleep(1 * time.Second) + } + } + }() + + var wg sync.WaitGroup + wg.Add(gorutineNum * loopNum) + nazalog.Debug("> loop.") + for i := 0; i < gorutineNum; i++ { + go func() { + if strategy != 3 { + for j := 0; j < loopNum; j++ { + bufferPoolFunc() + wg.Done() + } + } else { + for j := 0; j < loopNum; j++ { + originFunc() + wg.Done() + } + } + }() + } + wg.Wait() + memfd, err := os.Create(fmt.Sprintf("/tmp/mem%d.prof", strategy)) + nazalog.FatalIfErrorNotNil(err) + _ = pprof.WriteHeapProfile(memfd) + _ = memfd.Close() + nazalog.Debug("> GC.") + runtime.GC() + nazalog.Debug("< GC.") + if strategy != 5 { + nazalog.Debugf("%+v", bp.RetrieveStatus()) + } + nazalog.Debug("< loop.") +} + +func parseFlag() int { + strategy := flag.Int("t", 0, "type: 1. multi std pool 2. multi slice pool 3. origin") + flag.Parse() + if *strategy < 1 || *strategy > 3 { + flag.Usage() + os.Exit(1) + } + return *strategy +} diff --git a/pkg/slicebytepool/default.go b/pkg/slicebytepool/default.go new file mode 100644 index 0000000..c5331f3 --- /dev/null +++ b/pkg/slicebytepool/default.go @@ -0,0 +1,31 @@ +// Copyright 2019, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package slicebytepool + +var defaultPool SliceBytePool + +func Get(size int) []byte { + return defaultPool.Get(size) +} + +func Put(buf []byte) { + defaultPool.Put(buf) +} + +func RetrieveStatus() Status { + return defaultPool.RetrieveStatus() +} + +func Init(strategy Strategy) { + defaultPool = NewSliceBytePool(strategy) +} + +func init() { + Init(StrategyMultiSlicePoolBucket) +} diff --git a/pkg/slicebytepool/interface.go b/pkg/slicebytepool/interface.go new file mode 100644 index 0000000..53901b8 --- /dev/null +++ b/pkg/slicebytepool/interface.go @@ -0,0 +1,60 @@ +// Copyright 2019, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package slicebytepool + +type SliceBytePool interface { + Get(size int) []byte + + Put(buf []byte) + + RetrieveStatus() Status +} + +type Status struct { + getCount int64 + putCount int64 + hitCount int64 + sizeBytes int64 +} + +type Strategy int + +const ( + StrategyMultiStdPoolBucket = iota + 1 + + StrategyMultiSlicePoolBucket +) + +type Bucket interface { + Get(size int) []byte + + Put(buf []byte) +} + +func NewSliceBytePool(strategy Strategy) SliceBytePool { + var capToFreeBucket map[int]Bucket + + switch strategy { + case StrategyMultiStdPoolBucket: + capToFreeBucket = make(map[int]Bucket) + for i := minSize; i <= maxSize; i <<= 1 { + capToFreeBucket[i] = NewStdPoolBucket() + } + case StrategyMultiSlicePoolBucket: + capToFreeBucket = make(map[int]Bucket) + for i := minSize; i <= maxSize; i <<= 1 { + capToFreeBucket[i] = NewSliceBucket() + } + } + + return &sliceBytePool{ + strategy: strategy, + capToFreeBucket: capToFreeBucket, + } +} diff --git a/pkg/slicebytepool/slice_bucket.go b/pkg/slicebytepool/slice_bucket.go new file mode 100644 index 0000000..fd5ad3e --- /dev/null +++ b/pkg/slicebytepool/slice_bucket.go @@ -0,0 +1,39 @@ +// Copyright 2019, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package slicebytepool + +import ( + "sync" +) + +type SliceBucket struct { + m sync.Mutex + core [][]byte +} + +func NewSliceBucket() *SliceBucket { + return new(SliceBucket) +} + +func (b *SliceBucket) Get(size int) []byte { + b.m.Lock() + defer b.m.Unlock() + if len(b.core) == 0 { + return nil + } + buf := b.core[len(b.core)-1] + b.core = b.core[:len(b.core)-1] + return buf[0:size] +} + +func (b *SliceBucket) Put(buf []byte) { + b.m.Lock() + defer b.m.Unlock() + b.core = append(b.core, buf) +} diff --git a/pkg/slicebytepool/slicebytepool.go b/pkg/slicebytepool/slicebytepool.go new file mode 100644 index 0000000..a664e07 --- /dev/null +++ b/pkg/slicebytepool/slicebytepool.go @@ -0,0 +1,101 @@ +// Copyright 2019, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package slicebytepool + +import ( + "sync/atomic" +) + +var ( + minSize = 1024 + maxSize = 1073741824 +) + +type sliceBytePool struct { + strategy Strategy + capToFreeBucket map[int]Bucket + status Status +} + +func (bp *sliceBytePool) Get(size int) []byte { + atomic.AddInt64(&bp.status.getCount, 1) + + ss := up2power(size) + if ss < minSize { + ss = minSize + } + bucket := bp.capToFreeBucket[ss] + + buf := bucket.Get(size) + if buf == nil { + buf = make([]byte, size, ss) + return buf + } + + atomic.AddInt64(&bp.status.hitCount, 1) + atomic.AddInt64(&bp.status.sizeBytes, int64(-cap(buf))) + return buf +} + +func (bp *sliceBytePool) Put(buf []byte) { + c := cap(buf) + atomic.AddInt64(&bp.status.putCount, 1) + atomic.AddInt64(&bp.status.sizeBytes, int64(c)) + + size := down2power(c) + if size < minSize { + size = minSize + } + + bucket := bp.capToFreeBucket[size] + + bucket.Put(buf) +} + +func (bp *sliceBytePool) RetrieveStatus() Status { + return Status{ + getCount: atomic.LoadInt64(&bp.status.getCount), + putCount: atomic.LoadInt64(&bp.status.putCount), + hitCount: atomic.LoadInt64(&bp.status.hitCount), + sizeBytes: atomic.LoadInt64(&bp.status.sizeBytes), + } +} + +// @return 范围为 [2, 4, 8, 16, ..., 1073741824]，如果大于等于1073741824，则直接返回n +func up2power(n int) int { + if n >= maxSize { + return n + } + + var i uint32 + for ; n > (2 << i); i++ { + } + return 2 << i +} + +// @return 范围为 [2, 4, 8, 16, ..., 1073741824] +func down2power(n int) int { + if n < 2 { + return 2 + } else if n >= maxSize { + return maxSize + } + + var i uint32 + for { + nn := 2 << i + if n > nn { + i++ + } else if n == nn { + return n + } else if n < nn { + return 2 << (i - 1) + } + } +} diff --git a/pkg/slicebytepool/stdpool_bucket.go b/pkg/slicebytepool/stdpool_bucket.go new file mode 100644 index 0000000..6f42aa8 --- /dev/null +++ b/pkg/slicebytepool/stdpool_bucket.go @@ -0,0 +1,36 @@ +// Copyright 2019, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package slicebytepool + +import ( + "sync" +) + +type StdPoolBucket struct { + core *sync.Pool +} + +func NewStdPoolBucket() *StdPoolBucket { + return &StdPoolBucket{ + core: new(sync.Pool), + } +} + +func (b *StdPoolBucket) Get(size int) []byte { + v := b.core.Get() + if v == nil { + return nil + } + vv := v.([]byte) + return vv[0:size] +} + +func (b *StdPoolBucket) Put(buf []byte) { + b.core.Put(buf) +}