- 新增 package ratelimit:限速器,令牌桶

- 新增 package bitrate:计算带宽
- 新增 package fake
- 删除 package mockwriter
- 删除 package mockserver
- demo/add_blog_license:行尾增加两个空格,便于部分 markdown 解析器解析为为换行
pull/2/head
q191201771 5 years ago
parent b1349bbc76
commit 7722ee012a

2
.gitignore vendored

@ -12,4 +12,6 @@ profile*.pdf
/TODO.md
/pkg/tag
/pkg/nazatime
/demo/samefile
/demo/time

@ -40,11 +40,11 @@ pkg/ ...... 源码包
|-- taskpool/ ...... 协程池
|-- connection/ ...... 对 net.Conn 接口的二次封装
|-- filebatch/ ...... 文件批处理操作
|-- unique/ ...... 对象唯一 ID
|-- nazamd5/ ...... md5 操作
|-- ratelimit/ ...... 限流器,令牌桶
|-- ic/ ...... 将整型切片压缩成二进制字节切片
|-- mockserver/ ...... 模拟一些服务端,用于快速测试其它代码
|-- mockwriter/ ...... 模拟 Writer 接口,用于快速测试其它代码
|-- unique/ ...... 对象唯一 ID
|-- fake/ ...... 实现一些常用的接口,辅助测试其它代码
demo/ ...... 示例相关的代码
bin/ ...... 可执行文件编译输出目录
```

@ -34,6 +34,7 @@ cd ${ROOT_DIR}/demo/add_go_license && go build -ldflags "$LDFlags" -o ${ROOT_DIR
cd ${ROOT_DIR}/demo/taskpool && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/taskpool &&
cd ${ROOT_DIR}/demo/slicebytepool && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/slicebytepool &&
cd ${ROOT_DIR}/demo/myapp && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/myapp &&
#cd ${ROOT_DIR}/demo/time && go build -ldflags "$LDFlags" -o ${ROOT_DIR}/bin/time &&
ls -lrt ${ROOT_DIR}/bin &&
cd ${ROOT_DIR} && ./bin/myapp -v &&
echo 'build done.'

@ -23,9 +23,9 @@ import (
//> **声明:** 本文后续所有修改都会第一时间在原始地址更新。本文欢迎任何形式转载,转载时注明原始出处即可。`
var licenseTmpl = `
> **** [https://pengrl.com/p/%s/](https://pengrl.com/p/%s/)
> **** [yoko blog](https://pengrl.com) (https://pengrl.com)
> **** yoko
> **** [https://pengrl.com/p/%s/](https://pengrl.com/p/%s/)
> **** [yoko blog](https://pengrl.com) (https://pengrl.com)
> **** yoko
> **** `
func main() {

@ -0,0 +1,91 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package bitrate
import (
"sync"
"time"
)
// 先来个最简单的,后续再精细化配置:
//
// - 收包时间目前只能由内部获取当前时间,应提供接口支持外部传入
// - 返回的rate单位固定为 kbit/s
// - 不需要存储Time结构体存毫秒级的 unix 时间戳
type Bitrate struct {
option Option
mu sync.Mutex
bucketSlice []bucket
}
type Option struct {
WindowMS int
}
var defaultOption = Option{
WindowMS: 1000,
}
type bucket struct {
n int
t time.Time
}
type ModOption func(option *Option)
func NewBitrate(modOptions ...ModOption) *Bitrate {
option := defaultOption
for _, fn := range modOptions {
fn(&option)
}
return &Bitrate{
option: option,
}
}
func (b *Bitrate) Add(bytes int) {
now := time.Now()
b.mu.Lock()
defer b.mu.Unlock()
b.sweepStale(now)
b.bucketSlice = append(b.bucketSlice, bucket{
n: bytes,
t: now,
})
}
// @return 返回值单位 kbit/s
func (b *Bitrate) Rate() int {
now := time.Now()
b.mu.Lock()
defer b.mu.Unlock()
b.sweepStale(now)
var total int
for i := range b.bucketSlice {
total += b.bucketSlice[i].n
}
// total * 8 / 1000 * 1000 / b.windowMS
return total * 8 / b.option.WindowMS
}
func (b *Bitrate) sweepStale(now time.Time) {
for i := range b.bucketSlice {
if now.Sub(b.bucketSlice[i].t) > time.Duration(b.option.WindowMS)*time.Millisecond {
b.bucketSlice = b.bucketSlice[1:]
} else {
break
}
}
}

@ -0,0 +1,29 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package bitrate_test
import (
"testing"
"time"
"github.com/q191201771/naza/pkg/assert"
"github.com/q191201771/naza/pkg/bitrate"
)
func TestBitrate(t *testing.T) {
var b *bitrate.Bitrate
b = bitrate.NewBitrate(func(option *bitrate.Option) {
option.WindowMS = 10
})
b.Add(1000)
r := b.Rate()
assert.Equal(t, 800, r)
time.Sleep(100 * time.Millisecond)
b.Rate()
}

@ -6,15 +6,13 @@
//
// Author: Chef (191201771@qq.com)
package mockwriter
package fake
import (
"bytes"
"errors"
)
// TODO chef: 可以添加一个接口,获取内部 buffer 的内容
type WriterType uint8
const (
@ -24,28 +22,28 @@ const (
)
var (
ErrMockWriter = errors.New("naza.mockwriter: a mock error")
ErrFakeWriter = errors.New("naza.fake: a fake writer error")
)
type MockWriter struct {
type Writer struct {
t WriterType
ts map[uint32]WriterType
count uint32
b bytes.Buffer
B bytes.Buffer
}
func NewMockWriter(t WriterType) *MockWriter {
return &MockWriter{
func NewWriter(t WriterType) *Writer {
return &Writer{
t: t,
}
}
// 为某些写操作指定特定的类型,次数从 0 开始计数
func (w *MockWriter) SetSpecificType(ts map[uint32]WriterType) {
func (w *Writer) SetSpecificType(ts map[uint32]WriterType) {
w.ts = ts
}
func (w *MockWriter) Write(b []byte) (int, error) {
func (w *Writer) Write(b []byte) (int, error) {
t, exist := w.ts[w.count]
w.count++
if !exist {
@ -55,11 +53,8 @@ func (w *MockWriter) Write(b []byte) (int, error) {
case WriterTypeDoNothing:
return len(b), nil
case WriterTypeReturnError:
return 0, ErrMockWriter
//case WriterTypeIntoBuffer:
// return w.b.Write(b)
return 0, ErrFakeWriter
}
return w.b.Write(b)
//panic("never reach here.")
return w.B.Write(b)
}

@ -0,0 +1,83 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package fake_test
import (
"testing"
"github.com/q191201771/naza/pkg/fake"
"github.com/q191201771/naza/pkg/assert"
)
func TestNewWriter(t *testing.T) {
_ = fake.NewWriter(fake.WriterTypeDoNothing)
}
func TestWriter_Write(t *testing.T) {
var (
w *fake.Writer
n int
err error
b = []byte("hello")
)
w = fake.NewWriter(fake.WriterTypeDoNothing)
n, err = w.Write(b)
assert.Equal(t, 5, n)
assert.Equal(t, nil, err)
w = fake.NewWriter(fake.WriterTypeReturnError)
n, err = w.Write(b)
assert.Equal(t, 0, n)
assert.Equal(t, fake.ErrFakeWriter, err)
w = fake.NewWriter(fake.WriterTypeIntoBuffer)
n, err = w.Write(b)
assert.Equal(t, 5, n)
assert.Equal(t, nil, err)
}
func TestWriter_SetSpecificType(t *testing.T) {
var (
w *fake.Writer
n int
err error
b = []byte("hello")
)
w = fake.NewWriter(fake.WriterTypeDoNothing)
w.SetSpecificType(map[uint32]fake.WriterType{
0: fake.WriterTypeReturnError,
2: fake.WriterTypeReturnError,
4: fake.WriterTypeDoNothing,
})
expectedLen := map[int]int{
0: 0,
1: 5,
2: 0,
3: 5,
4: 5,
5: 5,
}
expectedErr := map[int]error{
0: fake.ErrFakeWriter,
1: nil,
2: fake.ErrFakeWriter,
3: nil,
4: nil,
5: nil,
}
for i := 0; i < 6; i++ {
n, err = w.Write(b)
assert.Equal(t, expectedLen[i], n)
assert.Equal(t, expectedErr[i], err)
}
}

@ -1,44 +0,0 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package mockserver
import (
"net"
"sync"
)
type MockAcceptServer struct {
l net.Listener
conns []net.Conn
m sync.Mutex
}
func (s *MockAcceptServer) Run(addr string) (err error) {
s.m.Lock()
s.l, err = net.Listen("tcp", addr)
s.m.Unlock()
if err != nil {
return
}
c, err := s.l.Accept()
if err != nil {
return
}
s.conns = append(s.conns, c)
return
}
func (s *MockAcceptServer) Dispose() {
s.m.Lock()
if s.l != nil {
s.l.Close()
}
s.m.Unlock()
}

@ -1,41 +0,0 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package mockserver
import (
"net"
"time"
"github.com/q191201771/naza/pkg/nazalog"
)
// 建立一个server端的监听在内部创建n个连接快速消耗掉listen队列达到对外模拟不处理连接的情况
type MockListenServer struct {
l net.Listener
}
func (s *MockListenServer) Run(addr string) (err error) {
s.l, err = net.Listen("tcp", addr)
if err != nil {
return
}
for i := 0; ; i++ {
if _, err := net.DialTimeout("tcp", addr, time.Duration(200)*time.Millisecond); err != nil {
nazalog.Infof("Dial failed. i=%d, err=%+v", i, err)
break
}
}
return
}
func (s *MockListenServer) Dispose() {
_ = s.l.Close()
}

@ -1,49 +0,0 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package mockserver
import (
"net"
"testing"
"time"
"github.com/q191201771/naza/pkg/assert"
)
var addr = ":10027"
func TestMockListenServer(t *testing.T) {
//var s MockListenServer
//err := s.Run(addr)
//assert.Equal(t, nil, err)
//defer s.Dispose()
//_, err = net.DialTimeout("tcp", addr, time.Duration(1000) * time.Millisecond)
//assert.IsNotNil(t, err)
}
func TestMockAcceptServer(t *testing.T) {
var s MockAcceptServer
var conns []net.Conn
go s.Run(addr)
for i := 0; i < 16; i++ {
c, err := net.DialTimeout("tcp", addr, time.Duration(1000)*time.Millisecond)
if err != nil {
break
}
//assert.Equal(t, nil, err)
conns = append(conns, c)
}
s.Dispose()
}
func TestCorner(t *testing.T) {
var s MockListenServer
err := s.Run("wrong addr")
assert.IsNotNil(t, err)
}

@ -1,105 +0,0 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package mockwriter
import (
"testing"
"github.com/q191201771/naza/pkg/assert"
)
func TestNewMockWriter(t *testing.T) {
_ = NewMockWriter(WriterTypeDoNothing)
}
func TestMockWriter_Write(t *testing.T) {
var (
w *MockWriter
n int
err error
b = []byte("hello")
)
w = NewMockWriter(WriterTypeDoNothing)
n, err = w.Write(b)
assert.Equal(t, 5, n)
assert.Equal(t, nil, err)
w = NewMockWriter(WriterTypeReturnError)
n, err = w.Write(b)
assert.Equal(t, 0, n)
assert.Equal(t, ErrMockWriter, err)
w = NewMockWriter(WriterTypeIntoBuffer)
n, err = w.Write(b)
assert.Equal(t, 5, n)
assert.Equal(t, nil, err)
}
func TestMockWriter_SetSpecificType(t *testing.T) {
var (
w *MockWriter
n int
err error
b = []byte("hello")
)
w = NewMockWriter(WriterTypeDoNothing)
w.SetSpecificType(map[uint32]WriterType{
0: WriterTypeReturnError,
2: WriterTypeReturnError,
4: WriterTypeDoNothing,
})
expectedLen := map[int]int{
0: 0,
1: 5,
2: 0,
3: 5,
4: 5,
5: 5,
}
expectedErr := map[int]error{
0: ErrMockWriter,
1: nil,
2: ErrMockWriter,
3: nil,
4: nil,
5: nil,
}
for i := 0; i < 6; i++ {
n, err = w.Write(b)
assert.Equal(t, expectedLen[i], n)
assert.Equal(t, expectedErr[i], err)
}
}
func BenchmarkNewMockWriter(b *testing.B) {
b.ReportAllocs()
var tmp uint32
for i := 0; i < b.N; i++ {
mw := NewMockWriter(WriterTypeDoNothing)
tmp = tmp + mw.count
}
}
func newMockWriter2(t WriterType) MockWriter {
return MockWriter{
t: t,
}
}
func BenchmarkNewMockWriter2(b *testing.B) {
b.ReportAllocs()
var tmp uint32
for i := 0; i < b.N; i++ {
mw := newMockWriter2(WriterTypeDoNothing)
tmp = tmp + mw.count
}
}

@ -0,0 +1,70 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package ratelimit
import (
"sync"
"time"
)
type RateLimit struct {
num int
option Option
neededWait time.Duration
mu sync.Mutex
last time.Time
}
type Option struct {
Duration time.Duration
}
var defaultOption = Option{
Duration: 1 * time.Second,
}
type ModOption func(option *Option)
func New(num int, modOptions ...ModOption) *RateLimit {
option := defaultOption
for _, fn := range modOptions {
fn(&option)
}
return &RateLimit{
num: num,
option: option,
neededWait: option.Duration / time.Duration(num),
}
}
func (rl *RateLimit) Wait() {
rl.mu.Lock()
now := time.Now()
if rl.last.IsZero() {
rl.last = now
rl.mu.Unlock()
return
}
diff := now.Sub(rl.last)
if diff > rl.neededWait {
rl.last = now
rl.mu.Unlock()
return
}
rl.last = rl.last.Add(rl.neededWait)
rl.mu.Unlock()
t := time.NewTimer(rl.neededWait - diff)
<-t.C
t.Stop()
return
}

@ -0,0 +1,100 @@
// Copyright 2019, Chef. All rights reserved.
// https://github.com/q191201771/naza
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)
package ratelimit_test
import (
"math/rand"
"sync"
"testing"
"time"
"github.com/q191201771/naza/pkg/nazalog"
"github.com/q191201771/naza/pkg/ratelimit"
)
var (
duration = 100 * time.Millisecond
num = 50
)
func TestNew(t *testing.T) {
rl := ratelimit.New(num)
rl = ratelimit.New(num, func(option *ratelimit.Option) {
option.Duration = duration
})
rl.Wait()
}
func TestRateLimit_Wait(t *testing.T) {
rl := ratelimit.New(num, func(option *ratelimit.Option) {
option.Duration = duration
})
b := time.Now()
for i := 0; i < num; i++ {
rl.Wait()
}
nazalog.Debugf("cost:%v", time.Now().Sub(b))
}
func TestRateLimit_Wait2(t *testing.T) {
rl := ratelimit.New(num, func(option *ratelimit.Option) {
option.Duration = duration
})
b := time.Now()
var wg sync.WaitGroup
wg.Add(num)
for i := 0; i < num; i++ {
go func(ii int) {
rl.Wait()
wg.Done()
}(i)
}
wg.Wait()
nazalog.Debugf("cost:%v", time.Now().Sub(b))
}
func TestRateLimit_Wait3(t *testing.T) {
rand.Seed(time.Now().Unix())
rl := ratelimit.New(num, func(option *ratelimit.Option) {
option.Duration = duration
})
b := time.Now()
var wg sync.WaitGroup
wg.Add(num)
for i := 0; i < num; i++ {
go func(ii int) {
time.Sleep(time.Duration(rand.Int63n(int64(duration) / 2)))
rl.Wait()
wg.Done()
}(i)
}
wg.Wait()
nazalog.Debugf("cost:%v", time.Now().Sub(b))
}
func TestRateLimit_Wait4(t *testing.T) {
rl := ratelimit.New(num, func(option *ratelimit.Option) {
option.Duration = duration
})
b := time.Now()
for i := 0; i < num; i++ {
time.Sleep(time.Duration(int64(duration) * 2 / int64(num)))
rl.Wait()
}
nazalog.Debugf("cost:%v", time.Now().Sub(b))
}
func BenchmarkRateLimit_Wait(b *testing.B) {
for i := 0; i < b.N; i++ {
rl := ratelimit.New(num)
for i := 0; i < num; i++ {
rl.Wait()
}
}
}
Loading…
Cancel
Save