diff --git a/CHANGELOG.md b/CHANGELOG.md index fe9e2ab..507dd80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ - [perf] 减小锁粒度 - [test] 测试覆盖率增加至100% - [refactor] 删除FatalIfErrorNotNil, PanicIfErrorNotNil, Outputf, writeShortFile四个函数函数 -- package nazabit: +- package nazabits: - [feat] 新增GetBit16,GetBits16函数 - package fake: - [feat] 新增WithRecover函数 diff --git a/README.md b/README.md index 481ba3b..8432b9f 100644 --- a/README.md +++ b/README.md @@ -34,19 +34,20 @@ pkg/ ...... 源码包 |-- taskpool/ ...... 非阻塞协程池,协程数量可动态增长,可配置最大协程并发数量,可手动释放空闲的协程 |-- bele/ ...... 大小端转换操作 |-- nazabits/ ...... 位操作 + |-- bitrate/ ...... 计算带宽 + |-- lru ...... LRU缓存 |-- fake/ ...... 实现一些常用的接口,hook一些不方便测试的代码 + |-- consistenthash/ ...... 一致性哈希 + |-- assert/ ...... 提供了单元测试时的断言功能,减少一些模板代码 + |-- nazajson/ ...... json操作 + |-- nazahttp/ ...... http操作 + |-- filebatch/ ...... 文件批处理操作 |-- nazaatomic/ ...... 原子操作 |-- snowflake/ ...... 分布式唯一性ID生成器 - |-- bitrate/ ...... 计算带宽 - |-- consistenthash/ ...... 一致性哈希 |-- slicebytepool/ ...... []byte内存池 - |-- assert/ ...... 提供了单元测试时的断言功能,减少一些模板代码 |-- nazastring/ ...... string和[]byte相关的操作 |-- ratelimit/ ...... 限流器,令牌桶,漏桶 - |-- nazajson/ ...... json操作 - |-- filebatch/ ...... 文件批处理操作 |-- nazamd5/ ...... md5操作 - |-- nazahttp/ ...... http操作 |-- connection/ ...... 对net.Conn接口的二次封装 |-- ic/ ...... 将整型切片压缩成二进制字节切片 |-- unique/ ...... 对象唯一ID diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go new file mode 100644 index 0000000..61fa42b --- /dev/null +++ b/pkg/lru/lru.go @@ -0,0 +1,73 @@ +// Copyright 2020, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package lru + +import "container/list" + +type LRU struct { + c int // capacity + m map[interface{}]*list.Element // mapping key -> index + l *list.List // value +} + +type pair struct { + k interface{} + v interface{} +} + +func New(capacity int) *LRU { + return &LRU{ + c: capacity, + m: make(map[interface{}]*list.Element), + l: list.New(), + } +} + +// 注意: +// 1. 无论插入前,元素是否已经存在,插入后,元素都会存在于lru容器中 +// 2. 插入元素时,也会更新热度(不管插入前元素是否已经存在) +// @return 插入前元素已经存在则返回false +func (lru *LRU) Put(k interface{}, v interface{}) bool { + var ( + exist bool + e *list.Element + ) + e, exist = lru.m[k] + if exist { + lru.l.Remove(e) + delete(lru.m, k) + } + + // 头部更热 + e = lru.l.PushFront(pair{k, v}) + lru.m[k] = e + + if lru.l.Len() > lru.c { + k = lru.l.Back().Value.(pair).k + + lru.l.Remove(lru.l.Back()) + delete(lru.m, k) + } + + return !exist +} + +func (lru *LRU) Get(k interface{}) (v interface{}, exist bool) { + e, exist := lru.m[k] + if !exist { + return nil, false + } + pair := e.Value.(pair) + lru.l.MoveToFront(e) + return pair.v, true +} + +func (lru *LRU) Size() int { + return lru.l.Len() +} diff --git a/pkg/lru/lru_test.go b/pkg/lru/lru_test.go new file mode 100644 index 0000000..ca65b4b --- /dev/null +++ b/pkg/lru/lru_test.go @@ -0,0 +1,70 @@ +// Copyright 2020, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package lru_test + +import ( + "testing" + + "github.com/q191201771/naza/pkg/assert" + + "github.com/q191201771/naza/pkg/lru" +) + +func TestLRU(t *testing.T) { + l := lru.New(3) + l.Put("chef", 1) + l.Put("yoko", 2) + l.Put("tom", 3) + l.Put("jerry", 4) // 超过容器大小,淘汰最老的`chef` + + v, exist := l.Get("chef") + assert.Equal(t, false, exist) + + v, exist = l.Get("yoko") + assert.Equal(t, true, exist) + assert.Equal(t, 2, v.(int)) + + l.Put("garfield", 5) // 超过容器大小,注意,由于`yoko`刚才读取时会更新热度,所以淘汰的是`tom` + + v, exist = l.Get("yoko") + assert.Equal(t, true, exist) + assert.Equal(t, 2, v.(int)) + + v, exist = l.Get("tom") + assert.Equal(t, false, exist) + + l = lru.New(3) + v, exist = l.Get("notexist") + assert.Equal(t, false, exist) + assert.Equal(t, 0, l.Size()) + + l.Put("chef", 60) + assert.Equal(t, 1, l.Size()) + + v, exist = l.Get("chef") + assert.Equal(t, true, exist) + assert.Equal(t, 60, v.(int)) + assert.Equal(t, 1, l.Size()) + + v, exist = l.Get("ne") + assert.Equal(t, false, exist) + assert.Equal(t, 1, l.Size()) + + l.Put("yoko", 100) + assert.Equal(t, 2, l.Size()) + + l.Put("coco", 33) + assert.Equal(t, 3, l.Size()) + + l.Put("dad", 44) + assert.Equal(t, 3, l.Size()) + + isNewPut := l.Put("coco", 1000) + assert.Equal(t, false, isNewPut) +} diff --git a/pkg/nazahttp/http.go b/pkg/nazahttp/http.go index d85521f..b0a0c6b 100644 --- a/pkg/nazahttp/http.go +++ b/pkg/nazahttp/http.go @@ -10,12 +10,30 @@ package nazahttp import ( "io" + "io/ioutil" "net/http" "os" "time" ) -// 获取 http 文件保存至本地 +// 获取http文件 +// @return 成功返回nil +func GetHttpFile(url string, timeoutMSec int) ([]byte, error) { + var c http.Client + if timeoutMSec > 0 { + c.Timeout = time.Duration(timeoutMSec) * time.Millisecond + } + resp, err := c.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + content, err := ioutil.ReadAll(resp.Body) + return content, err +} + +// 获取http文件保存至本地 func DownloadHttpFile(url string, saveTo string, timeoutMSec int) (int64, error) { var c http.Client if timeoutMSec > 0 { diff --git a/pkg/nazahttp/http_test.go b/pkg/nazahttp/http_test.go index 08eb75f..2bb1741 100644 --- a/pkg/nazahttp/http_test.go +++ b/pkg/nazahttp/http_test.go @@ -15,6 +15,16 @@ import ( "github.com/q191201771/naza/pkg/nazahttp" ) +func TestGetHttpFile(t *testing.T) { + content, err := nazahttp.GetHttpFile("http://pengrl.com", 10000) + assert.IsNotNil(t, content) + assert.Equal(t, nil, err) + + content, err = nazahttp.GetHttpFile("http://127.0.0.1:12356", 10000) + assert.Equal(t, nil, content) + assert.IsNotNil(t, err) +} + func TestDownloadHttpFile(t *testing.T) { n, err := nazahttp.DownloadHttpFile("http://pengrl.com", "/tmp/index.html", 10000) assert.Equal(t, true, n > 0) @@ -23,6 +33,7 @@ func TestDownloadHttpFile(t *testing.T) { n, err = nazahttp.DownloadHttpFile("http://127.0.0.1:12356", "/tmp/index.html", 10000) assert.IsNotNil(t, err) + // 保存文件至不存在的本地目录下 n, err = nazahttp.DownloadHttpFile("http://pengrl.com", "/notexist/index.html", 10000) assert.IsNotNil(t, err) }