github-forks
/
naza
mirror of https://github.com/q191201771/naza
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

pkg ic patch

Browse Source
pull/2/head
q191201771 5 years ago
parent f6a3d64d3f
commit 119e26b529
6 changed files with 98 additions and 82 deletions
Show Stats Download Patch File Download Diff File

1
README.md
Unescape Escape View File

@ -42,6 +42,7 @@ pkg/ ...... 源码包
|-- filebatch/ ...... 文件批处理操作
|-- unique/ ...... 对象唯一 ID
|-- nazamd5/ ...... md5 操作
|-- ic/ ...... 将整型切片压缩成二进制字节切片
|-- mockserver/ ...... 模拟一些服务端，用于快速测试其它代码
|-- mockwriter/ ...... 模拟 Writer 接口，用于快速测试其它代码
demo/ ...... 示例相关的代码

11
pkg/ic/ic.go
Unescape Escape View File

@ -0,0 +1,11 @@
// package ic 将整型切片压缩成二进制字节切片
package ic
// 具体使用见 LFCompressor 和 OriginCompressor
type Compressor interface {
// 将整型切片压缩成二进制字节切片
Marshal(ids []uint32) (ret []byte)
// 将二进制字节切片反序列化为整型切片
// 反序列化后得到的整型切片，切片中整型的顺序和序列化之前保持不变
Unmarshal(b []byte) (ids []uint32)
}

103
pkg/ic/ic_test.go
Unescape Escape View File

@ -1,43 +1,39 @@
package ic
import (
"bytes"
"io/ioutil"
"log"
"os"
"strconv"
"testing"
)
// 从文件中读取 uid 列表
func obtainUIDList(filename string) (uids IDSlice) {
fp, err := os.Open(filename)
if err != nil {
panic(err)
}
buf, err := ioutil.ReadAll(fp)
if err != nil {
panic(err)
}
lines := bytes.Split(buf, []byte("\n"))
for _, line := range lines {
if len(line) == 0 {
continue
}
item, err := strconv.ParseUint(string(line), 10, 32)
if err != nil {
panic(err)
}
uids = append(uids, uint32(item))
}
return uids
}
//// 从文件中读取 uid 列表
//func obtainUIDList(filename string) (uids IDSlice) {
// fp, err := os.Open(filename)
// if err != nil {
// panic(err)
// }
// buf, err := ioutil.ReadAll(fp)
// if err != nil {
// panic(err)
// }
// lines := bytes.Split(buf, []byte("\n"))
// for _, line := range lines {
// if len(line) == 0 {
// continue
// }
// item, err := strconv.ParseUint(string(line), 10, 32)
// if err != nil {
// panic(err)
// }
// uids = append(uids, uint32(item))
// }
// return uids
//}
//var FILENAME = "uid.txt"
func marshalWrap(ids IDSlice) (ret []byte) {
func marshalWrap(ids []uint32) (ret []byte) {
log.Println("> sort.")
sortIDSlice(ids)
Sort(ids)
log.Println("< sort.")
log.Println("> marshal.")
@ -55,7 +51,7 @@ func marshalWrap(ids IDSlice) (ret []byte) {
return
}
func unmarshalWrap(b []byte) (ret IDSlice) {
func unmarshalWrap(b []byte) (ret []uint32) {
b = zlibRead(b)
//var oc OriginCompressor
@ -71,7 +67,7 @@ func TestIC(t *testing.T) {
log.SetFlags(log.Lmicroseconds)
// 单元测试 case
uidss := []IDSlice{
uidss := [][]uint32{
{1, 2, 3, 18, 32, 100},
{1, 2, 3, 18, 32},
{1, 2, 3, 18},
@ -86,30 +82,39 @@ func TestIC(t *testing.T) {
{1},
}
// 从文件加载 uid 白名单
//uids := obtainUIDList(FILENAME)
//var uidss []IDSlice
//uidss = append(uidss, uids)
var compressors []Compressor
for _, uids := range uidss {
log.Println("-----")
log.Println("in uid len:", len(uids))
compressors = append(compressors, &OriginCompressor{})
compressors = append(compressors, &OriginCompressor{ZlibExt: true})
b := marshalWrap(uids)
log.Println("len(b):", len(b))
compressors = append(compressors, &LFCompressor{FB: 0})
compressors = append(compressors, &LFCompressor{FB: 0, ZlibExt: true})
compressors = append(compressors, &LFCompressor{FB: 2})
compressors = append(compressors, &LFCompressor{FB: 4})
compressors = append(compressors, &LFCompressor{FB: 4, ZlibExt: true})
uids2 := unmarshalWrap(b)
log.Println("out uid len:", len(uids2))
for _, c := range compressors {
for _, uids := range uidss {
log.Println("-----")
log.Println("in uid len:", len(uids))
// assert check
if len(uids) != len(uids2) {
panic(0)
}
for i := range uids {
if uids[i] != uids2[i] {
Sort(uids)
b := c.Marshal(uids)
log.Println("len(b):", len(b))
uids2 := c.Unmarshal(b)
log.Println("out uid len:", len(uids2))
// assert check
if len(uids) != len(uids2) {
panic(0)
}
for i := range uids {
if uids[i] != uids2[i] {
panic(0)
}
}
log.Println("-----")
}
log.Println("-----")
}
}

21
pkg/ic/lf_compressor.go
Unescape Escape View File

@ -5,13 +5,20 @@ import (
)
type LFCompressor struct {
FB uint32 // 用几个字节的 bit 表示跟随的数据
FB uint32 // 用几个字节的 bit 表示跟随的数据
ZlibExt bool // 压缩之后，是否再用 zlib 进一步压缩
oc OriginCompressor // FB 为0时，退化成使用 OriginCompressor
}
func (lfc *LFCompressor) Marshal(ids IDSlice) (ret []byte) {
// 传入的整型切片必须是从小到大有序排列
func (lfc *LFCompressor) Marshal(ids []uint32) (ret []byte) {
if lfc.FB == 0 {
return lfc.oc.Marshal(ids)
ret = lfc.oc.Marshal(ids)
if lfc.ZlibExt {
ret = zlibWrite(ret)
}
return ret
}
lBuf := make([]byte, 4)
@ -62,10 +69,16 @@ func (lfc *LFCompressor) Marshal(ids IDSlice) (ret []byte) {
ret = append(ret, lBuf...)
ret = append(ret, fBuf...)
}
if lfc.ZlibExt {
ret = zlibWrite(ret)
}
return
}
func (lfc *LFCompressor) Unmarshal(b []byte) (ids IDSlice) {
func (lfc *LFCompressor) Unmarshal(b []byte) (ids []uint32) {
if lfc.ZlibExt {
b = zlibRead(b)
}
if lfc.FB == 0 {
return lfc.oc.Unmarshal(b)
}

12
pkg/ic/origin_compressor.go
Unescape Escape View File

@ -3,17 +3,25 @@ package ic
import "encoding/binary"
type OriginCompressor struct {
ZlibExt bool // 压缩之后，是否再用 zlib 进一步压缩
}
func (oc *OriginCompressor) Marshal(ids IDSlice) (ret []byte) {
// 并不强制要求整型切片有序
func (oc *OriginCompressor) Marshal(ids []uint32) (ret []byte) {
ret = make([]byte, len(ids)*4)
for i, id := range ids {
binary.LittleEndian.PutUint32(ret[i*4:], id)
}
if oc.ZlibExt {
ret = zlibWrite(ret)
}
return
}
func (oc *OriginCompressor) Unmarshal(b []byte) (ids IDSlice) {
func (oc *OriginCompressor) Unmarshal(b []byte) (ids []uint32) {
if oc.ZlibExt {
b = zlibRead(b)
}
n := len(b) / 4
for i := 0; i < n; i++ {
id := binary.LittleEndian.Uint32(b[i*4:])

32
pkg/ic/util.go
Unescape Escape View File

@ -7,11 +7,11 @@ import (
"sort"
)
type IDSlice []uint32
func (a IDSlice) Len() int { return len(a) }
func (a IDSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a IDSlice) Less(i, j int) bool { return a[i] < a[j] }
func Sort(ids []uint32) {
sort.Slice(ids, func(i, j int) bool {
return ids[i] < ids[j]
})
}
func resetBuf(b []byte) []byte {
for i := 0; i < len(b); i++ {
@ -20,10 +20,6 @@ func resetBuf(b []byte) []byte {
return b
}
func sortIDSlice(ids IDSlice) {
sort.Sort(ids)
}
func zlibWrite(in []byte) []byte {
var b bytes.Buffer
w := zlib.NewWriter(&b)
@ -38,21 +34,3 @@ func zlibRead(in []byte) (ret []byte) {
ret, _ = ioutil.ReadAll(r)
return
}
//func isBufEmpty(b []byte) bool {
// for i := 0; i < len(b); i++ {
// if b[i] != 0 {
// return false
// }
// }
// return true
//}
//
//func dumpIDSlice(ids IDSlice, filename string) {
// fp, _ := os.Create(filename)
// for _, id := range ids {
// _, _ = fp.WriteString(fmt.Sprintf("%d", id))
// _, _ = fp.WriteString("\n")
// }
// _ = fp.Close()
//}

Write Preview
Loading…
Cancel
Save