From ee31064541f0404440c0510eea9386f435e50c73 Mon Sep 17 00:00:00 2001 From: q191201771 <191201771@qq.com> Date: Sat, 17 Apr 2021 11:20:03 +0800 Subject: [PATCH] =?UTF-8?q?[feat]=20package=20filesystemlayer:=20=E6=96=B0?= =?UTF-8?q?=E7=9A=84=E5=8C=85=EF=BC=8C=E6=8F=90=E4=BE=9B=E4=B8=80=E5=B1=82?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=9A=84=E6=8A=BD=E8=B1=A1?= =?UTF-8?q?=EF=BC=8C=E5=8F=AF=E4=BB=A5=E4=BD=BF=E7=94=A8=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E7=A3=81=E7=9B=98=E4=BD=9C=E4=B8=BA=E5=AD=98?= =?UTF-8?q?=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/filesystemlayer/disk.go | 49 +++++++++++ pkg/filesystemlayer/fsl.go | 17 ++++ pkg/filesystemlayer/fsl_test.go | 112 ++++++++++++++++++++++++ pkg/filesystemlayer/global.go | 50 +++++++++++ pkg/filesystemlayer/interface.go | 50 +++++++++++ pkg/filesystemlayer/memory.go | 144 +++++++++++++++++++++++++++++++ 6 files changed, 422 insertions(+) create mode 100644 pkg/filesystemlayer/disk.go create mode 100644 pkg/filesystemlayer/fsl.go create mode 100644 pkg/filesystemlayer/fsl_test.go create mode 100644 pkg/filesystemlayer/global.go create mode 100644 pkg/filesystemlayer/interface.go create mode 100644 pkg/filesystemlayer/memory.go diff --git a/pkg/filesystemlayer/disk.go b/pkg/filesystemlayer/disk.go new file mode 100644 index 0000000..0bd51cc --- /dev/null +++ b/pkg/filesystemlayer/disk.go @@ -0,0 +1,49 @@ +// Copyright 2021, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package filesystemlayer + +import ( + "io/ioutil" + "os" +) + +type FSLDisk struct { +} + +func (f *FSLDisk) Type() FSLType { + return FSLTypeDisk +} + +func (f *FSLDisk) Create(name string) (IFile, error) { + return os.Create(name) +} + +func (f *FSLDisk) Rename(oldpath string, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func (f *FSLDisk) MkdirAll(path string, perm uint32) error { + return os.MkdirAll(path, os.FileMode(perm)) +} + +func (f *FSLDisk) Remove(name string) error { + return os.Remove(name) +} + +func (f *FSLDisk) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (f *FSLDisk) ReadFile(filename string) ([]byte, error) { + return ioutil.ReadFile(filename) +} + +func (f *FSLDisk) WriteFile(filename string, data []byte, perm uint32) error { + return ioutil.WriteFile(filename, data, os.FileMode(perm)) +} diff --git a/pkg/filesystemlayer/fsl.go b/pkg/filesystemlayer/fsl.go new file mode 100644 index 0000000..c73a0cd --- /dev/null +++ b/pkg/filesystemlayer/fsl.go @@ -0,0 +1,17 @@ +// Copyright 2021, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package filesystemlayer + +import "os" + +var _ IFileSystemLayer = &FSLDisk{} +var _ IFileSystemLayer = &FSLMemory{} + +var _ IFile = &os.File{} +var _ IFile = &file{} diff --git a/pkg/filesystemlayer/fsl_test.go b/pkg/filesystemlayer/fsl_test.go new file mode 100644 index 0000000..aa5d3f2 --- /dev/null +++ b/pkg/filesystemlayer/fsl_test.go @@ -0,0 +1,112 @@ +// Copyright 2021, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package filesystemlayer_test + +import ( + "fmt" + "sync" + "testing" + + "github.com/q191201771/naza/pkg/nazalog" + + "github.com/q191201771/naza/pkg/assert" + + "github.com/q191201771/naza/pkg/filesystemlayer" +) + +func TestCase1(t *testing.T) { + fslCtx := filesystemlayer.FSLFactory(filesystemlayer.FSLTypeMemory) + + var wg sync.WaitGroup + wg.Add(16) + + for i := 0; i < 16; i++ { + go func(ii int) { + dir := fmt.Sprintf("/tmp/lal/hls/test%d", ii) + err := fslCtx.MkdirAll(dir, 0777) + assert.Equal(t, nil, err) + + for j := 0; j < 32; j++ { + filename := fmt.Sprintf("/tmp/lal/hls/test%d/%d.ts", ii, j) + nazalog.Infof("%d %d %s", ii, j, filename) + fp, err := fslCtx.Create(filename) + assert.Equal(t, nil, err) + + n, err := fp.Write([]byte("hello")) + assert.Equal(t, nil, err) + assert.Equal(t, 5, n) + + n, err = fp.Write([]byte("world")) + assert.Equal(t, nil, err) + assert.Equal(t, 5, n) + + err = fp.Close() + assert.Equal(t, nil, err) + } + + wg.Done() + }(i) + } + + wg.Wait() + + // 正常读 + b, err := fslCtx.ReadFile("/tmp/lal/hls/test1/1.ts") + assert.Equal(t, nil, err) + assert.Equal(t, []byte("helloworld"), b) + + // 删文件 + err = fslCtx.Remove("/tmp/lal/hls/test1/1.ts") + assert.Equal(t, nil, err) + b, err = fslCtx.ReadFile("/tmp/lal/hls/test1/1.ts") + assert.Equal(t, filesystemlayer.ErrNotFound, err) + assert.Equal(t, nil, b) + + // 正常读 + b, err = fslCtx.ReadFile("/tmp/lal/hls/test2/2.ts") + assert.Equal(t, nil, err) + assert.Equal(t, []byte("helloworld"), b) + + // 文件重命名 + err = fslCtx.Rename("/tmp/lal/hls/test2/2.ts", "/tmp/lal/hls/test2/new2.ts") + b, err = fslCtx.ReadFile("/tmp/lal/hls/test2/2.ts") + assert.Equal(t, filesystemlayer.ErrNotFound, err) + assert.Equal(t, nil, b) + b, err = fslCtx.ReadFile("/tmp/lal/hls/test2/new2.ts") + assert.Equal(t, nil, err) + assert.Equal(t, []byte("helloworld"), b) + + // 删文件夹 + err = fslCtx.RemoveAll("/tmp/lal/hls/test1") + assert.Equal(t, nil, err) + b, err = fslCtx.ReadFile("/tmp/lal/hls/test1/1.ts") + assert.Equal(t, filesystemlayer.ErrNotFound, err) + assert.Equal(t, nil, b) + + // 创建已经存在的文件 + fp, err := fslCtx.Create("/tmp/lal/hls/test3/3.ts") + assert.Equal(t, nil, err) + b, err = fslCtx.ReadFile("/tmp/lal/hls/test3/3.ts") + assert.Equal(t, nil, err) + assert.Equal(t, nil, b) + n, err := fp.Write([]byte("asd")) + assert.Equal(t, nil, err) + assert.Equal(t, 3, n) + b, err = fslCtx.ReadFile("/tmp/lal/hls/test3/3.ts") + assert.Equal(t, nil, err) + assert.Equal(t, []byte("asd"), b) + + // 删除不存在的文件 + err = fslCtx.Remove("/tmp/lal/hls/test1/1.ts") + assert.Equal(t, filesystemlayer.ErrNotFound, err) + + // 重命名不存在的文件 + err = fslCtx.Rename("/tmp/lal/hls/test1/1.ts", "/tmp/lal/hls/test1/new1.ts") + assert.Equal(t, filesystemlayer.ErrNotFound, err) +} diff --git a/pkg/filesystemlayer/global.go b/pkg/filesystemlayer/global.go new file mode 100644 index 0000000..9144ee9 --- /dev/null +++ b/pkg/filesystemlayer/global.go @@ -0,0 +1,50 @@ +// Copyright 2021, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package filesystemlayer + +//var global IFileSystemLayer +// +//func Config(t FSLType) { +// global = FSLFactory(t) +//} +// +//func Type() FSLType { +// return global.Type() +//} +// +//func Create(name string) (IFile, error) { +// return global.Create(name) +//} +//func Rename(oldpath string, newpath string) error { +// return global.Rename(oldpath, newpath) +//} +// +//func MkdirAll(path string, perm uint32) error { +// return MkdirAll(path, perm) +//} +// +//func Remove(name string) error { +// return Remove(name) +//} +// +//func RemoveAll(path string) error { +// return RemoveAll(path) +//} +// +//func ReadFile(filename string) ([]byte, error) { +// return global.ReadFile(filename) +//} +// +//func WriteFile(filename string, data []byte, perm uint32) error { +// return global.WriteFile(filename, data, perm) +//} +// +//func init() { +// global = FSLFactory(FSLTypeDisk) +//} diff --git a/pkg/filesystemlayer/interface.go b/pkg/filesystemlayer/interface.go new file mode 100644 index 0000000..8d22868 --- /dev/null +++ b/pkg/filesystemlayer/interface.go @@ -0,0 +1,50 @@ +// Copyright 2021, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package filesystemlayer + +// 注意，这个package并没有完整实现所有的文件操作，使用内存作为存储时，存在一些限制 +// 目前只是服务于我另一个项目中的特定场景 https://github.com/q191201771/lal + +type IFileSystemLayer interface { + Type() FSLType + + // 创建文件 + // 原始语义：如果文件已经存在，原文件内容被清空 + Create(name string) (IFile, error) + + Rename(oldpath string, newpath string) error + MkdirAll(path string, perm uint32) error + Remove(name string) error + RemoveAll(path string) error + + ReadFile(filename string) ([]byte, error) + WriteFile(filename string, data []byte, perm uint32) error +} + +type IFile interface { + Write(b []byte) (n int, err error) + Close() error +} + +type FSLType int + +const ( + FSLTypeDisk FSLType = 1 + FSLTypeMemory = 2 +) + +func FSLFactory(t FSLType) IFileSystemLayer { + switch t { + case FSLTypeDisk: + return &FSLDisk{} + case FSLTypeMemory: + return NewFSLMemory() + } + return nil +} diff --git a/pkg/filesystemlayer/memory.go b/pkg/filesystemlayer/memory.go new file mode 100644 index 0000000..338e3db --- /dev/null +++ b/pkg/filesystemlayer/memory.go @@ -0,0 +1,144 @@ +// Copyright 2021, Chef. All rights reserved. +// https://github.com/q191201771/naza +// +// Use of this source code is governed by a MIT-style license +// that can be found in the License file. +// +// Author: Chef (191201771@qq.com) + +package filesystemlayer + +import ( + "errors" + "fmt" + "os" + "strings" + "sync" +) + +var ErrNotFound = errors.New("naza filesystemlayer: not found") + +type FSLMemory struct { + mu sync.Mutex + files map[string]*file // key filename +} + +type file struct { + buf []byte +} + +func NewFSLMemory() *FSLMemory { + return &FSLMemory{ + files: make(map[string]*file), + } +} + +func (f *FSLMemory) Type() FSLType { + return FSLTypeMemory +} + +func (f *FSLMemory) Create(name string) (IFile, error) { + return f.openFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} + +func (f *FSLMemory) Rename(oldpath string, newpath string) error { + f.mu.Lock() + defer f.mu.Unlock() + fi, exist := f.files[oldpath] + if !exist { + return ErrNotFound + } + delete(f.files, oldpath) + f.files[newpath] = fi + return nil +} + +func (f *FSLMemory) MkdirAll(path string, perm uint32) error { + return nil +} + +func (f *FSLMemory) Remove(name string) error { + f.mu.Lock() + defer f.mu.Unlock() + _, exist := f.files[name] + if !exist { + return ErrNotFound + } + delete(f.files, name) + return nil +} + +func (f *FSLMemory) RemoveAll(path string) error { + if !os.IsPathSeparator(path[len(path)-1]) { + path = fmt.Sprintf("%s%c", path, os.PathSeparator) + } + f.mu.Lock() + defer f.mu.Unlock() + files := make(map[string]*file) + for k, v := range f.files { + if !strings.HasPrefix(k, path) { + files[k] = v + } + } + f.files = files + + return nil +} + +func (f *FSLMemory) ReadFile(filename string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + fi, exist := f.files[filename] + if !exist { + return nil, ErrNotFound + } + return fi.clone(), nil +} + +func (f *FSLMemory) WriteFile(filename string, data []byte, perm uint32) error { + fi, err := f.openFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = fi.Write(data) + if err1 := fi.Close(); err == nil { + err = err1 + } + return err +} + +func (f *FSLMemory) openFile(name string, flag int, perm uint32) (IFile, error) { + f.mu.Lock() + defer f.mu.Unlock() + fi, ok := f.files[name] + if !ok { + fi = &file{} + f.files[name] = fi + return fi, nil + } + + fi.truncate() + return fi, nil +} + +func (f *file) Write(b []byte) (n int, err error) { + f.buf = append(f.buf, b...) + return len(b), nil +} + +func (f *file) Close() error { + return nil +} + +func (f *file) truncate() { + f.buf = nil +} + +func (f *file) clone() []byte { + if f.buf == nil { + return nil + } + b := make([]byte, len(f.buf)) + copy(b, f.buf) + return b +}