This commit is contained in:
ljw
2024-09-13 15:57:29 +08:00
commit c53df223d1
112 changed files with 14353 additions and 0 deletions

71
lib/cache/cache.go vendored Normal file
View File

@@ -0,0 +1,71 @@
package cache
import (
"encoding/json"
)
type Handler interface {
Get(key string, value interface{}) error
Set(key string, value interface{}, exp int) error
Gc() error
}
// MaxTimeOut 最大超时时间
const (
TypeMem = "memory"
TypeRedis = "redis"
TypeFile = "file"
MaxTimeOut = 365 * 24 * 3600
)
func New(typ string) Handler {
var cache Handler
switch typ {
case TypeFile:
cache = NewFileCache()
case TypeRedis:
cache = new(RedisCache)
case TypeMem: // memory
cache = NewMemoryCache(0)
default:
cache = NewMemoryCache(0)
}
return cache
}
func EncodeValue(value interface{}) (string, error) {
/*if v, ok := value.(string); ok {
return v, nil
}
if v, ok := value.([]byte); ok {
return string(v), nil
}*/
b, err := json.Marshal(value)
if err != nil {
return "", err
}
return string(b), nil
}
func DecodeValue(value string, rtv interface{}) error {
//判断rtv的类型是否是string如果是string直接赋值并返回
/*switch rtv.(type) {
case *string:
*(rtv.(*string)) = value
return nil
case *[]byte:
*(rtv.(*[]byte)) = []byte(value)
return nil
//struct
case *interface{}:
err := json.Unmarshal(([]byte)(value), rtv)
return err
default:
err := json.Unmarshal(([]byte)(value), rtv)
return err
}
*/
err := json.Unmarshal(([]byte)(value), rtv)
return err
}

92
lib/cache/cache_test.go vendored Normal file
View File

@@ -0,0 +1,92 @@
package cache
import (
"fmt"
"github.com/go-redis/redis/v8"
"reflect"
"testing"
)
func TestSimpleCache(t *testing.T) {
type st struct {
A string
B string
}
items := map[string]interface{}{}
items["a"] = "b"
items["b"] = "c"
ab := &st{
A: "a",
B: "b",
}
items["ab"] = *ab
a := items["a"]
fmt.Println(a)
b := items["b"]
fmt.Println(b)
ab.A = "aa"
ab2 := st{}
ab2 = (items["ab"]).(st)
fmt.Println(ab2, reflect.TypeOf(ab2))
}
func TestFileCacheSet(t *testing.T) {
fc := New("file")
err := fc.Set("123", "ddd", 0)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("写入失败")
}
}
func TestFileCacheGet(t *testing.T) {
fc := New("file")
err := fc.Set("123", "45156", 300)
if err != nil {
t.Fatalf("写入失败")
}
res := ""
err = fc.Get("123", &res)
if err != nil {
t.Fatalf("读取失败")
}
fmt.Println("res", res)
}
func TestRedisCacheSet(t *testing.T) {
rc := NewRedis(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
err := rc.Set("123", "ddd", 0)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("写入失败")
}
}
func TestRedisCacheGet(t *testing.T) {
rc := NewRedis(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
err := rc.Set("123", "451156", 300)
if err != nil {
t.Fatalf("写入失败")
}
res := ""
err = rc.Get("123", &res)
if err != nil {
t.Fatalf("读取失败")
}
fmt.Println("res", res)
}

103
lib/cache/file.go vendored Normal file
View File

@@ -0,0 +1,103 @@
package cache
import (
"crypto/md5"
"fmt"
"os"
"sync"
"time"
)
type FileCache struct {
mu sync.Mutex
locks map[string]*sync.Mutex
Dir string
}
func (fc *FileCache) getLock(key string) *sync.Mutex {
fc.mu.Lock()
defer fc.mu.Unlock()
if fc.locks == nil {
fc.locks = make(map[string]*sync.Mutex)
}
if _, ok := fc.locks[key]; !ok {
fc.locks[key] = new(sync.Mutex)
}
return fc.locks[key]
}
func (c *FileCache) Get(key string, value interface{}) error {
data, _ := c.getValue(key)
err := DecodeValue(data, value)
return err
}
// 获取值,如果文件不存在或者过期,返回空,过滤掉错误
func (c *FileCache) getValue(key string) (string, error) {
f := c.fileName(key)
fileInfo, err := os.Stat(f)
if err != nil {
//文件不存在
return "", nil
}
difT := time.Now().Sub(fileInfo.ModTime())
if difT >= 0 {
os.Remove(f)
return "", nil
}
data, err := os.ReadFile(f)
if err != nil {
return "", nil
}
return string(data), nil
}
// 保存值
func (c *FileCache) saveValue(key string, value string, exp int) error {
f := c.fileName(key)
lock := c.getLock(f)
lock.Lock()
defer lock.Unlock()
err := os.WriteFile(f, ([]byte)(value), 0644)
if err != nil {
return err
}
if exp <= 0 {
exp = MaxTimeOut
}
expFromNow := time.Now().Add(time.Duration(exp) * time.Second)
err = os.Chtimes(f, expFromNow, expFromNow)
return err
}
func (c *FileCache) Set(key string, value interface{}, exp int) error {
str, err := EncodeValue(value)
if err != nil {
return err
}
err = c.saveValue(key, str, exp)
return err
}
func (c *FileCache) SetDir(path string) {
c.Dir = path
}
func (c *FileCache) fileName(key string) string {
f := c.Dir + string(os.PathSeparator) + fmt.Sprintf("%x", md5.Sum([]byte(key)))
return f
}
func (c *FileCache) Gc() error {
//检查文件过期时间,并删除
return nil
}
func NewFileCache() *FileCache {
return &FileCache{
locks: make(map[string]*sync.Mutex),
Dir: os.TempDir(),
}
}

94
lib/cache/file_test.go vendored Normal file
View File

@@ -0,0 +1,94 @@
package cache
import (
"fmt"
"reflect"
"testing"
)
func TestFileSet(t *testing.T) {
fc := NewFileCache()
err := fc.Set("123", "ddd", 0)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("写入失败")
}
}
func TestFileGet(t *testing.T) {
fc := NewFileCache()
res := ""
err := fc.Get("123", &res)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("读取失败")
}
fmt.Println("res", res)
}
func TestFileSetGet(t *testing.T) {
fc := NewFileCache()
err := fc.Set("key1", "ddd", 0)
res := ""
err = fc.Get("key1", &res)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("读取失败")
}
fmt.Println("res", res)
}
func TestFileGetJson(t *testing.T) {
fc := NewFileCache()
old := &r{
A: "a", B: "b",
}
fc.Set("123", old, 0)
res := &r{}
err2 := fc.Get("123", res)
fmt.Println("res", res)
if err2 != nil {
t.Fatalf("读取失败" + err2.Error())
}
}
func TestFileSetGetJson(t *testing.T) {
fc := NewFileCache()
old_rr := &rr{AA: "aa", BB: "bb"}
old := &r{
A: "a", B: "b",
R: old_rr,
}
err := fc.Set("123", old, 300)
if err != nil {
t.Fatalf("写入失败")
}
//old_rr.AA = "aaa"
fmt.Println("old_rr", old)
res := &r{}
err2 := fc.Get("123", res)
fmt.Println("res", res)
if err2 != nil {
t.Fatalf("读取失败" + err2.Error())
}
if !reflect.DeepEqual(res, old) {
t.Fatalf("读取错误")
}
}
func BenchmarkSet(b *testing.B) {
fc := NewFileCache()
b.ResetTimer()
for i := 0; i < b.N; i++ {
fc.Set("123", "{dsv}", 1000)
}
}
func BenchmarkGet(b *testing.B) {
fc := NewFileCache()
b.ResetTimer()
v := ""
for i := 0; i < b.N; i++ {
fc.Get("123", &v)
}
}

215
lib/cache/memory.go vendored Normal file
View File

@@ -0,0 +1,215 @@
package cache
import (
"container/heap"
"container/list"
"errors"
"reflect"
"sync"
"time"
)
type MemoryCache struct {
data map[string]*CacheItem
ll *list.List // 用于实现LRU
pq PriorityQueue // 用于实现TTL
quit chan struct{}
mu sync.Mutex
maxBytes int64
usedBytes int64
}
type CacheItem struct {
Key string
Value string
Expiration int64
Index int
ListEle *list.Element
}
type PriorityQueue []*CacheItem
func (pq PriorityQueue) Len() int { return len(pq) }
func (pq PriorityQueue) Less(i, j int) bool {
return pq[i].Expiration < pq[j].Expiration
}
func (pq PriorityQueue) Swap(i, j int) {
pq[i], pq[j] = pq[j], pq[i]
pq[i].Index = i
pq[j].Index = j
}
func (pq *PriorityQueue) Push(x interface{}) {
item := x.(*CacheItem)
item.Index = len(*pq)
*pq = append(*pq, item)
}
func (pq *PriorityQueue) Pop() interface{} {
old := *pq
n := len(old)
item := old[n-1]
old[n-1] = nil // avoid memory leak
item.Index = -1 // for safety
*pq = old[0 : n-1]
return item
}
func (m *MemoryCache) Get(key string, value interface{}) error {
// 使用反射将存储的值设置到传入的指针变量中
val := reflect.ValueOf(value)
if val.Kind() != reflect.Ptr {
return errors.New("value must be a pointer")
}
//设为空值
val.Elem().Set(reflect.Zero(val.Elem().Type()))
m.mu.Lock()
defer m.mu.Unlock()
if m.data == nil {
return nil
}
if item, ok := m.data[key]; ok {
if item.Expiration < time.Now().UnixNano() {
m.deleteItem(item)
return nil
}
//移动到队列尾部
m.ll.MoveToBack(item.ListEle)
err := DecodeValue(item.Value, value)
if err != nil {
return err
}
}
return nil
}
func (m *MemoryCache) Set(key string, value interface{}, exp int) error {
m.mu.Lock()
defer m.mu.Unlock()
v, err := EncodeValue(value)
if err != nil {
return err
}
//key 所占用的内存
keyBytes := int64(len(key))
//value所占用的内存空间大小
valueBytes := int64(len(v))
//判断是否超过最大内存限制
if m.maxBytes != 0 && m.maxBytes < keyBytes+valueBytes {
return errors.New("exceed maxBytes")
}
m.usedBytes += keyBytes + valueBytes
if m.maxBytes != 0 && m.usedBytes > m.maxBytes {
m.RemoveOldest()
}
if exp <= 0 {
exp = MaxTimeOut
}
expiration := time.Now().Add(time.Duration(exp) * time.Second).UnixNano()
item, exists := m.data[key]
if exists {
item.Value = v
item.Expiration = expiration
heap.Fix(&m.pq, item.Index)
m.ll.MoveToBack(item.ListEle)
} else {
ele := m.ll.PushBack(key)
item = &CacheItem{
Key: key,
Value: v,
Expiration: expiration,
ListEle: ele,
}
m.data[key] = item
heap.Push(&m.pq, item)
}
return nil
}
func (m *MemoryCache) RemoveOldest() {
for m.maxBytes != 0 && m.usedBytes > m.maxBytes {
elem := m.ll.Front()
if elem != nil {
key := elem.Value.(string)
item := m.data[key]
m.deleteItem(item)
}
}
}
// evictExpiredItems removes all expired items from the cache.
func (m *MemoryCache) evictExpiredItems() {
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now().UnixNano()
for m.pq.Len() > 0 {
item := m.pq[0]
if item.Expiration > now {
break
}
m.deleteItem(item)
}
}
// startEviction starts a goroutine that evicts expired items from the cache.
func (m *MemoryCache) startEviction() {
ticker := time.NewTicker(1 * time.Second)
go func() {
for {
select {
case <-ticker.C:
m.evictExpiredItems()
case <-m.quit:
ticker.Stop()
return
}
}
}()
}
// stopEviction 停止定时清理
func (m *MemoryCache) stopEviction() {
close(m.quit)
}
// deleteItem removes a key from the cache.
func (m *MemoryCache) deleteItem(item *CacheItem) {
m.ll.Remove(item.ListEle)
m.usedBytes -= int64(len(item.Key)) + int64(len(item.Value))
heap.Remove(&m.pq, item.Index)
delete(m.data, item.Key)
}
func (m *MemoryCache) Gc() error {
m.mu.Lock()
defer m.mu.Unlock()
m.data = make(map[string]*CacheItem)
m.ll = list.New()
m.pq = make(PriorityQueue, 0)
heap.Init(&m.pq)
m.usedBytes = 0
return nil
}
// NewMemoryCache creates a new MemoryCache.default maxBytes is 0, means no limit.
func NewMemoryCache(maxBytes int64) *MemoryCache {
cache := &MemoryCache{
data: make(map[string]*CacheItem),
pq: make(PriorityQueue, 0),
quit: make(chan struct{}),
ll: list.New(),
maxBytes: maxBytes,
}
heap.Init(&cache.pq)
cache.startEviction()
return cache
}

107
lib/cache/memory_test.go vendored Normal file
View File

@@ -0,0 +1,107 @@
package cache
import (
"fmt"
"testing"
"time"
)
func TestMemorySet(t *testing.T) {
mc := NewMemoryCache(0)
err := mc.Set("123", "44567", 0)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("写入失败")
}
}
func TestMemoryGet(t *testing.T) {
mc := NewMemoryCache(0)
mc.Set("123", "44567", 0)
res := ""
err := mc.Get("123", &res)
fmt.Println("res", res)
if err != nil {
t.Fatalf("读取失败 " + err.Error())
}
if res != "44567" {
t.Fatalf("读取错误")
}
}
func TestMemorySetExpGet(t *testing.T) {
mc := NewMemoryCache(0)
//mc.stopEviction()
mc.Set("1", "10", 10)
mc.Set("2", "5", 5)
err := mc.Set("3", "3", 3)
if err != nil {
t.Fatalf("写入失败")
}
res := ""
err = mc.Get("3", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res 3", res)
time.Sleep(4 * time.Second)
//res = ""
err = mc.Get("3", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res 3", res)
err = mc.Get("2", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res 2", res)
err = mc.Get("1", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res 1", res)
}
func TestMemoryLru(t *testing.T) {
mc := NewMemoryCache(18)
mc.Set("1", "1111", 10)
mc.Set("2", "2222", 5)
//读取一次2就会被放到最后
mc.Get("1", nil)
err := mc.Set("3", "三", 3)
if err != nil {
//t.Fatalf("写入失败")
}
res := ""
err = mc.Get("3", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res3", res)
res = ""
err = mc.Get("2", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res2", res)
res = ""
err = mc.Get("1", &res)
if err != nil {
t.Fatalf("读取失败" + err.Error())
}
fmt.Println("res1", res)
}
func BenchmarkMemorySet(b *testing.B) {
mc := NewMemoryCache(0)
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i)
value := fmt.Sprintf("value%d", i)
mc.Set(key, value, 1000)
}
}

49
lib/cache/redis.go vendored Normal file
View File

@@ -0,0 +1,49 @@
package cache
import (
"context"
"github.com/go-redis/redis/v8"
"time"
)
var ctx = context.Background()
type RedisCache struct {
rdb *redis.Client
}
func RedisCacheInit(conf *redis.Options) *RedisCache {
c := &RedisCache{}
c.rdb = redis.NewClient(conf)
return c
}
func (c *RedisCache) Get(key string, value interface{}) error {
data, err := c.rdb.Get(ctx, key).Result()
if err != nil {
return err
}
err1 := DecodeValue(data, value)
return err1
}
func (c *RedisCache) Set(key string, value interface{}, exp int) error {
str, err := EncodeValue(value)
if err != nil {
return err
}
if exp <= 0 {
exp = MaxTimeOut
}
_, err1 := c.rdb.Set(ctx, key, str, time.Duration(exp)*time.Second).Result()
return err1
}
func (c *RedisCache) Gc() error {
return nil
}
func NewRedis(conf *redis.Options) *RedisCache {
cache := RedisCacheInit(conf)
return cache
}

94
lib/cache/redis_test.go vendored Normal file
View File

@@ -0,0 +1,94 @@
package cache
import (
"fmt"
"github.com/go-redis/redis/v8"
"reflect"
"testing"
)
func TestRedisSet(t *testing.T) {
//rc := New("redis")
rc := RedisCacheInit(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
err := rc.Set("123", "ddd", 0)
if err != nil {
fmt.Println(err.Error())
t.Fatalf("写入失败")
}
}
func TestRedisGet(t *testing.T) {
rc := RedisCacheInit(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
err := rc.Set("123", "451156", 300)
if err != nil {
t.Fatalf("写入失败")
}
res := ""
err = rc.Get("123", &res)
if err != nil {
t.Fatalf("读取失败")
}
fmt.Println("res", res)
}
func TestRedisGetJson(t *testing.T) {
rc := RedisCacheInit(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
type r struct {
Aa string `json:"a"`
B string `json:"c"`
}
old := &r{
Aa: "ab", B: "cdc",
}
err := rc.Set("1233", old, 300)
if err != nil {
t.Fatalf("写入失败")
}
res := &r{}
err2 := rc.Get("1233", res)
if err2 != nil {
t.Fatalf("读取失败")
}
if !reflect.DeepEqual(res, old) {
t.Fatalf("读取错误")
}
fmt.Println(res, res.Aa)
}
func BenchmarkRSet(b *testing.B) {
rc := RedisCacheInit(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
b.ResetTimer()
for i := 0; i < b.N; i++ {
rc.Set("123", "{dsv}", 1000)
}
}
func BenchmarkRGet(b *testing.B) {
rc := RedisCacheInit(&redis.Options{
Addr: "192.168.1.168:6379",
Password: "", // no password set
DB: 0, // use default DB
})
b.ResetTimer()
v := ""
for i := 0; i < b.N; i++ {
rc.Get("123", &v)
}
}

65
lib/cache/simple_cache.go vendored Normal file
View File

@@ -0,0 +1,65 @@
package cache
import (
"errors"
"reflect"
"sync"
)
// 此处实现了一个简单的缓存,用于测试
// SimpleCache is a simple cache implementation
type SimpleCache struct {
data map[string]interface{}
mu sync.Mutex
maxBytes int64
usedBytes int64
}
func (s *SimpleCache) Get(key string, value interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
// 使用反射将存储的值设置到传入的指针变量中
val := reflect.ValueOf(value)
if val.Kind() != reflect.Ptr {
return errors.New("value must be a pointer")
}
v, ok := s.data[key]
if !ok {
//设为空值
val.Elem().Set(reflect.Zero(val.Elem().Type()))
return nil
}
vval := reflect.ValueOf(v)
if val.Elem().Type() != vval.Type() {
//设为空值
val.Elem().Set(reflect.Zero(val.Elem().Type()))
return nil
}
val.Elem().Set(reflect.ValueOf(v))
return nil
}
func (s *SimpleCache) Set(key string, value interface{}, exp int) error {
s.mu.Lock()
defer s.mu.Unlock()
// 检查传入的值是否是指针,如果是则取其值
val := reflect.ValueOf(value)
if val.Kind() == reflect.Ptr {
val = val.Elem()
}
s.data[key] = val.Interface()
return nil
}
func (s *SimpleCache) Gc() error {
return nil
}
func NewSimpleCache() *SimpleCache {
return &SimpleCache{
data: make(map[string]interface{}),
}
}

108
lib/cache/simple_cache_test.go vendored Normal file
View File

@@ -0,0 +1,108 @@
package cache
import (
"fmt"
"testing"
)
func TestSimpleCache_Set(t *testing.T) {
s := NewSimpleCache()
err := s.Set("key", "value", 0)
if err != nil {
t.Fatalf("写入失败")
}
err = s.Set("key", 111, 0)
if err != nil {
t.Fatalf("写入失败")
}
}
func TestSimpleCache_Get(t *testing.T) {
s := NewSimpleCache()
err := s.Set("key", "value", 0)
value := ""
err = s.Get("key", &value)
fmt.Println("value", value)
if err != nil {
t.Fatalf("读取失败")
}
err = s.Set("key1", 11, 0)
value1 := 0
err = s.Get("key1", &value1)
fmt.Println("value1", value1)
if err != nil {
t.Fatalf("读取失败")
}
err = s.Set("key2", []byte{'a', 'b'}, 0)
value2 := []byte{}
err = s.Get("key2", &value2)
fmt.Println("value2", string(value2))
if err != nil {
t.Fatalf("读取失败")
}
err = s.Set("key3", 33.33, 0)
var value3 int
err = s.Get("key3", &value3)
fmt.Println("value3", value3)
if err != nil {
t.Fatalf("读取失败")
}
}
type r struct {
A string `json:"a"`
B string `json:"b"`
R *rr `json:"r"`
}
type r2 struct {
A string `json:"a"`
B string `json:"b"`
}
type rr struct {
AA string `json:"aa"`
BB string `json:"bb"`
}
func TestSimpleCache_GetStruct(t *testing.T) {
s := NewSimpleCache()
old_rr := &rr{
AA: "aa", BB: "bb",
}
old := &r{
A: "ab", B: "cdc",
R: old_rr,
}
err := s.Set("key", old, 300)
if err != nil {
t.Fatalf("写入失败")
}
res := &r{}
err2 := s.Get("key", res)
fmt.Println("res", res)
if err2 != nil {
t.Fatalf("读取失败" + err2.Error())
}
//修改原始值,看后面是否会变化
old.A = "aa"
old_rr.AA = "aaa"
fmt.Println("old", old)
res2 := &r{}
err3 := s.Get("key", res2)
fmt.Println("res2", res2, res2.R.AA, res2.R.BB)
if err3 != nil {
t.Fatalf("读取失败" + err3.Error())
}
//if reflect.DeepEqual(res, old) {
// t.Fatalf("读取错误")
//}
}