From 4f234a2cb8e62f4ec861dd45bdf950291414a9a9 Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Mon, 1 Jan 2024 22:29:19 +0800 Subject: [PATCH 1/5] refactor: Use `sync.Pool` to reuse the []byte with specific length --- internal/rdb/rdb_test.go | 6 ++-- internal/rdb/structure/byte.go | 37 +++++++++++++++++++++-- internal/rdb/structure/int.go | 50 +++++++++++++++++++++++-------- internal/rdb/structure/ziplist.go | 6 +++- 4 files changed, 81 insertions(+), 18 deletions(-) diff --git a/internal/rdb/rdb_test.go b/internal/rdb/rdb_test.go index 955fbce8..f50896f4 100644 --- a/internal/rdb/rdb_test.go +++ b/internal/rdb/rdb_test.go @@ -10,19 +10,19 @@ import ( // BenchmarkParseRDB is a benchmark for ParseRDB // The baseline is "20 350030327 ns/op 213804114 B/op 1900715 allocs/op" func BenchmarkParseRDB(b *testing.B) { + b.ResetTimer() b.ReportAllocs() b.ResetTimer() tempChan := make(chan *entry.Entry, 1024) updateFunc := func(offset int64) { } - b.N = 20 for i := 0; i < b.N; i++ { loader := NewLoader("rdb", updateFunc, "./dump.rdb", tempChan) go func() { - for temp := range tempChan { - print(temp.CmdName) + for _ = range tempChan { + } }() loader.ParseRDB(context.Background()) diff --git a/internal/rdb/structure/byte.go b/internal/rdb/structure/byte.go index f52ae0f5..59894ff6 100644 --- a/internal/rdb/structure/byte.go +++ b/internal/rdb/structure/byte.go @@ -2,13 +2,46 @@ package structure import ( "io" + "sync" "RedisShake/internal/log" ) +var BytesPoolWithCap1 = sync.Pool{ + New: func() interface{} { + tmp := make([]byte, 1) + return tmp + }, +} + +var BytesPoolWithCap2 = sync.Pool{ + New: func() interface{} { + tmp := make([]byte, 2) + return tmp + }, +} + +var BytesPoolWithCap4 = sync.Pool{ + New: func() interface{} { + tmp := make([]byte, 4) + return tmp + }, +} + +var BytesPoolWithCap8 = sync.Pool{ + New: func() interface{} { + tmp := make([]byte, 8) + return tmp + }, +} + func ReadByte(rd io.Reader) byte { - b := ReadBytes(rd, 1)[0] - return b + data := BytesPoolWithCap1.Get().([]byte) + defer BytesPoolWithCap1.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return data[0] } func ReadBytes(rd io.Reader, n int) []byte { diff --git a/internal/rdb/structure/int.go b/internal/rdb/structure/int.go index 8ce5f6f0..6a30628c 100644 --- a/internal/rdb/structure/int.go +++ b/internal/rdb/structure/int.go @@ -3,6 +3,8 @@ package structure import ( "encoding/binary" "io" + + "RedisShake/internal/log" ) func ReadUint8(rd io.Reader) uint8 { @@ -11,8 +13,12 @@ func ReadUint8(rd io.Reader) uint8 { } func ReadUint16(rd io.Reader) uint16 { - buf := ReadBytes(rd, 2) - return binary.LittleEndian.Uint16(buf) + data := BytesPoolWithCap2.Get().([]byte) + defer BytesPoolWithCap2.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return binary.LittleEndian.Uint16(data) } func ReadUint24(rd io.Reader) uint32 { @@ -22,13 +28,21 @@ func ReadUint24(rd io.Reader) uint32 { } func ReadUint32(rd io.Reader) uint32 { - buf := ReadBytes(rd, 4) - return binary.LittleEndian.Uint32(buf) + data := BytesPoolWithCap4.Get().([]byte) + defer BytesPoolWithCap4.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return binary.LittleEndian.Uint32(data) } func ReadUint64(rd io.Reader) uint64 { - buf := ReadBytes(rd, 8) - return binary.LittleEndian.Uint64(buf) + data := BytesPoolWithCap8.Get().([]byte) + defer BytesPoolWithCap8.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return binary.LittleEndian.Uint64(data) } func ReadInt8(rd io.Reader) int8 { @@ -37,8 +51,12 @@ func ReadInt8(rd io.Reader) int8 { } func ReadInt16(rd io.Reader) int16 { - buf := ReadBytes(rd, 2) - return int16(binary.LittleEndian.Uint16(buf)) + data := BytesPoolWithCap2.Get().([]byte) + defer BytesPoolWithCap2.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return int16(binary.LittleEndian.Uint16(data)) } func ReadInt24(rd io.Reader) int32 { @@ -48,11 +66,19 @@ func ReadInt24(rd io.Reader) int32 { } func ReadInt32(rd io.Reader) int32 { - buf := ReadBytes(rd, 4) - return int32(binary.LittleEndian.Uint32(buf)) + data := BytesPoolWithCap4.Get().([]byte) + defer BytesPoolWithCap4.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return int32(binary.LittleEndian.Uint32(data)) } func ReadInt64(rd io.Reader) int64 { - buf := ReadBytes(rd, 8) - return int64(binary.LittleEndian.Uint64(buf)) + data := BytesPoolWithCap8.Get().([]byte) + defer BytesPoolWithCap8.Put(data) + if _, err := io.ReadFull(rd, data); err != nil { + log.Panicf(err.Error()) + } + return int64(binary.LittleEndian.Uint64(data)) } diff --git a/internal/rdb/structure/ziplist.go b/internal/rdb/structure/ziplist.go index a1c47b54..40b99899 100644 --- a/internal/rdb/structure/ziplist.go +++ b/internal/rdb/structure/ziplist.go @@ -80,8 +80,12 @@ func readZipListEntry(rd io.Reader, firstByte byte) string { length := (int(firstByte&0x3f) << 8) | int(secondByte) return string(ReadBytes(rd, length)) case zipStr32B: - lenBytes := ReadBytes(rd, 4) + lenBytes := BytesPoolWithCap4.Get().([]byte) + if _, err := io.ReadFull(rd, lenBytes); err != nil { + log.Panicf(err.Error()) + } length := binary.BigEndian.Uint32(lenBytes) + BytesPoolWithCap4.Put(lenBytes) return string(ReadBytes(rd, int(length))) } switch firstByte { From cdb47ab3114189c5a55aeb15a95e3aa08e9add87 Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Mon, 1 Jan 2024 23:07:34 +0800 Subject: [PATCH 2/5] update code --- internal/rdb/structure/byte.go | 5 +++-- internal/rdb/structure/int.go | 31 +++++++++++++++++++------------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/internal/rdb/structure/byte.go b/internal/rdb/structure/byte.go index 59894ff6..c6d5ad78 100644 --- a/internal/rdb/structure/byte.go +++ b/internal/rdb/structure/byte.go @@ -37,11 +37,12 @@ var BytesPoolWithCap8 = sync.Pool{ func ReadByte(rd io.Reader) byte { data := BytesPoolWithCap1.Get().([]byte) - defer BytesPoolWithCap1.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return data[0] + result := data[0] + BytesPoolWithCap1.Put(data) + return result } func ReadBytes(rd io.Reader, n int) []byte { diff --git a/internal/rdb/structure/int.go b/internal/rdb/structure/int.go index 6a30628c..ae0d1260 100644 --- a/internal/rdb/structure/int.go +++ b/internal/rdb/structure/int.go @@ -14,11 +14,13 @@ func ReadUint8(rd io.Reader) uint8 { func ReadUint16(rd io.Reader) uint16 { data := BytesPoolWithCap2.Get().([]byte) - defer BytesPoolWithCap2.Put(data) + BytesPoolWithCap2.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return binary.LittleEndian.Uint16(data) + result := binary.LittleEndian.Uint16(data) + BytesPoolWithCap2.Put(data) + return result } func ReadUint24(rd io.Reader) uint32 { @@ -29,20 +31,22 @@ func ReadUint24(rd io.Reader) uint32 { func ReadUint32(rd io.Reader) uint32 { data := BytesPoolWithCap4.Get().([]byte) - defer BytesPoolWithCap4.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return binary.LittleEndian.Uint32(data) + result := binary.LittleEndian.Uint32(data) + BytesPoolWithCap4.Put(data) + return result } func ReadUint64(rd io.Reader) uint64 { data := BytesPoolWithCap8.Get().([]byte) - defer BytesPoolWithCap8.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return binary.LittleEndian.Uint64(data) + result := binary.LittleEndian.Uint64(data) + BytesPoolWithCap8.Put(data) + return result } func ReadInt8(rd io.Reader) int8 { @@ -52,11 +56,12 @@ func ReadInt8(rd io.Reader) int8 { func ReadInt16(rd io.Reader) int16 { data := BytesPoolWithCap2.Get().([]byte) - defer BytesPoolWithCap2.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return int16(binary.LittleEndian.Uint16(data)) + result := int16(binary.LittleEndian.Uint16(data)) + BytesPoolWithCap2.Put(data) + return result } func ReadInt24(rd io.Reader) int32 { @@ -67,18 +72,20 @@ func ReadInt24(rd io.Reader) int32 { func ReadInt32(rd io.Reader) int32 { data := BytesPoolWithCap4.Get().([]byte) - defer BytesPoolWithCap4.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return int32(binary.LittleEndian.Uint32(data)) + result := int32(binary.LittleEndian.Uint32(data)) + BytesPoolWithCap4.Put(data) + return result } func ReadInt64(rd io.Reader) int64 { data := BytesPoolWithCap8.Get().([]byte) - defer BytesPoolWithCap8.Put(data) if _, err := io.ReadFull(rd, data); err != nil { log.Panicf(err.Error()) } - return int64(binary.LittleEndian.Uint64(data)) + result := int64(binary.LittleEndian.Uint64(data)) + BytesPoolWithCap8.Put(data) + return result } From ade5359568de6edcc01b40ab72884d2bee0860ca Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Mon, 1 Jan 2024 23:21:43 +0800 Subject: [PATCH 3/5] update code --- internal/rdb/structure/byte.go | 14 ++++++------ internal/rdb/structure/int.go | 37 +++++++++++++++---------------- internal/rdb/structure/ziplist.go | 6 ++--- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/internal/rdb/structure/byte.go b/internal/rdb/structure/byte.go index c6d5ad78..9373ca73 100644 --- a/internal/rdb/structure/byte.go +++ b/internal/rdb/structure/byte.go @@ -10,37 +10,37 @@ import ( var BytesPoolWithCap1 = sync.Pool{ New: func() interface{} { tmp := make([]byte, 1) - return tmp + return &tmp }, } var BytesPoolWithCap2 = sync.Pool{ New: func() interface{} { tmp := make([]byte, 2) - return tmp + return &tmp }, } var BytesPoolWithCap4 = sync.Pool{ New: func() interface{} { tmp := make([]byte, 4) - return tmp + return &tmp }, } var BytesPoolWithCap8 = sync.Pool{ New: func() interface{} { tmp := make([]byte, 8) - return tmp + return &tmp }, } func ReadByte(rd io.Reader) byte { - data := BytesPoolWithCap1.Get().([]byte) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap1.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := data[0] + result := (*data)[0] BytesPoolWithCap1.Put(data) return result } diff --git a/internal/rdb/structure/int.go b/internal/rdb/structure/int.go index ae0d1260..44c015bc 100644 --- a/internal/rdb/structure/int.go +++ b/internal/rdb/structure/int.go @@ -13,12 +13,11 @@ func ReadUint8(rd io.Reader) uint8 { } func ReadUint16(rd io.Reader) uint16 { - data := BytesPoolWithCap2.Get().([]byte) - BytesPoolWithCap2.Put(data) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap2.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := binary.LittleEndian.Uint16(data) + result := binary.LittleEndian.Uint16(*data) BytesPoolWithCap2.Put(data) return result } @@ -30,21 +29,21 @@ func ReadUint24(rd io.Reader) uint32 { } func ReadUint32(rd io.Reader) uint32 { - data := BytesPoolWithCap4.Get().([]byte) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap4.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := binary.LittleEndian.Uint32(data) + result := binary.LittleEndian.Uint32(*data) BytesPoolWithCap4.Put(data) return result } func ReadUint64(rd io.Reader) uint64 { - data := BytesPoolWithCap8.Get().([]byte) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap8.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := binary.LittleEndian.Uint64(data) + result := binary.LittleEndian.Uint64(*data) BytesPoolWithCap8.Put(data) return result } @@ -55,11 +54,11 @@ func ReadInt8(rd io.Reader) int8 { } func ReadInt16(rd io.Reader) int16 { - data := BytesPoolWithCap2.Get().([]byte) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap2.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := int16(binary.LittleEndian.Uint16(data)) + result := int16(binary.LittleEndian.Uint16(*data)) BytesPoolWithCap2.Put(data) return result } @@ -71,21 +70,21 @@ func ReadInt24(rd io.Reader) int32 { } func ReadInt32(rd io.Reader) int32 { - data := BytesPoolWithCap4.Get().([]byte) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap4.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := int32(binary.LittleEndian.Uint32(data)) + result := int32(binary.LittleEndian.Uint32(*data)) BytesPoolWithCap4.Put(data) return result } func ReadInt64(rd io.Reader) int64 { - data := BytesPoolWithCap8.Get().([]byte) - if _, err := io.ReadFull(rd, data); err != nil { + data := BytesPoolWithCap8.Get().(*[]byte) + if _, err := io.ReadFull(rd, *data); err != nil { log.Panicf(err.Error()) } - result := int64(binary.LittleEndian.Uint64(data)) + result := int64(binary.LittleEndian.Uint64(*data)) BytesPoolWithCap8.Put(data) return result } diff --git a/internal/rdb/structure/ziplist.go b/internal/rdb/structure/ziplist.go index 40b99899..5900fa60 100644 --- a/internal/rdb/structure/ziplist.go +++ b/internal/rdb/structure/ziplist.go @@ -80,11 +80,11 @@ func readZipListEntry(rd io.Reader, firstByte byte) string { length := (int(firstByte&0x3f) << 8) | int(secondByte) return string(ReadBytes(rd, length)) case zipStr32B: - lenBytes := BytesPoolWithCap4.Get().([]byte) - if _, err := io.ReadFull(rd, lenBytes); err != nil { + lenBytes := BytesPoolWithCap4.Get().(*[]byte) + if _, err := io.ReadFull(rd, *lenBytes); err != nil { log.Panicf(err.Error()) } - length := binary.BigEndian.Uint32(lenBytes) + length := binary.BigEndian.Uint32(*lenBytes) BytesPoolWithCap4.Put(lenBytes) return string(ReadBytes(rd, int(length))) } From 2abde284514b865d5fd01054ecec43efb38ca41a Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Tue, 2 Jan 2024 01:03:09 +0800 Subject: [PATCH 4/5] update code --- internal/rdb/rdb_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/internal/rdb/rdb_test.go b/internal/rdb/rdb_test.go index f50896f4..86054a70 100644 --- a/internal/rdb/rdb_test.go +++ b/internal/rdb/rdb_test.go @@ -2,6 +2,8 @@ package rdb import ( "context" + "io" + "os" "testing" "RedisShake/internal/entry" @@ -10,6 +12,25 @@ import ( // BenchmarkParseRDB is a benchmark for ParseRDB // The baseline is "20 350030327 ns/op 213804114 B/op 1900715 allocs/op" func BenchmarkParseRDB(b *testing.B) { + sourcePath := "./dump.rdb" + sourceFile, err := os.Open(sourcePath) + if err != nil { + panic(err) + } + defer sourceFile.Close() + + destPath := "/tmp/dump.rdb" + destFile, err := os.Create(destPath) + if err != nil { + panic(err) + } + defer destFile.Close() + + // 复制文件内容 + _, err = io.Copy(destFile, sourceFile) + if err != nil { + panic(err) + } b.ResetTimer() b.ReportAllocs() b.ResetTimer() From eba9e7410d8c4d3a1c388e0a0137a710dd9080ce Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Tue, 2 Jan 2024 01:05:57 +0800 Subject: [PATCH 5/5] update code --- internal/rdb/rdb_test.go | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/internal/rdb/rdb_test.go b/internal/rdb/rdb_test.go index 86054a70..7361a793 100644 --- a/internal/rdb/rdb_test.go +++ b/internal/rdb/rdb_test.go @@ -12,24 +12,23 @@ import ( // BenchmarkParseRDB is a benchmark for ParseRDB // The baseline is "20 350030327 ns/op 213804114 B/op 1900715 allocs/op" func BenchmarkParseRDB(b *testing.B) { - sourcePath := "./dump.rdb" - sourceFile, err := os.Open(sourcePath) - if err != nil { - panic(err) - } - defer sourceFile.Close() - - destPath := "/tmp/dump.rdb" - destFile, err := os.Create(destPath) - if err != nil { - panic(err) - } - defer destFile.Close() - - // 复制文件内容 - _, err = io.Copy(destFile, sourceFile) - if err != nil { - panic(err) + if _, err := os.Stat("/tmp/dump.rdb"); err != nil && os.IsNotExist(err) { + sourcePath := "./dump.rdb" + sourceFile, err := os.Open(sourcePath) + if err != nil { + panic(err) + } + destPath := "/tmp/dump.rdb" + destFile, err := os.Create(destPath) + if err != nil { + panic(err) + } + _, err = io.Copy(destFile, sourceFile) + if err != nil { + panic(err) + } + destFile.Close() + sourceFile.Close() } b.ResetTimer() b.ReportAllocs() @@ -40,7 +39,7 @@ func BenchmarkParseRDB(b *testing.B) { } for i := 0; i < b.N; i++ { - loader := NewLoader("rdb", updateFunc, "./dump.rdb", tempChan) + loader := NewLoader("rdb", updateFunc, "/tmp/dump.rdb", tempChan) go func() { for _ = range tempChan {