From 660df1ccc7ca45f8d3e3fb9aeeef4afe1d6c9935 Mon Sep 17 00:00:00 2001 From: Vedant Das Date: Wed, 8 Jul 2026 18:18:37 +0000 Subject: [PATCH 1/4] perf: optimized timestamp memory consumption --- internal/cache/metadata/stat_cache.go | 14 +-- .../metadata/stat_cache_benchmark_test.go | 85 +++++++++++++++++++ internal/storage/gcs/folder.go | 5 +- internal/storage/gcs/folder_test.go | 14 ++- 4 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 internal/cache/metadata/stat_cache_benchmark_test.go diff --git a/internal/cache/metadata/stat_cache.go b/internal/cache/metadata/stat_cache.go index dfcddfa7ec..ed4d029041 100644 --- a/internal/cache/metadata/stat_cache.go +++ b/internal/cache/metadata/stat_cache.go @@ -113,7 +113,7 @@ type statCacheBucketView struct { type entry struct { m *gcs.MinObject f *gcs.Folder - expiration time.Time + expiration int64 // Set to true only for implicit directory entries. This flag will always remain false for negative entries and explicit objects. implicitDir bool } @@ -201,7 +201,7 @@ func (sc *statCacheBucketView) Insert(m *gcs.MinObject, expiration time.Time) { // Insert an entry. e := entry{ m: m, - expiration: expiration, + expiration: expiration.Unix(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -237,7 +237,7 @@ func (sc *statCacheBucketView) InsertImplicitDir(objectName string, expiration t // Insert an entry. e := entry{ implicitDir: true, - expiration: expiration, + expiration: expiration.Unix(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -251,7 +251,7 @@ func (sc *statCacheBucketView) AddNegativeEntry(objectName string, expiration ti // Insert a negative entry. e := entry{ m: nil, - expiration: expiration, + expiration: expiration.Unix(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -265,7 +265,7 @@ func (sc *statCacheBucketView) AddNegativeEntryForFolder(folderName string, expi // Insert a negative entry. e := entry{ f: nil, - expiration: expiration, + expiration: expiration.Unix(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -315,7 +315,7 @@ func (sc *statCacheBucketView) sharedCacheLookup(key string, now time.Time) (boo e := value.(entry) // Has this entry expired? - if e.expiration.Before(now) { + if e.expiration < now.Unix() { sc.Erase(key) return false, nil } @@ -328,7 +328,7 @@ func (sc *statCacheBucketView) InsertFolder(f *gcs.Folder, expiration time.Time) e := entry{ f: f, - expiration: expiration, + expiration: expiration.Unix(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { diff --git a/internal/cache/metadata/stat_cache_benchmark_test.go b/internal/cache/metadata/stat_cache_benchmark_test.go new file mode 100644 index 0000000000..bf9157faf7 --- /dev/null +++ b/internal/cache/metadata/stat_cache_benchmark_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata_test + +import ( + "fmt" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +func generateKeys(count int) []string { + keys := make([]string, count) + for i := 0; i < count; i++ { + keys[i] = fmt.Sprintf("object-%d", i) + } + return keys +} + +func setupStatCache(capacity uint64) metadata.StatCache { + // 1000 items * approx 120 bytes each = ~120,000 bytes capacity + lruCache := lru.NewCache(capacity) + return metadata.NewStatCacheBucketView(lruCache, "") +} + +func BenchmarkStatCache_Insert(b *testing.B) { + keys := generateKeys(10000) + cache := setupStatCache(uint64(len(keys) * 500)) + now := time.Now() + + i := 0 + for b.Loop() { + key := keys[i%len(keys)] + m := &gcs.MinObject{Name: key} + cache.Insert(m, now) + i++ + } +} + +func BenchmarkStatCache_AddNegativeEntry(b *testing.B) { + keys := generateKeys(10000) + cache := setupStatCache(uint64(len(keys) * 500)) + now := time.Now() + + i := 0 + for b.Loop() { + key := keys[i%len(keys)] + cache.AddNegativeEntry(key, now) + i++ + } +} + +func BenchmarkStatCache_LookUp(b *testing.B) { + keys := generateKeys(10000) + cache := setupStatCache(uint64(len(keys) * 500)) + now := time.Now() + + // Pre-fill the cache with unexpired entries. + for _, key := range keys { + m := &gcs.MinObject{Name: key} + cache.Insert(m, now.Add(time.Hour)) + } + + i := 0 + for b.Loop() { + key := keys[i%len(keys)] + _, _ = cache.LookUp(key, now) + i++ + } +} diff --git a/internal/storage/gcs/folder.go b/internal/storage/gcs/folder.go index fbd3b488fe..921bcbcc57 100644 --- a/internal/storage/gcs/folder.go +++ b/internal/storage/gcs/folder.go @@ -16,21 +16,20 @@ package gcs import ( "strings" - "time" "cloud.google.com/go/storage/control/apiv2/controlpb" ) type Folder struct { Name string - UpdateTime time.Time + UpdateTime int64 } func GCSFolder(bucketName string, attrs *controlpb.Folder) *Folder { // Setting the parameters in Folder and doing conversions as necessary. return &Folder{ Name: getFolderName(bucketName, attrs.Name), - UpdateTime: attrs.GetUpdateTime().AsTime(), + UpdateTime: attrs.GetUpdateTime().AsTime().Unix(), } } diff --git a/internal/storage/gcs/folder_test.go b/internal/storage/gcs/folder_test.go index 2c2a51b0a3..668bd15d83 100644 --- a/internal/storage/gcs/folder_test.go +++ b/internal/storage/gcs/folder_test.go @@ -48,5 +48,17 @@ func TestGCSFolder(t *testing.T) { gcsFolder := GCSFolder(TestBucketName, &attrs) assert.Equal(t, attrs.Name, gcsFolder.Name) - assert.Equal(t, attrs.UpdateTime.AsTime(), gcsFolder.UpdateTime) + assert.Equal(t, attrs.UpdateTime.AsTime().Unix(), gcsFolder.UpdateTime) +} + +func BenchmarkGCSFolder(b *testing.B) { + attrs := &controlpb.Folder{ + Name: "projects/_/buckets/my-bucket/folders/my-folder", + UpdateTime: timestamppb.Now(), + } + // Wait for b.Loop() to start the timer + for b.Loop() { + // This tests pure allocation speed and memory overhead of creating a Folder! + _ = GCSFolder("my-bucket", attrs) + } } From 4aff0fe4885526659c4e2bd9c4e170882e84447e Mon Sep 17 00:00:00 2001 From: Vedant Das Date: Thu, 9 Jul 2026 03:56:42 +0000 Subject: [PATCH 2/4] removed folder.go timestamp changes --- internal/storage/gcs/folder.go | 5 +++-- internal/storage/gcs/folder_test.go | 14 +------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/internal/storage/gcs/folder.go b/internal/storage/gcs/folder.go index 921bcbcc57..fbd3b488fe 100644 --- a/internal/storage/gcs/folder.go +++ b/internal/storage/gcs/folder.go @@ -16,20 +16,21 @@ package gcs import ( "strings" + "time" "cloud.google.com/go/storage/control/apiv2/controlpb" ) type Folder struct { Name string - UpdateTime int64 + UpdateTime time.Time } func GCSFolder(bucketName string, attrs *controlpb.Folder) *Folder { // Setting the parameters in Folder and doing conversions as necessary. return &Folder{ Name: getFolderName(bucketName, attrs.Name), - UpdateTime: attrs.GetUpdateTime().AsTime().Unix(), + UpdateTime: attrs.GetUpdateTime().AsTime(), } } diff --git a/internal/storage/gcs/folder_test.go b/internal/storage/gcs/folder_test.go index 668bd15d83..2c2a51b0a3 100644 --- a/internal/storage/gcs/folder_test.go +++ b/internal/storage/gcs/folder_test.go @@ -48,17 +48,5 @@ func TestGCSFolder(t *testing.T) { gcsFolder := GCSFolder(TestBucketName, &attrs) assert.Equal(t, attrs.Name, gcsFolder.Name) - assert.Equal(t, attrs.UpdateTime.AsTime().Unix(), gcsFolder.UpdateTime) -} - -func BenchmarkGCSFolder(b *testing.B) { - attrs := &controlpb.Folder{ - Name: "projects/_/buckets/my-bucket/folders/my-folder", - UpdateTime: timestamppb.Now(), - } - // Wait for b.Loop() to start the timer - for b.Loop() { - // This tests pure allocation speed and memory overhead of creating a Folder! - _ = GCSFolder("my-bucket", attrs) - } + assert.Equal(t, attrs.UpdateTime.AsTime(), gcsFolder.UpdateTime) } From 67ad8a89b584dc821015eabe7e62791dc1468373 Mon Sep 17 00:00:00 2001 From: Vedant Das Date: Thu, 9 Jul 2026 09:50:17 +0000 Subject: [PATCH 3/4] changed to UnixNano() to avoid test failures caused by nanosecond increments --- internal/cache/metadata/stat_cache.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/cache/metadata/stat_cache.go b/internal/cache/metadata/stat_cache.go index ed4d029041..aeb27cab41 100644 --- a/internal/cache/metadata/stat_cache.go +++ b/internal/cache/metadata/stat_cache.go @@ -113,7 +113,7 @@ type statCacheBucketView struct { type entry struct { m *gcs.MinObject f *gcs.Folder - expiration int64 + expiration int64 // expiration timestamp // Set to true only for implicit directory entries. This flag will always remain false for negative entries and explicit objects. implicitDir bool } @@ -201,7 +201,7 @@ func (sc *statCacheBucketView) Insert(m *gcs.MinObject, expiration time.Time) { // Insert an entry. e := entry{ m: m, - expiration: expiration.Unix(), + expiration: expiration.UnixNano(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -237,7 +237,7 @@ func (sc *statCacheBucketView) InsertImplicitDir(objectName string, expiration t // Insert an entry. e := entry{ implicitDir: true, - expiration: expiration.Unix(), + expiration: expiration.UnixNano(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -251,7 +251,7 @@ func (sc *statCacheBucketView) AddNegativeEntry(objectName string, expiration ti // Insert a negative entry. e := entry{ m: nil, - expiration: expiration.Unix(), + expiration: expiration.UnixNano(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -265,7 +265,7 @@ func (sc *statCacheBucketView) AddNegativeEntryForFolder(folderName string, expi // Insert a negative entry. e := entry{ f: nil, - expiration: expiration.Unix(), + expiration: expiration.UnixNano(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -315,7 +315,7 @@ func (sc *statCacheBucketView) sharedCacheLookup(key string, now time.Time) (boo e := value.(entry) // Has this entry expired? - if e.expiration < now.Unix() { + if e.expiration < now.UnixNano() { sc.Erase(key) return false, nil } @@ -328,7 +328,7 @@ func (sc *statCacheBucketView) InsertFolder(f *gcs.Folder, expiration time.Time) e := entry{ f: f, - expiration: expiration.Unix(), + expiration: expiration.UnixNano(), } if _, err := sc.sharedCache.Insert(name, e); err != nil { From f2acd33f612a97531826afd4bec8a88608e68e63 Mon Sep 17 00:00:00 2001 From: Vedant Das Date: Thu, 9 Jul 2026 15:29:21 +0000 Subject: [PATCH 4/4] added utility function to avoid incorrect eviction --- internal/cache/metadata/stat_cache.go | 34 ++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/internal/cache/metadata/stat_cache.go b/internal/cache/metadata/stat_cache.go index aeb27cab41..aba86fc182 100644 --- a/internal/cache/metadata/stat_cache.go +++ b/internal/cache/metadata/stat_cache.go @@ -145,6 +145,28 @@ func (e entry) Size() uint64 { return size } +// timeToUnixNano safely converts a time.Time to Unix nanoseconds. +// It prevents int64 overflow for dates beyond the year 2262, returning +// math.MaxInt64 instead, which safely mimics "never expire". +func timeToUnixNano(t time.Time) int64 { + if t.IsZero() { + return math.MinInt64 + } + + // 9223372036 is math.MaxInt64 / 1,000,000,000 (max seconds representable in nanoseconds) + const maxSeconds int64 = math.MaxInt64 / 1000000000 + + unixSecs := t.Unix() + if unixSecs > maxSeconds { + return math.MaxInt64 + } + if unixSecs < -maxSeconds { + return math.MinInt64 + } + + return t.UnixNano() +} + // Should the supplied object for a new positive entry replace the given // existing entry? func shouldReplace(m *gcs.MinObject, existing entry) bool { @@ -201,7 +223,7 @@ func (sc *statCacheBucketView) Insert(m *gcs.MinObject, expiration time.Time) { // Insert an entry. e := entry{ m: m, - expiration: expiration.UnixNano(), + expiration: timeToUnixNano(expiration), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -237,7 +259,7 @@ func (sc *statCacheBucketView) InsertImplicitDir(objectName string, expiration t // Insert an entry. e := entry{ implicitDir: true, - expiration: expiration.UnixNano(), + expiration: timeToUnixNano(expiration), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -251,7 +273,7 @@ func (sc *statCacheBucketView) AddNegativeEntry(objectName string, expiration ti // Insert a negative entry. e := entry{ m: nil, - expiration: expiration.UnixNano(), + expiration: timeToUnixNano(expiration), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -265,7 +287,7 @@ func (sc *statCacheBucketView) AddNegativeEntryForFolder(folderName string, expi // Insert a negative entry. e := entry{ f: nil, - expiration: expiration.UnixNano(), + expiration: timeToUnixNano(expiration), } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -315,7 +337,7 @@ func (sc *statCacheBucketView) sharedCacheLookup(key string, now time.Time) (boo e := value.(entry) // Has this entry expired? - if e.expiration < now.UnixNano() { + if e.expiration < timeToUnixNano(now) { sc.Erase(key) return false, nil } @@ -328,7 +350,7 @@ func (sc *statCacheBucketView) InsertFolder(f *gcs.Folder, expiration time.Time) e := entry{ f: f, - expiration: expiration.UnixNano(), + expiration: timeToUnixNano(expiration), } if _, err := sc.sharedCache.Insert(name, e); err != nil {