diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 258c5cbfd4..917ca09585 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -27,6 +27,7 @@ import ( "reflect" "slices" "strings" + "sync" "syscall" "time" @@ -61,6 +62,9 @@ import ( "github.com/spf13/viper" ) +// readPoolBufferSize is the size of each buffer in readBufferPool (1 MiB). +const readPoolBufferSize = util.MiB + type ServerConfig struct { // A clock used for cache expiration. It is *not* used for inode times, for // which we use the wall clock. @@ -225,6 +229,11 @@ func NewFileSystem(ctx context.Context, serverCfg *ServerConfig) (fuseutil.FileS globalMaxWriteBlocksSem: semaphore.NewWeighted(serverCfg.NewConfig.Write.GlobalMaxBlocks), globalMaxReadBlocksSem: semaphore.NewWeighted(serverCfg.NewConfig.Read.GlobalMaxBlocks), globalMetadataPrefetchSem: semaphore.NewWeighted(serverCfg.NewConfig.MetadataCache.MetadataPrefetchMaxWorkers), + readBufferPool: sync.Pool{ + New: func() interface{} { + return make([]byte, readPoolBufferSize) + }, + }, } // Initialize MRD cache if enabled @@ -676,6 +685,9 @@ type fileSystem struct { // mrdCache manages the cache of inactive MultiRangeDownloaders. mrdCache *lru.Cache + + // readBufferPool is a sync.Pool of readPoolBufferSize (1 MiB) buffers. + readBufferPool sync.Pool } //////////////////////////////////////////////////////////////////////// @@ -3118,23 +3130,74 @@ func (fs *fileSystem) ReadFile( } // Serve the read. + req := gcsx.ReadRequest{ + Offset: op.Offset, + Size: op.Size, + } + + useReadBufferPool := op.Dst == nil + if useReadBufferPool { + if !fs.newConfig.FileSystem.EnableKernelReader || fh.Inode().Bucket().BucketType().IsRapid() { + logger.Errorf("ReadFile: buffer pool allocation is only supported for regional buckets with"+ + " kernel reader enabled (EnableKernelReader: %v, IsRapid: %v)", + fs.newConfig.FileSystem.EnableKernelReader, + fh.Inode().Bucket().BucketType().IsRapid()) + fh.Inode().Unlock() + return syscall.ENOTSUP + } + + var buffers [][]byte + remaining := op.Size + for remaining > 0 { + bufVal := fs.readBufferPool.Get() + poolBuf := bufVal.([]byte) + size := min(remaining, readPoolBufferSize) + buffers = append(buffers, poolBuf[:size]) + remaining -= size + } + req.Buffers = buffers + } else { + req.Buffer = op.Dst + } + if fs.newConfig.FileSystem.EnableKernelReader { var resp gcsx.ReadResponse - req := &gcsx.ReadRequest{ - Buffer: op.Dst, - Offset: op.Offset, - } - resp, err = fh.ReadWithKernelReader(ctx, req) + resp, err = fh.ReadWithKernelReader(ctx, &req) op.BytesRead = resp.Size - op.Data = resp.Data - op.Callback = resp.Callback + if useReadBufferPool { + if len(resp.Data) > 0 { + op.Data = resp.Data + // Zero-copy hit: we did not use the pool buffers at all. Return all of them immediately. + for _, buf := range req.Buffers { + fs.readBufferPool.Put(buf[:cap(buf)]) + } + req.Buffers = nil + } else { + op.Data = util.LimitBuffers(req.Buffers, resp.Size) + // Return unused buffers (from index len(op.Data) onwards) back to the pool immediately. + usedCount := len(op.Data) + for j := usedCount; j < len(req.Buffers); j++ { + fs.readBufferPool.Put(req.Buffers[j][:cap(req.Buffers[j])]) + } + // Shrink req.Buffers so the callback only returns the used ones. + req.Buffers = req.Buffers[:usedCount] + } + + op.Callback = func() { + if resp.Callback != nil { + resp.Callback() + } + for _, buf := range req.Buffers { + fs.readBufferPool.Put(buf[:cap(buf)]) + } + } + } else { + op.Data = resp.Data + op.Callback = resp.Callback + } } else if fs.newConfig.EnableNewReader { var resp gcsx.ReadResponse - req := &gcsx.ReadRequest{ - Buffer: op.Dst, - Offset: op.Offset, - } - resp, err = fh.ReadWithReadManager(ctx, req, fs.sequentialReadSizeMb) + resp, err = fh.ReadWithReadManager(ctx, &req, fs.sequentialReadSizeMb) op.BytesRead = resp.Size op.Data = resp.Data op.Callback = resp.Callback diff --git a/internal/fs/large_read_regional_test.go b/internal/fs/large_read_regional_test.go new file mode 100644 index 0000000000..6b9d478200 --- /dev/null +++ b/internal/fs/large_read_regional_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "context" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/fuse/fuseutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupLargeRead(ctx context.Context, t *testing.T, isZonal bool, enableKernelReader bool, fileContent string) (fuseutil.FileSystem, *fuseops.ReadFileOp) { + t.Helper() + bucketName := "regional-bucket" + if isZonal { + bucketName = "zonal-bucket" + } + bucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{Zonal: isZonal, Hierarchical: false}) + + serverCfg := &fs.ServerConfig{ + NewConfig: &cfg.Config{ + Write: cfg.WriteConfig{ + GlobalMaxBlocks: 1, + }, + FileSystem: cfg.FileSystemConfig{ + EnableKernelReader: enableKernelReader, + }, + }, + CacheClock: &timeutil.SimulatedClock{}, + BucketName: bucketName, + BucketManager: &fakeBucketManager{ + buckets: map[string]gcs.Bucket{ + bucketName: bucket, + }, + }, + SequentialReadSizeMb: 200, + TraceHandle: tracing.NewOTELTracer(), + MetricHandle: metrics.NewNoopMetrics(), + } + + server, err := fs.NewFileSystem(ctx, serverCfg) + require.NoError(t, err) + + fileName := "large-file" + createWithContents(ctx, t, bucket, fileName, fileContent) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err = server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + + readOp := &fuseops.ReadFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Size: int64(len(fileContent)), + Dst: nil, + } + + return server, readOp +} + +func TestLargeReadRegional_Success(t *testing.T) { + // Arrange + ctx := context.Background() + fileContent := "regional bucket content for vectored read verification." + server, readOp := setupLargeRead(ctx, t, false, true, fileContent) + + // Act + err := server.ReadFile(ctx, readOp) + + // Assert + require.NoError(t, err) + assert.Equal(t, len(fileContent), readOp.BytesRead) + require.Len(t, readOp.Data, 1) + assert.Equal(t, []byte(fileContent), readOp.Data[0]) + if readOp.Callback != nil { + readOp.Callback() + } +} + +func TestLargeReadRegional_NotSupportedForZonal(t *testing.T) { + // Arrange + ctx := context.Background() + server, readOp := setupLargeRead(ctx, t, true, true, "zonal bucket content.") + + // Act + err := server.ReadFile(ctx, readOp) + + // Assert + assert.ErrorIs(t, err, syscall.ENOTSUP) +} + +func TestLargeReadRegional_NotSupportedWhenKernelReaderDisabled(t *testing.T) { + // Arrange + ctx := context.Background() + server, readOp := setupLargeRead(ctx, t, false, false, "regional bucket content.") + + // Act + err := server.ReadFile(ctx, readOp) + + // Assert + assert.ErrorIs(t, err, syscall.ENOTSUP) +} diff --git a/internal/util/util.go b/internal/util/util.go index 74164b836e..96812d2150 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -96,3 +96,21 @@ func BytesToHigherMiBs(bytes uint64) uint64 { // Use integer arithmetic and bitwise shift to avoid slow float conversions return (bytes + bytesInOneMiB - 1) >> 20 } + +// LimitBuffers reslices the buffers slice in-place to represent the first 'limit' +// bytes. It avoids allocating a new slice of slices by modifying the slice elements +// in-place and returning a subslice of the original buffers slice. +func LimitBuffers(buffers [][]byte, limit int) [][]byte { + if limit <= 0 { + return nil + } + var current int + for i, b := range buffers { + if current+len(b) >= limit { + buffers[i] = b[:limit-current] + return buffers[:i+1] + } + current += len(b) + } + return buffers +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go index c30363e0f4..c8f4fc2804 100644 --- a/internal/util/util_test.go +++ b/internal/util/util_test.go @@ -213,3 +213,57 @@ func (ts *UtilTest) TestBytesToHigherMiBs() { assert.Equal(ts.T(), tc.mib, BytesToHigherMiBs(tc.bytes)) } } + +func (ts *UtilTest) TestLimitBuffers() { + cases := []struct { + name string + buffers [][]byte + limit int + expected [][]byte + }{ + { + name: "limit zero", + buffers: [][]byte{[]byte("hello"), []byte("world")}, + limit: 0, + expected: nil, + }, + { + name: "limit negative", + buffers: [][]byte{[]byte("hello"), []byte("world")}, + limit: -5, + expected: nil, + }, + { + name: "limit within first buffer", + buffers: [][]byte{[]byte("hello"), []byte("world")}, + limit: 3, + expected: [][]byte{[]byte("hel")}, + }, + { + name: "limit at buffer boundary", + buffers: [][]byte{[]byte("hello"), []byte("world")}, + limit: 5, + expected: [][]byte{[]byte("hello")}, + }, + { + name: "limit spanning multiple buffers", + buffers: [][]byte{[]byte("hello"), []byte("world")}, + limit: 8, + expected: [][]byte{[]byte("hello"), []byte("wor")}, + }, + { + name: "limit greater than total length", + buffers: [][]byte{[]byte("hello"), []byte("world")}, + limit: 20, + expected: [][]byte{[]byte("hello"), []byte("world")}, + }, + } + + for _, tc := range cases { + ts.Run(tc.name, func() { + result := LimitBuffers(tc.buffers, tc.limit) + + assert.Equal(ts.T(), tc.expected, result) + }) + } +} diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index de174dc5d4..49b74ec02f 100755 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -426,7 +426,7 @@ TEST_DIR_PARALLEL_ZONAL=( mounting negative_stat_cache operations - rapid_appends + rapid_operations read_large_files # Reenable when b/461334834 is done. # readdirplus diff --git a/tools/integration_tests/cloud_profiler/cloud_profiler_test.go b/tools/integration_tests/cloud_profiler/cloud_profiler_test.go index fe83041aa4..d1e7de8fd2 100644 --- a/tools/integration_tests/cloud_profiler/cloud_profiler_test.go +++ b/tools/integration_tests/cloud_profiler/cloud_profiler_test.go @@ -80,20 +80,7 @@ func TestMain(m *testing.M) { // 1. Load and parse the common configuration. cfg := test_suite.ReadConfigFile(setup.ConfigFile()) if len(cfg.CloudProfiler) == 0 { - log.Println("No configuration found for cloud profiler tests in config. Using flags instead.") - - // Populate the config manually. - cfg.CloudProfiler = make([]test_suite.TestConfig, 1) - cfg.CloudProfiler[0].TestBucket = setup.TestBucket() - cfg.CloudProfiler[0].GKEMountedDirectory = setup.MountedDirectory() - cfg.CloudProfiler[0].Configs = make([]test_suite.ConfigItem, 1) - cfg.CloudProfiler[0].Configs[0].Flags = []string{ - "--enable-cloud-profiler --cloud-profiler-cpu --cloud-profiler-heap --cloud-profiler-goroutines --cloud-profiler-mutex --cloud-profiler-allocated-heap", - } - testVersionFlag := fmt.Sprintf(" --cloud-profiler-label=%s", testVersionName) - testServiceNameFlag := fmt.Sprintf(" --cloud-profiler-service-name=%s", testServiceName) - cfg.CloudProfiler[0].Configs[0].Flags[0] = cfg.CloudProfiler[0].Configs[0].Flags[0] + testVersionFlag + testServiceNameFlag - cfg.CloudProfiler[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + log.Fatal("No configuration found for CloudProfiler in config file.") } else if cfg.CloudProfiler[0].GKEMountedDirectory == "" { if len(cfg.CloudProfiler[0].Configs[0].Flags) != 1 { log.Fatalf("Expected exactly 1 config flag, got %d", len(cfg.CloudProfiler[0].Configs[0].Flags)) diff --git a/tools/integration_tests/improved_run_e2e_tests.sh b/tools/integration_tests/improved_run_e2e_tests.sh index 08feb11690..bfd0d2b65a 100755 --- a/tools/integration_tests/improved_run_e2e_tests.sh +++ b/tools/integration_tests/improved_run_e2e_tests.sh @@ -385,7 +385,7 @@ filter_array() { # Test packages for regional buckets. TEST_PACKAGES_FOR_RB=("${TEST_PACKAGES_COMMON[@]}" "inactive_stream_timeout" "cloud_profiler" "requester_pays_bucket") # Test packages for zonal buckets. -TEST_PACKAGES_FOR_ZB=("${TEST_PACKAGES_COMMON[@]}" "rapid_appends" "unfinalized_object") +TEST_PACKAGES_FOR_ZB=("${TEST_PACKAGES_COMMON[@]}" "rapid_operations" "unfinalized_object") # Test packages for TPC buckets. TEST_PACKAGES_FOR_TPC=("operations") diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index f395dffaa9..db2cb43fd4 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -60,18 +60,7 @@ func TestMain(m *testing.M) { // 1. Load and parse the common configuration. config := test_suite.ReadConfigFile(setup.ConfigFile()) if len(config.LogRotation) == 0 { - log.Println("No configuration found for log rotation tests in config. Using flags instead.") - // Populate the config manually. - config.LogRotation = make([]test_suite.TestConfig, 1) - config.LogRotation[0].TestBucket = setup.TestBucket() - config.LogRotation[0].GKEMountedDirectory = setup.MountedDirectory() - config.LogRotation[0].LogFile = setup.LogFile() - config.LogRotation[0].Configs = make([]test_suite.ConfigItem, 1) - config.LogRotation[0].Configs[0].Flags = []string{ - "--log-file=/gcsfuse-tmp/TestLogRotation.log --log-rotate-max-file-size-mb=2 --log-rotate-backup-file-count=2 --log-rotate-compress=false --log-severity=trace", - "--log-file=/gcsfuse-tmp/TestLogRotation.log --log-rotate-max-file-size-mb=2 --log-rotate-backup-file-count=2 --log-rotate-compress=true --log-severity=trace", - } - config.LogRotation[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + log.Fatal("No configuration found for LogRotation in config file.") } cfg = &config.LogRotation[0] diff --git a/tools/integration_tests/rapid_appends/appends_test.go b/tools/integration_tests/rapid_operations/appends_test.go similarity index 99% rename from tools/integration_tests/rapid_appends/appends_test.go rename to tools/integration_tests/rapid_operations/appends_test.go index 6652b3a773..b3119b1dd6 100644 --- a/tools/integration_tests/rapid_appends/appends_test.go +++ b/tools/integration_tests/rapid_operations/appends_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rapid_appends +package rapid_operations import ( "os" diff --git a/tools/integration_tests/rapid_operations/finalize_test.go b/tools/integration_tests/rapid_operations/finalize_test.go new file mode 100644 index 0000000000..7597a18c6c --- /dev/null +++ b/tools/integration_tests/rapid_operations/finalize_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_operations + +import ( + "path" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func (t *FinalizeRapidWritesTestSuite) TestFileClosedInFinalizedState() { + if !t.isFinalizeEnabled { + t.T().Skip("Skipping test since finalize-file-for-rapid is false") + } + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + defer t.deleteUnfinalizedObject() + initialContent := "test content" + bucket := testEnv.storageClient.Bucket(testEnv.cfg.TestBucket) + objHandle := bucket.Object(path.Join(testDirName, t.fileName)) + + operations.CreateFileWithContent(filePath, operations.FilePermission_0600, initialContent, t.T()) + + attrs, err := objHandle.Attrs(testEnv.ctx) + require.NoError(t.T(), err) + assert.False(t.T(), attrs.Finalized.IsZero(), "Finalized field should not be zero for a finalized object") +} + +func (t *FinalizeRapidWritesTestSuite) TestFileClosedInUnfinalizedState() { + if t.isFinalizeEnabled { + t.T().Skip("Skipping test since finalize-file-for-rapid is true") + } + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + defer t.deleteUnfinalizedObject() + initialContent := "test content" + bucket := testEnv.storageClient.Bucket(testEnv.cfg.TestBucket) + objHandle := bucket.Object(path.Join(testDirName, t.fileName)) + + operations.CreateFileWithContent(filePath, operations.FilePermission_0600, initialContent, t.T()) + + attrs, err := objHandle.Attrs(testEnv.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), attrs.Finalized.IsZero(), "Finalized field should be zero for an unfinalized object") +} + +//////////////////////////////////////////////////////////////////////// +// Test Runner +//////////////////////////////////////////////////////////////////////// + +func TestFinalizeRapidWritesTestSuite(t *testing.T) { + RunTests(t, "TestFinalizeRapidWritesTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &FinalizeRapidWritesTestSuite{ + BaseSuite: BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags}, + isFinalizeEnabled: strings.Contains(strings.Join(primaryFlags, " "), "finalize-file-for-rapid=true"), + } + }) +} diff --git a/tools/integration_tests/rapid_operations/helpers_test.go b/tools/integration_tests/rapid_operations/helpers_test.go new file mode 100644 index 0000000000..e6bf89d68c --- /dev/null +++ b/tools/integration_tests/rapid_operations/helpers_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_operations + +import ( + "math/rand/v2" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" +) + +func (t *BaseSuite) createGCSFile(useAppendableAPI bool, fileSize int64) (filePath string, content []byte) { + t.T().Helper() + + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + filePath = path.Join(t.primaryMount.testDirPath, t.fileName) + content, err := operations.GenerateRandomData(fileSize) + require.NoError(t.T(), err) + + // Create file directly in GCS using Go SDK with the chosen API. + err = client.CreateObjectWithAPI(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName), content, useAppendableAPI) + require.NoError(t.T(), err) + return filePath, content +} + +// declare a function type for read and verify +type readAndVerifyFunc func(t *testing.T, filePath string, expectedContent []byte) + +func readSequentiallyAndVerify(t *testing.T, filePath string, expectedContent []byte) { + file, err := os.OpenFile(filePath, os.O_RDONLY, setup.FilePermission_0600) + require.NoError(t, err) + readContent, err := operations.ReadFileSequentially(file, 1024*1024) + + // For sequential reads, we expect the content to be exactly as expected. + require.NoErrorf(t, err, "failed to read file %q sequentially: %v", filePath, err) + require.Equal(t, expectedContent, readContent) +} + +func readRandomlyAndVerify(t *testing.T, filePath string, expectedContent []byte) { + file, err := os.OpenFile(filePath, os.O_RDONLY, setup.FilePermission_0600) + require.NoErrorf(t, err, "failed to open file %q: %v", filePath, err) + defer operations.CloseFileShouldNotThrowError(t, file) // This line is already correct. + if len(expectedContent) == 0 { + t.SkipNow() + } + fileInfo, err := file.Stat() + require.NoError(t, err) + fileSize := fileInfo.Size() + require.GreaterOrEqualf(t, fileSize, int64(len(expectedContent)), "file %q is too small to read %d bytes", filePath, len(expectedContent)) + + // Content to be read from [0, maxOffset) . + maxOffset := len(expectedContent) + // Limit number of reads if the content to read is too small. + numReads := min(maxOffset, 10) + for i := range numReads { + // Ensure offset <= maxOffset-1 . + offset := rand.IntN(maxOffset) + // Ensure (offset+readSize) <= maxOffset and readSize >= 1. + readSize := rand.IntN(maxOffset-offset) + 1 + buffer := make([]byte, readSize) + + n, err := file.ReadAt(buffer, int64(offset)) + + require.NoErrorf(t, err, "Random-read failed at iter#%d to read file %q at [%d, %d): %v", i, filePath, offset, offset+readSize, err) + require.Equalf(t, readSize, n, "failed to read %v bytes from %q at offset %v. Read bytes = %v.", readSize, filePath, offset, n) + require.Equalf(t, expectedContent[offset:offset+n], buffer[:n], "content mismatch in random read at iter#%d at offset [%d, %d): expected %q, got %q", i, offset, offset+readSize, expectedContent[offset:offset+n], buffer[:n]) + } +} diff --git a/tools/integration_tests/rapid_operations/read_routing_test.go b/tools/integration_tests/rapid_operations/read_routing_test.go new file mode 100644 index 0000000000..f553661cec --- /dev/null +++ b/tools/integration_tests/rapid_operations/read_routing_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// limitations under the License. + +package rapid_operations + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/suite" +) + +const ( + fileSizeForReadRouting = 50 * operations.OneMiB +) + +// ReadRoutingTestSuite groups all tests related to read routing across different topologies. +type ReadRoutingTestSuite struct { + BaseSuite +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *ReadRoutingTestSuite) TestSequentialReadAfterAppendableUpload() { + filePath, expectedContent := t.createGCSFile(true, fileSizeForReadRouting) + + readSequentiallyAndVerify(t.T(), filePath, expectedContent) +} + +func (t *ReadRoutingTestSuite) TestRandomReadAfterAppendableUpload() { + filePath, expectedContent := t.createGCSFile(true, fileSizeForReadRouting) + + readRandomlyAndVerify(t.T(), filePath, expectedContent) +} + +func (t *ReadRoutingTestSuite) TestSequentialReadAfterResumableUpload() { + filePath, expectedContent := t.createGCSFile(false, fileSizeForReadRouting) + + readSequentiallyAndVerify(t.T(), filePath, expectedContent) +} + +func (t *ReadRoutingTestSuite) TestRandomReadAfterResumableUpload() { + filePath, expectedContent := t.createGCSFile(false, fileSizeForReadRouting) + + readRandomlyAndVerify(t.T(), filePath, expectedContent) +} + +//////////////////////////////////////////////////////////////////////// +// Test Runner +//////////////////////////////////////////////////////////////////////// + +func TestReadRoutingTestSuite(t *testing.T) { + RunTests(t, "ReadRoutingTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &ReadRoutingTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags}} + }) +} diff --git a/tools/integration_tests/rapid_appends/reads_after_appends_test.go b/tools/integration_tests/rapid_operations/reads_after_appends_test.go similarity index 69% rename from tools/integration_tests/rapid_appends/reads_after_appends_test.go rename to tools/integration_tests/rapid_operations/reads_after_appends_test.go index ee12148300..74c4b0b0a8 100644 --- a/tools/integration_tests/rapid_appends/reads_after_appends_test.go +++ b/tools/integration_tests/rapid_operations/reads_after_appends_test.go @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rapid_appends +package rapid_operations import ( - "math/rand/v2" - "os" "path" "syscall" "testing" @@ -24,58 +22,9 @@ import ( "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) -//////////////////////////////////////////////////////////////////////// -// Helpers -//////////////////////////////////////////////////////////////////////// - -// declare a function type for read and verify -type readAndVerifyFunc func(t *testing.T, filePath string, expectedContent []byte) - -func readSequentiallyAndVerify(t *testing.T, filePath string, expectedContent []byte) { - file, err := os.OpenFile(filePath, os.O_RDONLY, setup.FilePermission_0600) - require.NoError(t, err) - readContent, err := operations.ReadFileSequentially(file, 1024*1024) - - // For sequential reads, we expect the content to be exactly as expected. - require.NoErrorf(t, err, "failed to read file %q sequentially: %v", filePath, err) - require.Equal(t, expectedContent, readContent) -} - -func readRandomlyAndVerify(t *testing.T, filePath string, expectedContent []byte) { - file, err := os.OpenFile(filePath, os.O_RDONLY, setup.FilePermission_0600) - require.NoErrorf(t, err, "failed to open file %q: %v", filePath, err) - defer operations.CloseFileShouldNotThrowError(t, file) // This line is already correct. - if len(expectedContent) == 0 { - t.SkipNow() - } - fileInfo, err := file.Stat() - require.NoError(t, err) - fileSize := fileInfo.Size() - require.GreaterOrEqualf(t, fileSize, int64(len(expectedContent)), "file %q is too small to read %d bytes", filePath, len(expectedContent)) - - // Content to be read from [0, maxOffset) . - maxOffset := len(expectedContent) - // Limit number of reads if the content to read is too small. - numReads := min(maxOffset, 10) - for i := range numReads { - // Ensure offset <= maxOffset-1 . - offset := rand.IntN(maxOffset) - // Ensure (offset+readSize) <= maxOffset and readSize >= 1. - readSize := rand.IntN(maxOffset-offset) + 1 - buffer := make([]byte, readSize) - - n, err := file.ReadAt(buffer, int64(offset)) - - require.NoErrorf(t, err, "Random-read failed at iter#%d to read file %q at [%d, %d): %v", i, filePath, offset, offset+readSize, err) - require.Equalf(t, readSize, n, "failed to read %v bytes from %q at offset %v. Read bytes = %v.", readSize, filePath, offset, n) - require.Equalf(t, expectedContent[offset:offset+n], buffer[:n], "content mismatch in random read at iter#%d at offset [%d, %d): expected %q, got %q", i, offset, offset+readSize, expectedContent[offset:offset+n], buffer[:n]) - } -} - //////////////////////////////////////////////////////////////////////// // Tests for the SingleMountReadsTestSuite //////////////////////////////////////////////////////////////////////// diff --git a/tools/integration_tests/rapid_appends/setup_test.go b/tools/integration_tests/rapid_operations/setup_test.go similarity index 92% rename from tools/integration_tests/rapid_appends/setup_test.go rename to tools/integration_tests/rapid_operations/setup_test.go index ae380bbf67..ed6089a770 100644 --- a/tools/integration_tests/rapid_appends/setup_test.go +++ b/tools/integration_tests/rapid_operations/setup_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rapid_appends +package rapid_operations import ( "context" @@ -29,8 +29,8 @@ import ( ) const ( - testDirName = "RapidAppendsTest" - fileNamePrefix = "rapid-append-file-" + testDirName = "RapidOperationsTest" + fileNamePrefix = "rapid-operations-file-" contentSizeForBW = 3 blockSize = operations.OneMiB numAppends = 2 @@ -57,12 +57,12 @@ func TestMain(m *testing.M) { // 1. Load and parse the common configuration. cfg := test_suite.ReadConfigFile(setup.ConfigFile()) - if len(cfg.RapidAppends) == 0 { + if len(cfg.RapidOperations) == 0 { log.Fatal("No configuration found for RapidAppends in config file.") } testEnv.ctx = context.Background() - testEnv.cfg = &cfg.RapidAppends[0] + testEnv.cfg = &cfg.RapidOperations[0] testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) if !setup.IsZonalBucketRun() && !setup.IsPirloBucketRun() { log.Fatalf("This test package is only compatible for zonal and pirlo bucket runs") @@ -98,7 +98,7 @@ func TestMain(m *testing.M) { } testEnv.cfg.GCSFuseMountedDirectorySecondary = secondaryDir - log.Println("Running static mounting tests for rapid appends...") + log.Println("Running static mounting tests for rapid_operations...") successCode := m.Run() setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) diff --git a/tools/integration_tests/rapid_operations/stat_and_list_test.go b/tools/integration_tests/rapid_operations/stat_and_list_test.go new file mode 100644 index 0000000000..b32d0af368 --- /dev/null +++ b/tools/integration_tests/rapid_operations/stat_and_list_test.go @@ -0,0 +1,89 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_operations + +import ( + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func (t *StatAndListTestSuite) TestStatOfNewFile() { + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + defer t.deleteUnfinalizedObject() + expectedSize := int64(len("test content")) + client.CreateFinalizedObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "test content", t.T()) + + size := operations.RetryUntil(testEnv.ctx, t.T(), 2*time.Second, defaultMetadataCacheTTL, func() (int64, error) { + fi, err := os.Stat(filePath) + if err != nil { + return 0, err + } + if fi.Size() != expectedSize { + return 0, fmt.Errorf("expected size %d, got %d", expectedSize, fi.Size()) + } + return fi.Size(), nil + }) + + require.Equal(t.T(), expectedSize, size) +} + +func (t *StatAndListTestSuite) TestListOfNewFile() { + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + defer t.deleteUnfinalizedObject() + expectedSize := int64(len("test content")) + client.CreateFinalizedObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "test content", t.T()) + + size := operations.RetryUntil(testEnv.ctx, t.T(), 2*time.Second, defaultMetadataCacheTTL, func() (int64, error) { + entries, err := os.ReadDir(t.primaryMount.testDirPath) + if err != nil { + return 0, err + } + for _, entry := range entries { + if entry.Name() == t.fileName { + info, err := entry.Info() + if err != nil { + return 0, err + } + if info.Size() != expectedSize { + return 0, fmt.Errorf("listing expected size %d, got %d", expectedSize, info.Size()) + } + return info.Size(), nil + } + } + return 0, fmt.Errorf("file not found in directory listing") + }) + + require.Equal(t.T(), expectedSize, size) +} + +//////////////////////////////////////////////////////////////////////// +// Test Runner +//////////////////////////////////////////////////////////////////////// + +func TestStatAndListTestSuite(t *testing.T) { + RunTests(t, "TestStatAndListTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &StatAndListTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags}} + }) +} diff --git a/tools/integration_tests/rapid_appends/suites_test.go b/tools/integration_tests/rapid_operations/suites_test.go similarity index 95% rename from tools/integration_tests/rapid_appends/suites_test.go rename to tools/integration_tests/rapid_operations/suites_test.go index f958415a65..c407142098 100644 --- a/tools/integration_tests/rapid_appends/suites_test.go +++ b/tools/integration_tests/rapid_operations/suites_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rapid_appends +package rapid_operations import ( "log" @@ -43,7 +43,7 @@ const ( type mountPoint struct { rootDir string // Root directory of the test folder, which contains mnt and gcsfuse.log. mntDir string // Directory where the GCS bucket is mounted. This is 'mnt' inside rootDir. - testDirPath string // Path to the 'RapidAppendsTest' directory inside mntDir. + testDirPath string // Path to the 'RapidOperationsTest' directory inside mntDir. logFilePath string // Path to the GCSFuse log file. This is gcsfuse.log inside rootDir. } @@ -71,6 +71,15 @@ type SingleMountAppendsTestSuite struct{ BaseSuite } // DualMountAppendsTestSuite groups general dual-mount tests for append behavior. type DualMountAppendsTestSuite struct{ BaseSuite } +// FinalizeRapidWritesTestSuite groups tests for verifying transition to finalized state. +type FinalizeRapidWritesTestSuite struct { + BaseSuite + isFinalizeEnabled bool +} + +// StatAndListTestSuite groups tests for checking new file discovery. +type StatAndListTestSuite struct{ BaseSuite } + //////////////////////////////////////////////////////////////////////// // Common Suite Logic //////////////////////////////////////////////////////////////////////// diff --git a/tools/integration_tests/run_e2e_tests.sh b/tools/integration_tests/run_e2e_tests.sh deleted file mode 100755 index 33f6eb2390..0000000000 --- a/tools/integration_tests/run_e2e_tests.sh +++ /dev/null @@ -1,643 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. - -# true or false to run e2e tests on installedPackage -RUN_E2E_TESTS_ON_PACKAGE=$1 - -# Pass "true" to skip few non-essential tests. -# By default, this script runs all the integration tests. -SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE=$2 - -# e.g. us-west1 -BUCKET_LOCATION=$3 - -# Pass "true" to run e2e tests on TPC endpoint. -# The default value will be false. -RUN_TEST_ON_TPC_ENDPOINT=false -if [ $4 != "" ]; then - RUN_TEST_ON_TPC_ENDPOINT=$4 -fi -INTEGRATION_TEST_TIMEOUT_IN_MINS=90 - -RUN_TESTS_WITH_PRESUBMIT_FLAG=false -if [ $# -ge 5 ] ; then - # This parameter is set to true by caller, only for presubmit runs. - RUN_TESTS_WITH_PRESUBMIT_FLAG=$5 -fi - -# 6th parameter is set to enable/disable run for zonal bucket(s). -# If it is set to true, then the run will be only on zonal bucket(s), -# otherwise the run will only on non-zonal bucket(s). -RUN_TESTS_WITH_ZONAL_BUCKET=false -if [[ $# -ge 6 ]] ; then - if [[ "$6" == "true" ]]; then - RUN_TESTS_WITH_ZONAL_BUCKET=true - elif [[ "$6" != "false" ]]; then - echo "Error: Invalid value for 6th argument: "$6" . Expected: true or false." - exit 1 - fi -fi - -# 7th parameter is to determine whether we want to disable build by the script -# and let every test package build its own GCSFuse binary. -BUILD_BINARY_IN_SCRIPT=true -if [[ $# -ge 7 ]] ; then - if [[ "$7" == "false" ]]; then - BUILD_BINARY_IN_SCRIPT=false - fi -fi - - -if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then - if [ "${BUCKET_LOCATION}" != "us-west4" ] && [ "${BUCKET_LOCATION}" != "us-central1" ]; then - >&2 echo "For enabling zonal bucket run, BUCKET_LOCATION should be one of: us-west4, us-central1; passed: ${BUCKET_LOCATION}" - exit 1 - fi -fi - -if [ "$#" -lt 3 ] -then - echo "Incorrect number of arguments passed, please refer to the script and pass the three arguments required..." - exit 1 -fi - -if [ "$SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE" == true ]; then - GO_TEST_SHORT_FLAG="-short" - echo "Setting the flag to skip few un-important integration tests." - INTEGRATION_TEST_TIMEOUT_IN_MINS=$((INTEGRATION_TEST_TIMEOUT_IN_MINS-20)) -fi - -# Pass flag "-presubmit" to 'go test' command and lower timeout for presubmit runs. -if [ "$RUN_TESTS_WITH_PRESUBMIT_FLAG" == true ]; then - echo "This is a presubmit-run, which skips some tests." - PRESUBMIT_RUN_FLAG="-presubmit" - INTEGRATION_TEST_TIMEOUT_IN_MINS=$((INTEGRATION_TEST_TIMEOUT_IN_MINS-10)) -fi - -INTEGRATION_TEST_TIMEOUT=""${INTEGRATION_TEST_TIMEOUT_IN_MINS}"m" -echo "Setting the integration test timeout to: $INTEGRATION_TEST_TIMEOUT" - -readonly RANDOM_STRING_LENGTH=5 -# Test directory arrays -TEST_DIR_PARALLEL=( - "monitoring" - "local_file" - "log_rotation" - "mounting" - "read_cache" - # "grpc_validation" - "gzip" - "write_large_files" - "list_large_dir" - "rename_dir_limit" - "read_large_files" - "explicit_dir" - "implicit_dir" - "interrupt" - "operations" - "kernel_list_cache" - "benchmarking" - "mount_timeout" - "stale_handle" - "negative_stat_cache" - "streaming_writes" - "inactive_stream_timeout" - "cloud_profiler" - "release_version" - "readdirplus" - "dentry_cache" - "buffered_read" - "requester_pays_bucket" - "flag_optimizations" - "unsupported_path" - "symlink_handling" -) - -# These tests never become parallel as it is changing bucket permissions. -TEST_DIR_NON_PARALLEL=( - "readonly" - "managed_folders" - "readonly_creds" - "concurrent_operations" -) - -# Subset of TEST_DIR_PARALLEL, -# but only those tests which currently -# pass for zonal buckets. -TEST_DIR_PARALLEL_FOR_ZB=( - "benchmarking" - "explicit_dir" - "gzip" - "implicit_dir" - "interrupt" - "kernel_list_cache" - "local_file" - "log_rotation" - "monitoring" - "mount_timeout" - "mounting" - "negative_stat_cache" - "operations" - "rapid_appends" - "read_cache" - "read_large_files" - "rename_dir_limit" - "stale_handle" - "streaming_writes" - "write_large_files" - "unfinalized_object" - "release_version" - "readdirplus" - "dentry_cache" - "flag_optimizations" - "unsupported_path" - "symlink_handling" -) - -# Subset of TEST_DIR_NON_PARALLEL, -# but only those tests which currently -# pass for zonal buckets. -TEST_DIR_NON_PARALLEL_FOR_ZB=( - "concurrent_operations" - "list_large_dir" - "managed_folders" - "readonly" - "readonly_creds" -) - -# Create a temporary file to store the log file name. -TEST_LOGS_FILE=$(mktemp) - -# This variable will store the path if the script builds GCSFuse binaries (gcsfuse, mount.gcsfuse) -BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR="" -# This variable will hold flag and its value to be passed to GCSFuse tests (--gcsfuse_prebuilt_dir=...) -USE_PREBUILT_GCSFUSE_BINARY="" - -build_gcsfuse_once() { - local build_output_dir # For the final gcsfuse binaries - build_output_dir=$(mktemp -d -t gcsfuse_e2e_run_build_XXXXXX) - echo "GCSFuse binaries will be built in ${build_output_dir}..." - - local gcsfuse_src_dir - # Determine GCSFuse source directory - # If this script is in tools/integration_tests, project root is ../../ - SCRIPT_DIR_REALPATH=$(realpath "$(dirname "${BASH_SOURCE[0]}")") - gcsfuse_src_dir=$(realpath "${SCRIPT_DIR_REALPATH}/../../") - - if [[ ! -f "${gcsfuse_src_dir}/go.mod" ]]; then - echo "Error: Could not reliably determine GCSFuse project root from ${SCRIPT_DIR_REALPATH}. Expected go.mod at ${gcsfuse_src_dir}" >&2 - rm -rf "${build_output_dir}" - exit 1 - fi - echo "Using GCSFuse source directory: ${gcsfuse_src_dir}" - - echo "Building GCSFuse using 'go run ./tools/build_gcsfuse/main.go'..." - (cd "${gcsfuse_src_dir}" && go run ./tools/build_gcsfuse/main.go . "${build_output_dir}" "e2e-$(date +%s)") - if [ $? -ne 0 ]; then - echo "Error building GCSFuse binaries using 'go run ./tools/build_gcsfuse/main.go'." - rm -rf "${build_output_dir}" # Clean up created temp dir - return 1 - fi - - # Set the directory path for use by the script (to form the go test flag) - BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR="${build_output_dir}" - echo "GCSFuse binaries built by script in: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" - echo "GCSFuse executable: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}/bin/gcsfuse" - return 0 -} - - -cleanup_gcsfuse_once() { - if [ -n "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" ] && [ -d "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" ]; then - echo "Cleaning up GCSFuse build directory created by script: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" - rm -rf "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" - fi -} - -# Delete contents of the buckets (and then the buckets themselves) whose names are in the passed file. -# Args: -function delete_buckets_listed_in_file() { - local bucketNamesFile="${@}" - if test -f "${bucketNamesFile}"; then - cat "${bucketNamesFile}" | while read bucket; do - # Only if bucket-name is non-empty and contains - # something other than spaces. - if [ -n "${bucket}" ] && [ -n "${bucket// }" ]; then - # Delete the bucket and its contents. - if ! gcloud -q storage rm -r --verbosity=none gs://${bucket} ; then - >&2 echo "Failed to delete bucket ${bucket} !" - fi - fi - done - # At the end, delete the bucket-names file itself. - rm "${bucketNamesFile}" - else - echo "file ${bucketNamesFile} not found !" - fi -} - -function upgrade_gcloud_version() { - # Install latest gcloud. - ./perfmetrics/scripts/install_latest_gcloud.sh - export PATH="/usr/local/google-cloud-sdk/bin:$PATH" - export CLOUDSDK_PYTHON="$HOME/.local/python-3.11.9/bin/python3.11" - export PATH="$HOME/.local/python-3.11.9/bin:$PATH"" -} - -function install_packages() { - # Install required go version. - ./perfmetrics/scripts/install_go.sh "$(cat .go-version)" - export PATH="/usr/local/go/bin:$PATH" - - sudo apt-get update - sudo apt-get install -y python3 - # install python3-setuptools tools. - sudo apt-get install -y gcc python3-dev python3-setuptools - # Downloading composite object requires integrity checking with CRC32c in gsutil. - # it requires to install crcmod. - sudo apt install -y python3-crcmod -} - -function create_bucket() { - bucket_prefix=$1 - local -r project_id="gcs-fuse-test-ml" - # Generate bucket name with random string - bucket_name=${bucket_prefix}$(date +%Y%m%d-%H%M%S)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) - # We are using gcloud alpha because gcloud storage is giving issues running on Kokoro - gcloud alpha storage buckets create gs://$bucket_name --project=$project_id --location=$BUCKET_LOCATION --uniform-bucket-level-access - echo $bucket_name -} - -function create_hns_bucket() { - local -r hns_project_id="gcs-fuse-test" - # Generate bucket name with random string. - # Adding prefix `golang-grpc-test` to white list the bucket for grpc - # so that we can run grpc related e2e tests. - bucket_name="golang-grpc-test-gcsfuse-e2e-tests-hns-"$(date +%Y%m%d-%H%M%S)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) - gcloud alpha storage buckets create gs://$bucket_name --project=$hns_project_id --location=$BUCKET_LOCATION --uniform-bucket-level-access --enable-hierarchical-namespace - echo "$bucket_name" -} - -function create_zonal_bucket() { - local -r project_id="gcs-fuse-test-ml" - local -r region=${BUCKET_LOCATION} - local -r zone=${region}"-a" - - local -r hns_project_id="gcs-fuse-test" - # Generate bucket name with random string. - bucket_name="gcsfuse-e2e-tests-zb-"$(date +%Y%m%d-%H%M%S)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) - gcloud alpha storage buckets create gs://$bucket_name --project=$project_id --location=$region --placement=${zone} --default-storage-class=RAPID --uniform-bucket-level-access --enable-hierarchical-namespace - echo "${bucket_name}" -} - -function run_non_parallel_tests() { - local exit_code=0 - local -n test_array=$1 - local bucket_name_non_parallel=$2 - local zonal=false - if [ $# -ge 3 ] && [ "$3" = "true" ] ; then - zonal=true - fi - - for test_dir_np in "${test_array[@]}" - do - test_path_non_parallel="./tools/integration_tests/$test_dir_np" - # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming - # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). - local log_file="/tmp/${test_dir_np}_${bucket_name_non_parallel}.log" - echo $log_file >> $TEST_LOGS_FILE - - # Executing integration tests - echo "Running test package in non-parallel (with zonal=${zonal}): ${test_dir_np} ..." - GODEBUG=asyncpreemptoff=1 go test $test_path_non_parallel -p 1 $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --zonal=${zonal} --integrationTest -v --testbucket=$bucket_name_non_parallel --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE $USE_PREBUILT_GCSFUSE_BINARY -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 - exit_code_non_parallel=$? - if [ $exit_code_non_parallel != 0 ]; then - exit_code=$exit_code_non_parallel - echo "test fail in non parallel on package (with zonal=${zonal}): " $test_dir_np - else - echo "Passed test package in non-parallel (with zonal=${zonal}): " $test_dir_np - fi - done - return $exit_code -} - -function run_parallel_tests() { - local exit_code=0 - local -n test_array=$1 - local bucket_name_parallel=$2 - local zonal=false - if [ $# -ge 3 ] && [ "$3" = "true" ] ; then - zonal=true - fi - local benchmark_flags="" - declare -A pids - - for test_dir_p in "${test_array[@]}" - do - # Unlike regular tests,benchmark tests are not executed by default when using go test . - # The -bench flag yells go test to run the benchmark tests and report their results by - # enabling the benchmarking framework. - # The -benchtime flag specifies exact number of iterations a benchmark should run , in this - # case, setting this to 100 to avoid flakiness. - if [ $test_dir_p == "benchmarking" ]; then - benchmark_flags="-bench=. -benchtime=100x" - fi - test_path_parallel="./tools/integration_tests/$test_dir_p" - # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming - # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). - local log_file="/tmp/${test_dir_p}_${bucket_name_parallel}.log" - echo $log_file >> $TEST_LOGS_FILE - # Executing integration tests - echo "Queueing up test package in parallel (with zonal=${zonal}): ${test_dir_p} ..." - GODEBUG=asyncpreemptoff=1 go test $test_path_parallel $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --zonal=${zonal} $benchmark_flags -p 1 --integrationTest -v --testbucket=$bucket_name_parallel --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE $USE_PREBUILT_GCSFUSE_BINARY -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 & - pid=$! # Store the PID of the background process - echo "Queued up test package in parallel (with zonal=${zonal}): ${test_dir_p} with pid=${pid}" - pids[${test_dir_p}]=${pid} # Optionally add the PID to an array for later - done - - # Wait for processes and collect exit codes - for package_name in "${!pids[@]}"; do - pid="${pids[${package_name}]}" - echo "Waiting on test package ${package_name} (with zonal=${zonal}) through pid=${pid} ..." - # What if the process for this test package completed long back and its PID got - # re-assigned to another process since then ? - wait $pid - exit_code_parallel=$? - if [ $exit_code_parallel != 0 ]; then - exit_code=$exit_code_parallel - echo "test fail in parallel on package (with zonal=${zonal}): " $package_name - else - echo "Passed test package in parallel (with zonal=${zonal}): " $package_name - fi - done - return $exit_code -} - -function print_test_logs() { - readarray -t test_logs_array < "$TEST_LOGS_FILE" - rm "$TEST_LOGS_FILE" - for test_log_file in "${test_logs_array[@]}" - do - log_file=${test_log_file} - if [ -f "$log_file" ]; then - echo "=== Log for ${test_log_file} ===" - cat "$log_file" - echo "=========================================" - fi - done -} - -function run_e2e_tests_for_flat_bucket() { - # Adding prefix `golang-grpc-test` to white list the bucket for grpc so that - # we can run grpc related e2e tests. - bucketPrefix="golang-grpc-test-gcsfuse-np-e2e-tests-" - bucket_name_non_parallel=$(create_bucket $bucketPrefix) - echo "Bucket name for non parallel tests: "$bucket_name_non_parallel - echo ${bucket_name_non_parallel}>>"${bucketNamesFile}" - - bucketPrefix="golang-grpc-test-gcsfuse-p-e2e-tests-" - bucket_name_parallel=$(create_bucket $bucketPrefix) - echo "Bucket name for parallel tests: "$bucket_name_parallel - echo ${bucket_name_parallel}>>"${bucketNamesFile}" - - echo "Running parallel tests..." - run_parallel_tests TEST_DIR_PARALLEL $bucket_name_parallel & - parallel_tests_pid=$! - - echo "Running non parallel tests ..." - run_non_parallel_tests TEST_DIR_NON_PARALLEL $bucket_name_non_parallel & - non_parallel_tests_pid=$! - - # Wait for all tests to complete. - wait $parallel_tests_pid - parallel_tests_exit_code=$? - wait $non_parallel_tests_pid - non_parallel_tests_exit_code=$? - - if [ $non_parallel_tests_exit_code != 0 ] || [ $parallel_tests_exit_code != 0 ]; - then - return 1 - fi - return 0 -} - -function run_e2e_tests_for_hns_bucket(){ - hns_bucket_name_parallel_group=$(create_hns_bucket) - echo "Hns Bucket Created: "$hns_bucket_name_parallel_group - echo ${hns_bucket_name_parallel_group}>>"${bucketNamesFile}" - - hns_bucket_name_non_parallel_group=$(create_hns_bucket) - echo "Hns Bucket Created: "$hns_bucket_name_non_parallel_group - echo ${hns_bucket_name_non_parallel_group}>>"${bucketNamesFile}" - - echo "Running tests for HNS bucket" - run_parallel_tests TEST_DIR_PARALLEL "$hns_bucket_name_parallel_group" & - parallel_tests_hns_group_pid=$! - run_non_parallel_tests TEST_DIR_NON_PARALLEL "$hns_bucket_name_non_parallel_group" & - non_parallel_tests_hns_group_pid=$! - - # Wait for all tests to complete. - wait $parallel_tests_hns_group_pid - parallel_tests_hns_group_exit_code=$? - wait $non_parallel_tests_hns_group_pid - non_parallel_tests_hns_group_exit_code=$? - - if [ $parallel_tests_hns_group_exit_code != 0 ] || [ $non_parallel_tests_hns_group_exit_code != 0 ]; - then - return 1 - fi - return 0 -} - -function run_e2e_tests_for_zonal_bucket(){ - zonal_bucket_name_parallel_group=$(create_zonal_bucket) - echo "Zonal Bucket Created for parallel tests: "$zonal_bucket_name_parallel_group - echo ${zonal_bucket_name_parallel_group}>>"${bucketNamesFile}" - - zonal_bucket_name_non_parallel_group=$(create_zonal_bucket) - echo "Zonal Bucket Created for non-parallel tests: "$zonal_bucket_name_non_parallel_group - echo ${zonal_bucket_name_non_parallel_group}>>"${bucketNamesFile}" - - echo "Running tests for ZONAL bucket" - run_parallel_tests TEST_DIR_PARALLEL_FOR_ZB "$zonal_bucket_name_parallel_group" true & - parallel_tests_zonal_group_pid=$! - run_non_parallel_tests TEST_DIR_NON_PARALLEL_FOR_ZB "$zonal_bucket_name_non_parallel_group" true & - non_parallel_tests_zonal_group_pid=$! - - # Wait for all tests to complete. - wait $parallel_tests_zonal_group_pid - parallel_tests_zonal_group_exit_code=$? - wait $non_parallel_tests_zonal_group_pid - non_parallel_tests_zonal_group_exit_code=$? - - if [ $parallel_tests_zonal_group_exit_code != 0 ] || [ $non_parallel_tests_zonal_group_exit_code != 0 ]; - then - return 1 - fi - return 0 -} - -function run_e2e_tests_for_tpc() { - local bucket=$1 - if [ "$bucket" == "" ]; - then - echo "Bucket name is required" - return 1 - fi - - # Clean bucket before testing. - gcloud --verbosity=error storage rm -r gs://"$bucket"/* - - # Run Operations e2e tests in TPC to validate all the functionality. - GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... --testOnTPCEndPoint=$RUN_TEST_ON_TPC_ENDPOINT $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --zonal=false -p 1 --integrationTest -v --testbucket="$bucket" --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE $USE_PREBUILT_GCSFUSE_BINARY -timeout $INTEGRATION_TEST_TIMEOUT - exit_code=$? - - set -e - - # Delete data after testing. - gcloud --verbosity=error storage rm -r gs://"$bucket"/* - - if [ $exit_code != 0 ]; - then - return 1 - fi - return 0 -} - -function run_e2e_tests_for_emulator() { - ./tools/integration_tests/emulator_tests/emulator_tests.sh $RUN_E2E_TESTS_ON_PACKAGE -} - -function main(){ - # The name of a file containing the names of all the - # buckets to be cleaned-up while exiting this program. - bucketNamesFile=$(realpath ./bucketNames)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) - # Delete all these buckets when the program exits. - # Cleanup fuse build folder if created - trap "cleanup_gcsfuse_once; delete_buckets_listed_in_file ${bucketNamesFile}" EXIT - - set -e - - upgrade_gcloud_version - - install_packages - - set +e - - # Decide whether to build GCSFuse based on RUN_E2E_TESTS_ON_PACKAGE - if [ "$RUN_E2E_TESTS_ON_PACKAGE" != "true" ] && [ "$BUILD_BINARY_IN_SCRIPT" == "true" ]; then - echo "RUN_E2E_TESTS_ON_PACKAGE is not 'true' (value: '${RUN_E2E_TESTS_ON_PACKAGE}') and BUILD_BINARY_IN_SCRIPT is 'true'. Building GCSFuse..." - build_gcsfuse_once - if [ $? -ne 0 ]; then - echo "build_gcsfuse_once failed. Exiting." - # The trap will handle cleanup - exit 1 - fi - - USE_PREBUILT_GCSFUSE_BINARY="--gcsfuse_prebuilt_dir=${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" - echo "Script built GCSFuse at: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" - fi - - #run integration tests - exit_code=0 - - if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then - run_e2e_tests_for_zonal_bucket & - e2e_tests_zonal_bucket_pid=$! - wait $e2e_tests_zonal_bucket_pid - e2e_tests_zonal_bucket_status=$? - - if [ $e2e_tests_zonal_bucket_status != 0 ]; then - echo "The e2e tests for zonal bucket failed.." - exit_code=1 - fi - else - # Run tpc test and exit in case RUN_TEST_ON_TPC_ENDPOINT is true. - if [ "$RUN_TEST_ON_TPC_ENDPOINT" == true ]; then - # Run tests for flat bucket - run_e2e_tests_for_tpc gcsfuse-e2e-tests-tpc & - e2e_tests_tpc_flat_bucket_pid=$! - # Run tests for hns bucket - run_e2e_tests_for_tpc gcsfuse-e2e-tests-tpc-hns & - e2e_tests_tpc_hns_bucket_pid=$! - - wait $e2e_tests_tpc_flat_bucket_pid - e2e_tests_tpc_flat_bucket_status=$? - - wait $e2e_tests_tpc_hns_bucket_pid - e2e_tests_tpc_hns_bucket_status=$? - - if [ $e2e_tests_tpc_flat_bucket_status != 0 ]; - then - echo "The e2e tests for flat bucket failed.." - exit 1 - fi - if [ $e2e_tests_tpc_hns_bucket_status != 0 ]; - then - echo "The e2e tests for hns bucket failed.." - exit 1 - fi - # Exit to prevent the following code from executing for TPC. - exit 0 - fi - - run_e2e_tests_for_hns_bucket & - e2e_tests_hns_bucket_pid=$! - - run_e2e_tests_for_flat_bucket & - e2e_tests_flat_bucket_pid=$! - - run_e2e_tests_for_emulator & - e2e_tests_emulator_pid=$! - - wait $e2e_tests_emulator_pid - e2e_tests_emulator_status=$? - - wait $e2e_tests_flat_bucket_pid - e2e_tests_flat_bucket_status=$? - - wait $e2e_tests_hns_bucket_pid - e2e_tests_hns_bucket_status=$? - - if [ $e2e_tests_flat_bucket_status != 0 ]; - then - echo "The e2e tests for flat bucket failed.." - exit_code=1 - fi - - if [ $e2e_tests_hns_bucket_status != 0 ]; - then - echo "The e2e tests for hns bucket failed.." - exit_code=1 - fi - - if [ $e2e_tests_emulator_status != 0 ]; - then - echo "The e2e tests for emulator failed.." - exit_code=1 - fi - fi - - set -e - - print_test_logs - - exit $exit_code -} - -#Main method to run script -main diff --git a/tools/integration_tests/symlink_handling/symlink_suites_test.go b/tools/integration_tests/symlink_handling/symlink_suites_test.go index 47df874f11..8f0cfbb94c 100644 --- a/tools/integration_tests/symlink_handling/symlink_suites_test.go +++ b/tools/integration_tests/symlink_handling/symlink_suites_test.go @@ -85,18 +85,6 @@ func (s *BaseSymlinkSuite) createSymlink(linkName, target string) string { return linkPath } -func (s *BaseSymlinkSuite) createTempFile() string { - targetFile, err := os.CreateTemp("", "symlink-target") - s.Require().NoError(err) - s.T().Cleanup(func() { - if err := os.Remove(targetFile.Name()); err != nil { - s.T().Logf("Error removing temporary file %s: %v", targetFile.Name(), err) - } - }) - s.Require().NoError(targetFile.Close()) - return targetFile.Name() -} - // validateBackingGCSObjectForSymlink validates the GCS object created for a symlink. func (s *BaseSymlinkSuite) validateBackingGCSObjectForSymlink(linkName, target string, isStandardSymlink bool) { bucketName, objectName := setup.GetBucketAndObjectBasedOnTypeOfMount(path.Join(TestDirName, linkName)) diff --git a/tools/integration_tests/test_config.yaml b/tools/integration_tests/test_config.yaml index 35110734ea..aec4c6f84c 100644 --- a/tools/integration_tests/test_config.yaml +++ b/tools/integration_tests/test_config.yaml @@ -1707,7 +1707,7 @@ kernel_list_cache: run: "TestDisabledKernelListCacheTest" run_on_gke: true -rapid_appends: +rapid_operations: - mounted_directory: "${MOUNTED_DIR}" mounted_directory_secondary: "${MOUNTED_DIR_SECONDARY}" test_bucket: "${BUCKET_NAME}" @@ -1857,6 +1857,57 @@ rapid_appends: same_zone: true different_zone: false run_on_gke: true + - run: TestFinalizeRapidWritesTestSuite + flags: + - "--finalize-file-for-rapid=true" + - "--finalize-file-for-rapid=false" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + - run: TestFinalizeRapidWritesTestSuite + flags: + - "--experimental-enable-pirlo,--enable-rapid-writes=true,--finalize-file-for-rapid=true" + - "--experimental-enable-pirlo,--enable-rapid-writes=true,--finalize-file-for-rapid=false" + run_on_pirlo: + flat: + same_zone: true + different_zone: false + hns: + same_zone: true + different_zone: false + run_on_gke: true + - run: TestStatAndListTestSuite + flags: + - "--metadata-cache-ttl-secs=0" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + - run: TestStatAndListTestSuite + flags: + - "--experimental-enable-pirlo,--enable-rapid-writes=true,--finalize-file-for-rapid=false,--metadata-cache-ttl-secs=0" + run_on_pirlo: + flat: + same_zone: true + different_zone: false + hns: + same_zone: true + different_zone: false + run_on_gke: true + - run: ReadRoutingTestSuite + flags: + - "--experimental-enable-pirlo" + run_on_pirlo: + flat: + same_zone: true + different_zone: true + hns: + same_zone: true + different_zone: true + run_on_gke: true monitoring: - mounted_directory: "${MOUNTED_DIR}" diff --git a/tools/integration_tests/util/client/storage_client.go b/tools/integration_tests/util/client/storage_client.go index f6fcfa648a..d3a76a57b0 100644 --- a/tools/integration_tests/util/client/storage_client.go +++ b/tools/integration_tests/util/client/storage_client.go @@ -29,7 +29,6 @@ import ( "runtime" "strings" "sync" - "testing" "time" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" @@ -277,6 +276,24 @@ func CreateFinalizedObjectOnGCS(ctx context.Context, client *storage.Client, obj return nil } +// CreateObjectWithAPI creates an object on GCS with the given content, allowing the choice of whether to use the appendable API. +func CreateObjectWithAPI(ctx context.Context, client *storage.Client, object string, content []byte, useAppendableAPI bool) error { + bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) + o := getBucketHandle(client, bucket).Object(object) + + wc := o.NewWriter(ctx) + wc.Append = useAppendableAPI + wc.FinalizeOnClose = true + + if _, err := wc.Write(content); err != nil { + return fmt.Errorf("wc.Write failed for object %q: %w", object, err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("wc.Close failed for object %q: %w", object, err) + } + return nil +} + // CreateStorageClientWithCancel creates a new storage client with a cancelable context and returns a function that can be used to cancel the client's operations func CreateStorageClientWithCancel(ctx *context.Context, storageClient **storage.Client) func() error { var err error @@ -297,35 +314,6 @@ func CreateStorageClientWithCancel(ctx *context.Context, storageClient **storage } } -// DownloadObjectFromGCS downloads an object to a local file. -func DownloadObjectFromGCS(gcsFile string, destFileName string, t *testing.T) error { - bucket, gcsFile := setup.GetBucketAndObjectBasedOnTypeOfMount(gcsFile) - - ctx := context.Background() - var storageClient *storage.Client - closeStorageClient := CreateStorageClientWithCancel(&ctx, &storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - f := operations.CreateFile(destFileName, setup.FilePermission_0600, t) - defer operations.CloseFileShouldNotThrowError(t, f) - - rc, err := storageClient.Bucket(bucket).Object(gcsFile).NewReader(ctx) - if err != nil { - return fmt.Errorf("Object(%q).NewReader: %w", gcsFile, err) - } - defer rc.Close() - - if _, err := io.Copy(f, rc); err != nil { - return fmt.Errorf("io.Copy: %w", err) - } - - return nil -} - func DeleteObjectOnGCS(ctx context.Context, client *storage.Client, objectName string) error { bucket, _ := setup.GetBucketAndObjectBasedOnTypeOfMount("") diff --git a/tools/integration_tests/util/creds_tests/creds.go b/tools/integration_tests/util/creds_tests/creds.go index 50c7014b93..b4ca57971f 100644 --- a/tools/integration_tests/util/creds_tests/creds.go +++ b/tools/integration_tests/util/creds_tests/creds.go @@ -213,15 +213,3 @@ func RunTestsForDifferentAuthMethods(ctx context.Context, cfg *test_suite.TestCo return successCode } - -// Deprecated: Use RunTestsForDifferentAuthMethods instead. -// TODO(b/438068132): cleanup deprecated methods after migration is complete. -func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Context, storageClient *storage.Client, testFlagSet [][]string, permission string, m *testing.M) (successCode int) { - config := &test_suite.TestConfig{ - TestBucket: setup.TestBucket(), - GKEMountedDirectory: setup.MountedDirectory(), - GCSFuseMountedDirectory: setup.MntDir(), - LogFile: setup.LogFile(), - } - return RunTestsForDifferentAuthMethods(ctx, config, storageClient, testFlagSet, permission, m) -} diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index b9a174aaae..e29ca8ee31 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -37,17 +37,6 @@ func MountGcsfuseWithDynamicMountingWithConfig(cfg *test_suite.TestConfig, flags return err } -// MountGcsfuseWithDynamicMounting is deprecated. Use MountGcsfuseWithDynamicMountingWithConfig instead. -func MountGcsfuseWithDynamicMounting(flags []string) (err error) { - cfg := &test_suite.TestConfig{ - GKEMountedDirectory: setup.MountedDirectory(), - GCSFuseMountedDirectory: setup.MntDir(), - TestBucket: setup.TestBucket(), - LogFile: setup.LogFile(), - } - return MountGcsfuseWithDynamicMountingWithConfig(cfg, flags) -} - func runTestsOnGivenMountedTestBucket(cfg *test_suite.TestConfig, flags [][]string, rootMntDir string, m *testing.M) (successCode int) { for i := range flags { if err := MountGcsfuseWithDynamicMountingWithConfig(cfg, flags[i]); err != nil { diff --git a/tools/integration_tests/util/operations/dir_operations.go b/tools/integration_tests/util/operations/dir_operations.go index 6077ede50c..9b58b659c3 100644 --- a/tools/integration_tests/util/operations/dir_operations.go +++ b/tools/integration_tests/util/operations/dir_operations.go @@ -26,7 +26,6 @@ import ( "path" "path/filepath" "strconv" - "strings" "sync" "testing" ) @@ -211,30 +210,3 @@ func DirSizeMiB(dirPath string) (dirSizeMB int64, err error) { return dirSizeMB, err } - -func DeleteManagedFoldersInBucket(managedFolderPath, bucket string) { - gcloudDeleteManagedFolderCmd := fmt.Sprintf("alpha storage rm -r gs://%s/%s", bucket, managedFolderPath) - - _, err := ExecuteGcloudCommand(gcloudDeleteManagedFolderCmd) - if err != nil && !strings.Contains(err.Error(), "The following URLs matched no objects or files") { - log.Fatalf("Error while deleting managed folder: %v", err) - } -} - -func CreateManagedFoldersInBucket(managedFolderPath, bucket string) { - gcloudCreateManagedFolderCmd := fmt.Sprintf("alpha storage managed-folders create gs://%s/%s", bucket, managedFolderPath) - - _, err := ExecuteGcloudCommand(gcloudCreateManagedFolderCmd) - if err != nil && !strings.Contains(err.Error(), "The specified managed folder already exists") { - log.Fatalf("Error while creating managed folder: %v", err) - } -} - -func CopyFileInBucket(srcfilePath, destFilePath, bucket string, t *testing.T) { - gcloudCopyFileCmd := fmt.Sprintf("alpha storage cp %s gs://%s/%s/", srcfilePath, bucket, destFilePath) - - _, err := ExecuteGcloudCommand(gcloudCopyFileCmd) - if err != nil { - t.Fatalf("Error while copying file in bucket: %v", err) - } -} diff --git a/tools/integration_tests/util/operations/file_operations.go b/tools/integration_tests/util/operations/file_operations.go index 3c9ad02c26..1eb54c10f3 100644 --- a/tools/integration_tests/util/operations/file_operations.go +++ b/tools/integration_tests/util/operations/file_operations.go @@ -27,7 +27,6 @@ import ( "log" "os" "path" - "strconv" "strings" "syscall" "testing" @@ -454,42 +453,6 @@ func AreFilesIdentical(filepath1, filepath2 string) (bool, error) { return true, nil } -// Returns size of a give GCS object with path (without 'gs://'). -// Fails if the object doesn't exist or permission to read object's metadata is not -// available. -func GetGcsObjectSize(gcsObjPath string) (int, error) { - stdout, err := ExecuteGcloudCommandf("storage du -s gs://%s", gcsObjPath) - if err != nil { - return 0, err - } - - // The above gcloud command returns output in the following format: - // - // So, we need to pick out only the first string before ' '. - gcsObjectSize, err := strconv.Atoi(strings.TrimSpace(strings.Split(string(stdout), " ")[0])) - if err != nil { - return gcsObjectSize, err - } - - return gcsObjectSize, nil -} - -// Deletes a given GCS object (with path without 'gs://'). -// Fails if the object doesn't exist or permission to delete object is not -// available. -func DeleteGcsObject(gcsObjPath string) error { - _, err := ExecuteGcloudCommandf("rm gs://%s", gcsObjPath) - return err -} - -// Clears cache-control attributes on given GCS object (with path without 'gs://'). -// Fails if the file doesn't exist or permission to modify object's metadata is not -// available. -func ClearCacheControlOnGcsObject(gcsObjPath string) error { - _, err := ExecuteGcloudCommandf("storage objects update --cache-control='' gs://%s", gcsObjPath) - return err -} - func CreateFile(filePath string, filePerms os.FileMode, t testing.TB) (f *os.File) { t.Helper() // Creating a file shouldn't create file on GCS. @@ -690,17 +653,6 @@ func CalculateFileCRC32(filePath string) (uint32, error) { return CalculateCRC32(file) } -// SizeOfFile returns the size of the given file by path. -// by invoking a stat call on it. -func SizeOfFile(filepath string) (size int64, err error) { - fstat, err := StatFile(filepath) - if err != nil { - return 0, err - } - - return (*fstat).Size(), nil -} - func writeGzipToFile(f *os.File, filepath, content string, contentSize int) (string, error) { w := gzip.NewWriter(f) if w == nil { diff --git a/tools/integration_tests/util/operations/operations.go b/tools/integration_tests/util/operations/operations.go index 4c02e78ef5..67e191996b 100644 --- a/tools/integration_tests/util/operations/operations.go +++ b/tools/integration_tests/util/operations/operations.go @@ -34,14 +34,6 @@ func GenerateRandomData(sizeInBytes int64) ([]byte, error) { return data, nil } -// Executes any given tool (e.g. gsutil/gcloud) with given args. -func executeToolCommandf(tool string, format string, args ...any) ([]byte, error) { - cmdArgs := tool + " " + fmt.Sprintf(format, args...) - cmd := exec.Command("/bin/bash", "-c", cmdArgs) - - return runCommand(cmd) -} - // Executes any given tool (e.g. gsutil/gcloud). func executeToolCommand(tool string, command string) ([]byte, error) { cmdArgs := tool + " " + command @@ -74,11 +66,6 @@ func runCommand(cmd *exec.Cmd) ([]byte, error) { return stdout.Bytes(), nil } -// ExecuteGcloudCommandf executes any given gcloud command with given args. -func ExecuteGcloudCommandf(format string, args ...any) ([]byte, error) { - return executeToolCommandf("gcloud", format, args...) -} - // ExecuteGcloudCommand executes any given gcloud command. func ExecuteGcloudCommand(command string) ([]byte, error) { return executeToolCommand("gcloud", command) diff --git a/tools/integration_tests/util/operations/string_operations.go b/tools/integration_tests/util/operations/string_operations.go index 41c789bfbd..fbb89f3f1e 100644 --- a/tools/integration_tests/util/operations/string_operations.go +++ b/tools/integration_tests/util/operations/string_operations.go @@ -22,22 +22,6 @@ import ( "github.com/google/uuid" ) -func VerifyExpectedSubstrings(t *testing.T, input string, expectedSubstrings []string) { - for _, expectedSubstring := range expectedSubstrings { - if !strings.Contains(input, expectedSubstring) { - t.Errorf("input does not contain expected substring (%q)", expectedSubstring) - } - } -} - -func VerifyUnexpectedSubstrings(t *testing.T, input string, unexpectedSubstrings []string) { - for _, unexpectedSubstring := range unexpectedSubstrings { - if strings.Contains(input, unexpectedSubstring) { - t.Errorf("input contains unexpected substring (%q)", unexpectedSubstring) - } - } -} - func GetRandomName(t *testing.T) string { id, err := uuid.NewRandom() if err != nil { diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index 4eff40dd29..2c6657d75c 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -35,8 +35,6 @@ import ( auth2 "github.com/googlecloudplatform/gcsfuse/v3/internal/auth" "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" "github.com/googlecloudplatform/gcsfuse/v3/tools/util" - "go.opentelemetry.io/contrib/detectors/gcp" - "go.opentelemetry.io/otel/sdk/resource" "google.golang.org/api/iterator" "google.golang.org/api/option" ) @@ -59,8 +57,6 @@ const ( PathEnvVariable = "PATH" GCSFuseLogFilePrefix = "gcsfuse-failed-integration-test-logs-" ProxyServerLogFilePrefix = "proxy-server-failed-integration-test-logs-" - zoneMatcherRegex = "^[a-z]+-[a-z0-9]+-[a-z]$" - regionMatcherRegex = "^[a-z]+-[a-z0-9]+$" unsupportedCharactersInTestBucket = " " ) @@ -399,17 +395,6 @@ func IgnoreTestIfIntegrationTestFlagIsSet(t *testing.T) { } } -// IgnoreTestIfIntegrationTestFlagIsNotSet helps skip a test if --integrationTest flag is not set. -// If the test uses TestMain, then one usually calls os.Exit() to skip the test, -// but for non-TestMain tests, this helps skip integration tests if --integrationTest has not been passed. -func IgnoreTestIfIntegrationTestFlagIsNotSet(t *testing.T) { - flag.Parse() - - if !*integrationTest { - t.SkipNow() - } -} - func IgnoreTestIfPresubmitFlagIsSet(b *testing.B) { flag.Parse() @@ -427,15 +412,6 @@ func ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() { } } -// Deprecated: Use RunTestsForMountedDirectory instead. -// TODO(b/438068132): cleanup deprecated methods after migration is complete. -func RunTestsForMountedDirectoryFlag(m *testing.M) { - // Execute tests for the mounted directory. - if *mountedDirectory != "" { - os.Exit(RunTestsForMountedDirectory(*mountedDirectory, m)) - } -} - // RunTestsForMountedDirectory executes tests for the mounted directory. // User is expected to ensure that this function is called when mounted directory is set. // Returns exit code. @@ -699,21 +675,6 @@ func SetGlobalVars(cfg *test_suite.TestConfig) { onlyDirMounted = cfg.OnlyDir } -// Explicitly set the enable-hns config flag to true when running tests on the HNS bucket. -func AddHNSFlagForHierarchicalBucket(ctx context.Context, storageClient *storage.Client) ([]string, error) { - if !IsHierarchicalBucket(ctx, storageClient) { - return nil, fmt.Errorf("bucket is not Hierarchical") - } - - var flags []string - mountConfig4 := map[string]any{ - "enable-hns": true, - } - filePath4 := YAMLConfigFile(mountConfig4, "config_hns.yaml") - flags = append(flags, "--config-file="+filePath4) - return flags, nil -} - func separateBucketAndObjectName(bucket, object string) (string, string) { bucketAndObjectPath := strings.SplitN(bucket, "/", 2) bucket = bucketAndObjectPath[0] @@ -810,25 +771,6 @@ func RunTestsOnlyForStaticMount(mountDir string, t *testing.T) { } } -// AppendFlagsToAllFlagsInTheFlagsSet appends each flag in newFlags to every flags present in the -// flagsSet. -// Input flagsSet: [][]string{{"--x", "--y"}, {"--x", "--z"}} -// Input newFlags: {"--a", "--b", ""} -// Output modified flagsSet: [][]string{{"--x", "--y", "--a"}, {"--x", "--z", "--a"},{"--x", "--y", "--b"},{"--x", "--z", "--b"},{"--x", "--y"}, {"--x", "--z"}} -func AppendFlagsToAllFlagsInTheFlagsSet(flagsSet *[][]string, newFlags ...string) { - var resultFlagsSet [][]string - for _, flags := range *flagsSet { - for _, newFlag := range newFlags { - f := flags - if strings.Compare(newFlag, "") != 0 { - f = append(flags, newFlag) - } - resultFlagsSet = append(resultFlagsSet, f) - } - } - *flagsSet = resultFlagsSet -} - func CreateProxyServerLogFile(t *testing.T) string { proxyServerLogFile := path.Join(TestDir(), "proxy-server-log-"+GenerateRandomString(5)) _, err := os.Create(proxyServerLogFile) @@ -842,41 +784,6 @@ func AppendProxyEndpointToFlagSet(flagSet *[]string, port int) { *flagSet = append(*flagSet, "--custom-endpoint="+fmt.Sprintf("http://localhost:%d/storage/v1/", port)) } -// GetGCEZone returns the GCE zone of the current machine from -// the GCP resource detector. -func GetGCEZone(ctx context.Context) (string, error) { - detectedAttrs, err := resource.New(ctx, resource.WithDetectors(gcp.NewDetector())) - if err != nil { - return "", fmt.Errorf("failed to fetch GCP resource detector: %w", err) - } - attrs := detectedAttrs.Set() - if zoneValue, exists := attrs.Value("cloud.availability_zone"); exists { - zone := zoneValue.AsString() - // Confirm that the zone string is in right format e.g. us-central1-a. - if match, err := regexp.MatchString(zoneMatcherRegex, zone); !match || err != nil { - return zone, fmt.Errorf("zone %q returned by GCP resource detector is not a valid zone-string: %w", zone, err) - } - return zone, nil - } - return "", fmt.Errorf("cloud.availability_zone not found in GCP resource detector") -} - -// GetGCERegion return the GCE region for a given GCE zone. -// E.g. from us-central1-a, it returns us-central1. -func GetGCERegion(gceZone string) (string, error) { - indexOfLastHyphen := strings.LastIndex(gceZone, "-") - if indexOfLastHyphen < 0 { - return "", fmt.Errorf("input gceZone %q is not proper. It is expected to be of the form -- e.g. us-central1-a.", gceZone) - } - region := gceZone[:indexOfLastHyphen] - - // Confirm that the region string is in right format e.g. us-central1. - if match, err := regexp.MatchString(regionMatcherRegex, region); !match || err != nil { - return region, fmt.Errorf("zone %q returned by GCE metadata server is not a valid zone-string: %w", region, err) - } - return region, nil -} - // IsDynamicMount returns true if the mount is dynamic. // In dynamic mounts, rootDir contains all buckets, and mountDir is the specific bucket directory. func IsDynamicMount(mountDir, rootDir string) bool { diff --git a/tools/integration_tests/util/setup/yaml-config.go b/tools/integration_tests/util/setup/yaml-config.go deleted file mode 100644 index 997777e4e2..0000000000 --- a/tools/integration_tests/util/setup/yaml-config.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package setup - -import ( - "fmt" - "os" - "path" - - "gopkg.in/yaml.v3" -) - -func YAMLConfigFile(yamlContent any, fileName string) (filePath string) { - yamlData, err := yaml.Marshal(yamlContent) - if err != nil { - LogAndExit(fmt.Sprintf("Error while marshaling config file: %v", err)) - } - - filePath = path.Join(TestDir(), fileName) - err = os.WriteFile(filePath, yamlData, 0644) - if err != nil { - LogAndExit("Unable to write data into config file.") - } - return -} diff --git a/tools/integration_tests/util/test_suite/config.go b/tools/integration_tests/util/test_suite/config.go index 4ad3ce6355..35ebf04ca8 100644 --- a/tools/integration_tests/util/test_suite/config.go +++ b/tools/integration_tests/util/test_suite/config.go @@ -114,7 +114,7 @@ type Config struct { ReadGCSAlgo []TestConfig `yaml:"read_gcs_algo"` Interrupt []TestConfig `yaml:"interrupt"` UnfinalizedObject []TestConfig `yaml:"unfinalized_object"` - RapidAppends []TestConfig `yaml:"rapid_appends"` + RapidOperations []TestConfig `yaml:"rapid_operations"` MountTimeout []TestConfig `yaml:"mount_timeout"` Monitoring []TestConfig `yaml:"monitoring"` FlagOptimizations []TestConfig `yaml:"flag_optimizations"`