diff --git a/cfg/config.go b/cfg/config.go index 2edccc11681..922ebaf2e98 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -594,6 +594,8 @@ type FileSystemConfig struct { type GcsAuthConfig struct { AnonymousAccess bool `yaml:"anonymous-access"` + ExperimentalAuthTokenFile ResolvedPath `yaml:"experimental-auth-token-file"` + KeyFile ResolvedPath `yaml:"key-file"` ReuseTokenFromUrl bool `yaml:"reuse-token-from-url"` @@ -1027,6 +1029,8 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { return err } + flagSet.StringP("experimental-auth-token-file", "", "", "Experimental: Absolute path to a file containing an OAuth2 token response in JSON format ({\"access_token\": \"...\", \"expires_in\": 3600, \"token_type\": \"Bearer\"}), to be used directly as the GCS credential. Mounting fails if the token is expired. The file is re-read when the token expires, so an external process may refresh the credential by rewriting the file.") + flagSet.BoolP("experimental-enable-dentry-cache", "", false, "When enabled, it sets the Dentry cache entry timeout same as metadata-cache-ttl. This enables kernel to use cached entry to map the file paths to inodes, instead of making LookUpInode calls to GCSFuse.") if err := flagSet.MarkHidden("experimental-enable-dentry-cache"); err != nil { @@ -1640,6 +1644,10 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("gcs-auth.experimental-auth-token-file", flagSet.Lookup("experimental-auth-token-file")); err != nil { + return err + } + if err := v.BindPFlag("file-system.experimental-enable-dentry-cache", flagSet.Lookup("experimental-enable-dentry-cache")); err != nil { return err } diff --git a/cfg/params.yaml b/cfg/params.yaml index b297b142474..245a571d403 100644 --- a/cfg/params.yaml +++ b/cfg/params.yaml @@ -583,6 +583,16 @@ params: usage: "This flag disables authentication." default: false + - config-path: "gcs-auth.experimental-auth-token-file" + flag-name: "experimental-auth-token-file" + type: "resolvedPath" + usage: >- + Experimental: Absolute path to a file containing an OAuth2 token response + in JSON format ({"access_token": "...", "expires_in": 3600, "token_type": + "Bearer"}), to be used directly as the GCS credential. Mounting fails if + the token is expired. The file is re-read when the token expires, so an + external process may refresh the credential by rewriting the file. + - config-path: "gcs-auth.key-file" flag-name: "key-file" type: "resolvedPath" diff --git a/cfg/validate.go b/cfg/validate.go index bf174969466..2ed6ff0baa5 100644 --- a/cfg/validate.go +++ b/cfg/validate.go @@ -318,6 +318,18 @@ func isValidOptimizationProfile(config *Config) error { return nil } +func isValidGcsAuthConfig(config *GcsAuthConfig) error { + if config.ExperimentalAuthTokenFile != "" { + if config.KeyFile != "" { + return fmt.Errorf("experimental-auth-token-file cannot be used together with key-file") + } + if config.TokenUrl != "" { + return fmt.Errorf("experimental-auth-token-file cannot be used together with token-url") + } + } + return nil +} + // ValidateConfig returns a non-nil error if the config is invalid. func ValidateConfig(v *viper.Viper, config *Config) error { var err error @@ -342,6 +354,10 @@ func ValidateConfig(v *viper.Viper, config *Config) error { return fmt.Errorf("error parsing token-url config: %w", err) } + if err = isValidGcsAuthConfig(&config.GcsAuth); err != nil { + return fmt.Errorf("error parsing gcs-auth config: %w", err) + } + if err = isValidSequentialReadSizeMB(config.GcsConnection.SequentialReadSizeMb); err != nil { return fmt.Errorf("error parsing gcs-connection config: %w", err) } diff --git a/cfg/validate_test.go b/cfg/validate_test.go index 8e4b0d3b413..ee0c0affa5f 100644 --- a/cfg/validate_test.go +++ b/cfg/validate_test.go @@ -88,6 +88,29 @@ func TestValidateConfigSuccessful(t *testing.T) { }, }, }, + { + name: "Valid Config with experimental-auth-token-file set.", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsAuth: GcsAuthConfig{ + ExperimentalAuthTokenFile: "/token.json", + }, + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 200, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "disabled", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, { name: "Valid Config where input and expected custom endpoint differ.", config: &Config{ @@ -455,6 +478,40 @@ func TestValidateConfig_ErrorScenarios(t *testing.T) { }, }, }, + { + name: "Invalid Config due to experimental-auth-token-file set together with key-file", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsAuth: GcsAuthConfig{ + ExperimentalAuthTokenFile: "/token.json", + KeyFile: "/key.json", + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 200, + }, + }, + }, + { + name: "Invalid Config due to experimental-auth-token-file set together with token-url", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsAuth: GcsAuthConfig{ + ExperimentalAuthTokenFile: "/token.json", + TokenUrl: "https://www.abc.com", + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 200, + }, + }, + }, { name: "Sequential read size MB more than 1024 (max permissible value)", config: &Config{ diff --git a/cmd/legacy_main.go b/cmd/legacy_main.go index 46dc69360bb..3f110436961 100644 --- a/cmd/legacy_main.go +++ b/cmd/legacy_main.go @@ -154,6 +154,7 @@ func createStorageHandle(newConfig *cfg.Config, userAgent string, metricHandle m UserAgent: userAgent, CustomEndpoint: newConfig.GcsConnection.CustomEndpoint, KeyFile: string(newConfig.GcsAuth.KeyFile), + ExperimentalAuthTokenFile: string(newConfig.GcsAuth.ExperimentalAuthTokenFile), AnonymousAccess: newConfig.GcsAuth.AnonymousAccess, TokenUrl: newConfig.GcsAuth.TokenUrl, ReuseTokenFromUrl: newConfig.GcsAuth.ReuseTokenFromUrl, diff --git a/cmd/root_test.go b/cmd/root_test.go index 2174bc5d790..7126af17056 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -753,15 +753,26 @@ func TestArgsParsing_GCSAuthFlags(t *testing.T) { }, }, }, + { + name: "Test experimental-auth-token-file flag.", + args: []string{"gcsfuse", "--experimental-auth-token-file=token.json", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + GcsAuth: cfg.GcsAuthConfig{ + ExperimentalAuthTokenFile: cfg.ResolvedPath(path.Join(wd, "token.json")), + ReuseTokenFromUrl: true, + }, + }, + }, { name: "Test default gcs auth flags.", args: []string{"gcsfuse", "abc", "pqr"}, expectedConfig: &cfg.Config{ GcsAuth: cfg.GcsAuthConfig{ - AnonymousAccess: false, - KeyFile: "", - ReuseTokenFromUrl: true, - TokenUrl: "", + AnonymousAccess: false, + KeyFile: "", + ExperimentalAuthTokenFile: "", + ReuseTokenFromUrl: true, + TokenUrl: "", }, }, }, @@ -804,6 +815,14 @@ func TestArgsParsing_GCSAuthFlagsThrowsError(t *testing.T) { name: "Invalid value for token-url flag", args: []string{"gcsfuse", "--token-url=a_b://abc", "abc", "pqr"}, }, + { + name: "experimental-auth-token-file set together with key-file", + args: []string{"gcsfuse", "--experimental-auth-token-file=token.json", "--key-file=key.file", "abc", "pqr"}, + }, + { + name: "experimental-auth-token-file set together with token-url", + args: []string{"gcsfuse", "--experimental-auth-token-file=token.json", "--token-url=www.abc.com", "abc", "pqr"}, + }, } for _, tc := range tests { diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 1445652413e..6bc623414ae 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -88,6 +88,7 @@ func newTokenSourceFromPath(ctx context.Context, path string, scope string) (oau func GetTokenSource( ctx context.Context, keyFile string, + tokenFile string, tokenUrl string, reuseTokenFromUrl bool, ) (tokenSrc oauth2.TokenSource, err error) { @@ -95,7 +96,10 @@ func GetTokenSource( const scope = storagev1.DevstorageFullControlScope var method string - if keyFile != "" { + if tokenFile != "" { + tokenSrc, err = NewTokenSourceFromTokenFile(tokenFile) + method = "NewTokenSourceFromTokenFile" + } else if keyFile != "" { tokenSrc, err = newTokenSourceFromPath(ctx, keyFile, scope) method = "newTokenSourceFromPath" } else if tokenUrl != "" { diff --git a/internal/auth/file_token_source.go b/internal/auth/file_token_source.go new file mode 100644 index 00000000000..a4dea93e468 --- /dev/null +++ b/internal/auth/file_token_source.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "golang.org/x/oauth2" +) + +// fileTokenSource is an oauth2.TokenSource that reads an OAuth2 token +// response from a JSON file on disk. +type fileTokenSource struct { + path string +} + +func (ts fileTokenSource) Token() (*oauth2.Token, error) { + contents, err := os.ReadFile(ts.path) + if err != nil { + return nil, fmt.Errorf("fileTokenSource cannot read token file: %w", err) + } + + token := &oauth2.Token{} + if err := json.Unmarshal(contents, token); err != nil { + return nil, fmt.Errorf("fileTokenSource cannot decode token file %q: %w", ts.path, err) + } + + // The expires_in field is relative, so anchor it to the time the file was + // read to get an absolute expiry. An absolute expiry field, if present in + // the file, takes precedence. + if token.Expiry.IsZero() && token.ExpiresIn > 0 { + token.Expiry = time.Now().Add(time.Duration(token.ExpiresIn) * time.Second) + } + if !token.Valid() { + return nil, fmt.Errorf("token file %q contains an invalid or expired token", ts.path) + } + + return token, nil +} + +// NewTokenSourceFromTokenFile returns a TokenSource backed by a file +// containing an OAuth2 token response in JSON format, e.g. +// {"access_token": "...", "expires_in": 3600, "token_type": "Bearer"}. +// The file is read eagerly so that an invalid or expired token fails at mount +// time, and re-read when the token expires, so an external process may +// refresh the credential by rewriting the file. +func NewTokenSourceFromTokenFile(path string) (oauth2.TokenSource, error) { + ts := fileTokenSource{path: path} + token, err := ts.Token() + if err != nil { + return nil, err + } + return oauth2.ReuseTokenSource(token, ts), nil +} diff --git a/internal/auth/file_token_source_test.go b/internal/auth/file_token_source_test.go new file mode 100644 index 00000000000..1fe7ad0f8b6 --- /dev/null +++ b/internal/auth/file_token_source_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTokenFile(t *testing.T, contents string) string { + t.Helper() + tokenFile := path.Join(t.TempDir(), "token.json") + require.NoError(t, os.WriteFile(tokenFile, []byte(contents), 0o600)) + return tokenFile +} + +func Test_NewTokenSourceFromTokenFile_Success(t *testing.T) { + tokenFile := writeTokenFile(t, `{"access_token":"test-access-token","expires_in":3600,"token_type":"Bearer"}`) + + ts, err := NewTokenSourceFromTokenFile(tokenFile) + + require.NoError(t, err) + require.NotNil(t, ts) + token, err := ts.Token() + require.NoError(t, err) + assert.Equal(t, "test-access-token", token.AccessToken) + assert.Equal(t, "Bearer", token.TokenType) + // expires_in must be anchored to an absolute expiry. + assert.False(t, token.Expiry.IsZero()) + assert.True(t, token.Expiry.After(time.Now())) +} + +func Test_NewTokenSourceFromTokenFile_NoExpiry(t *testing.T) { + tokenFile := writeTokenFile(t, `{"access_token":"test-access-token","token_type":"Bearer"}`) + + ts, err := NewTokenSourceFromTokenFile(tokenFile) + + require.NoError(t, err) + token, err := ts.Token() + require.NoError(t, err) + assert.Equal(t, "test-access-token", token.AccessToken) + assert.True(t, token.Expiry.IsZero()) +} + +func Test_NewTokenSourceFromTokenFile_ExpiredToken(t *testing.T) { + expiry := time.Now().Add(-time.Hour).Format(time.RFC3339) + tokenFile := writeTokenFile(t, fmt.Sprintf(`{"access_token":"test-access-token","token_type":"Bearer","expiry":%q}`, expiry)) + + ts, err := NewTokenSourceFromTokenFile(tokenFile) + + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid or expired token") + assert.Nil(t, ts) +} + +func Test_NewTokenSourceFromTokenFile_MissingAccessToken(t *testing.T) { + tokenFile := writeTokenFile(t, `{"expires_in":3600,"token_type":"Bearer"}`) + + ts, err := NewTokenSourceFromTokenFile(tokenFile) + + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid or expired token") + assert.Nil(t, ts) +} + +func Test_NewTokenSourceFromTokenFile_InvalidJSON(t *testing.T) { + tokenFile := writeTokenFile(t, "not-json") + + ts, err := NewTokenSourceFromTokenFile(tokenFile) + + assert.Error(t, err) + assert.ErrorContains(t, err, "cannot decode token file") + assert.Nil(t, ts) +} + +func Test_NewTokenSourceFromTokenFile_FileNotFound(t *testing.T) { + ts, err := NewTokenSourceFromTokenFile(path.Join(t.TempDir(), "missing.json")) + + assert.Error(t, err) + assert.ErrorContains(t, err, "cannot read token file") + assert.Nil(t, ts) +} + +func TestFileTokenSource_RereadsFileOnEachCall(t *testing.T) { + tokenFile := writeTokenFile(t, `{"access_token":"token-1","token_type":"Bearer"}`) + ts := fileTokenSource{path: tokenFile} + token, err := ts.Token() + require.NoError(t, err) + require.Equal(t, "token-1", token.AccessToken) + + // An external process may refresh the credential by rewriting the file. + require.NoError(t, os.WriteFile(tokenFile, []byte(`{"access_token":"token-2","token_type":"Bearer"}`), 0o600)) + + token, err = ts.Token() + require.NoError(t, err) + assert.Equal(t, "token-2", token.AccessToken) +} diff --git a/internal/storage/storageutil/auth_client_option.go b/internal/storage/storageutil/auth_client_option.go index c2f4e1cc6b0..8c401abce7f 100644 --- a/internal/storage/storageutil/auth_client_option.go +++ b/internal/storage/storageutil/auth_client_option.go @@ -28,8 +28,19 @@ import ( "google.golang.org/api/option" ) -// GetClientAuthOptionsAndToken returns client options and a token source using either a token URL or fallback to key file/ADC. +// GetClientAuthOptionsAndToken returns client options and a token source using either an auth token file, a token URL or fallback to key file/ADC. func GetClientAuthOptionsAndToken(ctx context.Context, config *StorageClientConfig) ([]option.ClientOption, oauth2.TokenSource, error) { + // If an auth token file is provided, use its token directly as the credential. + if config.ExperimentalAuthTokenFile != "" { + tokenSrc, err := auth2.NewTokenSourceFromTokenFile(config.ExperimentalAuthTokenFile) + if err != nil { + return nil, nil, fmt.Errorf("while creating token source from token file: %w", err) + } + + clientOpts := []option.ClientOption{option.WithTokenSource(tokenSrc)} + return clientOpts, tokenSrc, nil + } + // If Token URL is provided, attempt to fetch token source directly. if config.TokenUrl != "" { tokenSrc, err := auth2.NewTokenSourceFromURL(ctx, config.TokenUrl, config.ReuseTokenFromUrl) diff --git a/internal/storage/storageutil/auth_client_option_test.go b/internal/storage/storageutil/auth_client_option_test.go index 9b2c295ecf3..e78442c3503 100644 --- a/internal/storage/storageutil/auth_client_option_test.go +++ b/internal/storage/storageutil/auth_client_option_test.go @@ -19,9 +19,12 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" + "path" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/api/option" ) @@ -29,6 +32,33 @@ import ( // Tests //////////////////////////////////////////////////////////////////////// +func Test_GetClientAuthOptionsAndToken_AuthTokenFileSuccess(t *testing.T) { + tokenFile := path.Join(t.TempDir(), "token.json") + require.NoError(t, os.WriteFile(tokenFile, []byte(`{"access_token":"dummy-token","expires_in":3600,"token_type":"Bearer"}`), 0o600)) + config := &StorageClientConfig{ + ExperimentalAuthTokenFile: tokenFile, + } + + clientOpts, tokenSrc, err := GetClientAuthOptionsAndToken(context.TODO(), config) + + assert.NoError(t, err) + assert.NotNil(t, tokenSrc) + assert.Len(t, clientOpts, 1) // Only tokenSource option attached + token, err := tokenSrc.Token() + assert.NoError(t, err) + assert.Equal(t, "dummy-token", token.AccessToken) +} + +func Test_GetClientAuthOptionsAndToken_AuthTokenFileError(t *testing.T) { + config := &StorageClientConfig{ExperimentalAuthTokenFile: path.Join(t.TempDir(), "missing.json")} + + clientOpts, tokenSrc, err := GetClientAuthOptionsAndToken(context.TODO(), config) + + assert.Error(t, err) + assert.Nil(t, tokenSrc) + assert.Empty(t, clientOpts) +} + func Test_GetClientAuthOptionsAndToken_TokenUrlSuccess(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/storage/storageutil/client.go b/internal/storage/storageutil/client.go index 2022e650a76..6931542d930 100644 --- a/internal/storage/storageutil/client.go +++ b/internal/storage/storageutil/client.go @@ -15,6 +15,7 @@ package storageutil import ( + "context" "crypto/tls" "fmt" "net" @@ -30,7 +31,6 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" - "golang.org/x/net/context" "golang.org/x/oauth2" ) @@ -56,6 +56,7 @@ type StorageClientConfig struct { UserAgent string CustomEndpoint string KeyFile string + ExperimentalAuthTokenFile string TokenUrl string ReuseTokenFromUrl bool ExperimentalNonrapidFolderApiStallRetry bool @@ -181,7 +182,7 @@ func CreateHttpClient(storageClientConfig *StorageClientConfig, tokenSrc oauth2. // It creates the token-source from the provided // key-file or using ADC search order (https://cloud.google.com/docs/authentication/application-default-credentials#order). func CreateTokenSource(storageClientConfig *StorageClientConfig) (tokenSrc oauth2.TokenSource, err error) { - return auth.GetTokenSource(context.Background(), storageClientConfig.KeyFile, storageClientConfig.TokenUrl, storageClientConfig.ReuseTokenFromUrl) + return auth.GetTokenSource(context.Background(), storageClientConfig.KeyFile, storageClientConfig.ExperimentalAuthTokenFile, storageClientConfig.TokenUrl, storageClientConfig.ReuseTokenFromUrl) } // StripScheme strips the scheme part of given url.